diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2021-06-13 19:31:46 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2021-06-13 19:37:19 +0000 |
commit | e8d8bef961a50d4dc22501cde4fb9fb0be1b2532 (patch) | |
tree | 94f04805f47bb7c59ae29690d8952b6074fff602 /contrib/llvm-project/llvm/lib/ExecutionEngine/Orc | |
parent | bb130ff39747b94592cb26d71b7cb097b9a4ea6b (diff) | |
parent | b60736ec1405bb0a8dd40989f67ef4c93da068ab (diff) |
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ExecutionEngine/Orc')
32 files changed, 3629 insertions, 2898 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index 9e38dc36faae..68878f6729e9 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -73,22 +73,21 @@ class PartitioningIRMaterializationUnit : public IRMaterializationUnit { public: PartitioningIRMaterializationUnit(ExecutionSession &ES, const IRSymbolMapper::ManglingOptions &MO, - ThreadSafeModule TSM, VModuleKey K, + ThreadSafeModule TSM, CompileOnDemandLayer &Parent) - : IRMaterializationUnit(ES, MO, std::move(TSM), std::move(K)), - Parent(Parent) {} + : IRMaterializationUnit(ES, MO, std::move(TSM)), Parent(Parent) {} PartitioningIRMaterializationUnit( - ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, + ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition, CompileOnDemandLayer &Parent) - : IRMaterializationUnit(std::move(TSM), std::move(K), - std::move(SymbolFlags), std::move(InitSymbol), + : IRMaterializationUnit(std::move(TSM), std::move(SymbolFlags), + std::move(InitSymbol), std::move(SymbolToDefinition)), Parent(Parent) {} private: - void materialize(MaterializationResponsibility R) override { + void materialize(std::unique_ptr<MaterializationResponsibility> R) override { Parent.emitPartition(std::move(R), std::move(TSM), std::move(SymbolToDefinition)); } @@ -128,15 +127,15 @@ void CompileOnDemandLayer::setPartitionFunction(PartitionFunction Partition) { void CompileOnDemandLayer::setImplMap(ImplSymbolMap *Imp) { this->AliaseeImpls = Imp; } -void CompileOnDemandLayer::emit(MaterializationResponsibility R, - ThreadSafeModule TSM) { +void CompileOnDemandLayer::emit( + std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM) { assert(TSM && "Null module"); auto &ES = getExecutionSession(); // Sort the callables and non-callables, build re-exports and lodge the // actual module with the implementation dylib. - auto &PDR = getPerDylibResources(R.getTargetJITDylib()); + auto &PDR = getPerDylibResources(R->getTargetJITDylib()); SymbolAliasMap NonCallables; SymbolAliasMap Callables; @@ -145,7 +144,7 @@ void CompileOnDemandLayer::emit(MaterializationResponsibility R, cleanUpModule(M); }); - for (auto &KV : R.getSymbols()) { + for (auto &KV : R->getSymbols()) { auto &Name = KV.first; auto &Flags = KV.second; if (Flags.isCallable()) @@ -158,19 +157,29 @@ void CompileOnDemandLayer::emit(MaterializationResponsibility R, // implementation dylib. if (auto Err = PDR.getImplDylib().define( std::make_unique<PartitioningIRMaterializationUnit>( - ES, *getManglingOptions(), std::move(TSM), R.getVModuleKey(), - *this))) { + ES, *getManglingOptions(), std::move(TSM), *this))) { ES.reportError(std::move(Err)); - R.failMaterialization(); + R->failMaterialization(); return; } if (!NonCallables.empty()) - R.replace(reexports(PDR.getImplDylib(), std::move(NonCallables), - JITDylibLookupFlags::MatchAllSymbols)); - if (!Callables.empty()) - R.replace(lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(), - std::move(Callables), AliaseeImpls)); + if (auto Err = + R->replace(reexports(PDR.getImplDylib(), std::move(NonCallables), + JITDylibLookupFlags::MatchAllSymbols))) { + getExecutionSession().reportError(std::move(Err)); + R->failMaterialization(); + return; + } + if (!Callables.empty()) { + if (auto Err = R->replace( + lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(), + std::move(Callables), AliaseeImpls))) { + getExecutionSession().reportError(std::move(Err)); + R->failMaterialization(); + return; + } + } } CompileOnDemandLayer::PerDylibResources & @@ -247,7 +256,7 @@ void CompileOnDemandLayer::expandPartition(GlobalValueSet &Partition) { } void CompileOnDemandLayer::emitPartition( - MaterializationResponsibility R, ThreadSafeModule TSM, + std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM, IRMaterializationUnit::SymbolNameToDefinitionMap Defs) { // FIXME: Need a 'notify lazy-extracting/emitting' callback to tie the @@ -257,8 +266,8 @@ void CompileOnDemandLayer::emitPartition( auto &ES = getExecutionSession(); GlobalValueSet RequestedGVs; - for (auto &Name : R.getRequestedSymbols()) { - if (Name == R.getInitializerSymbol()) + for (auto &Name : R->getRequestedSymbols()) { + if (Name == R->getInitializerSymbol()) TSM.withModuleDo([&](Module &M) { for (auto &GV : getStaticInitGVs(M)) RequestedGVs.insert(&GV); @@ -285,9 +294,14 @@ void CompileOnDemandLayer::emitPartition( // If the partition is empty, return the whole module to the symbol table. if (GVsToExtract->empty()) { - R.replace(std::make_unique<PartitioningIRMaterializationUnit>( - std::move(TSM), R.getVModuleKey(), R.getSymbols(), - R.getInitializerSymbol(), std::move(Defs), *this)); + if (auto Err = + R->replace(std::make_unique<PartitioningIRMaterializationUnit>( + std::move(TSM), R->getSymbols(), R->getInitializerSymbol(), + std::move(Defs), *this))) { + getExecutionSession().reportError(std::move(Err)); + R->failMaterialization(); + return; + } return; } @@ -308,7 +322,7 @@ void CompileOnDemandLayer::emitPartition( IRSymbolMapper::add(ES, *getManglingOptions(), PromotedGlobals, SymbolFlags); - if (auto Err = R.defineMaterializing(SymbolFlags)) + if (auto Err = R->defineMaterializing(SymbolFlags)) return std::move(Err); } @@ -348,12 +362,16 @@ void CompileOnDemandLayer::emitPartition( if (!ExtractedTSM) { ES.reportError(ExtractedTSM.takeError()); - R.failMaterialization(); + R->failMaterialization(); return; } - R.replace(std::make_unique<PartitioningIRMaterializationUnit>( - ES, *getManglingOptions(), std::move(TSM), R.getVModuleKey(), *this)); + if (auto Err = R->replace(std::make_unique<PartitioningIRMaterializationUnit>( + ES, *getManglingOptions(), std::move(TSM), *this))) { + ES.reportError(std::move(Err)); + R->failMaterialization(); + return; + } BaseLayer.emit(std::move(R), std::move(*ExtractedTSM)); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp index bad13cfebbc6..dfb558808c32 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -11,18 +11,19 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/Config/llvm-config.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" -#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/MSVCErrorWorkarounds.h" #include <condition_variable> -#if LLVM_ENABLE_THREADS #include <future> -#endif #define DEBUG_TYPE "orc" namespace llvm { namespace orc { +char ResourceTrackerDefunct::ID = 0; char FailedToMaterialize::ID = 0; char SymbolsNotFound::ID = 0; char SymbolsCouldNotBeRemoved::ID = 0; @@ -34,6 +35,45 @@ RegisterDependenciesFunction NoDependenciesToRegister = void MaterializationUnit::anchor() {} +ResourceTracker::ResourceTracker(JITDylibSP JD) { + assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 && + "JITDylib must be two byte aligned"); + JD->Retain(); + JDAndFlag.store(reinterpret_cast<uintptr_t>(JD.get())); +} + +ResourceTracker::~ResourceTracker() { + getJITDylib().getExecutionSession().destroyResourceTracker(*this); + getJITDylib().Release(); +} + +Error ResourceTracker::remove() { + return getJITDylib().getExecutionSession().removeResourceTracker(*this); +} + +void ResourceTracker::transferTo(ResourceTracker &DstRT) { + getJITDylib().getExecutionSession().transferResourceTracker(DstRT, *this); +} + +void ResourceTracker::makeDefunct() { + uintptr_t Val = JDAndFlag.load(); + Val |= 0x1U; + JDAndFlag.store(Val); +} + +ResourceManager::~ResourceManager() {} + +ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT) + : RT(std::move(RT)) {} + +std::error_code ResourceTrackerDefunct::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void ResourceTrackerDefunct::log(raw_ostream &OS) const { + OS << "Resource tracker " << (void *)RT.get() << " became defunct"; +} + FailedToMaterialize::FailedToMaterialize( std::shared_ptr<SymbolDependenceMap> Symbols) : Symbols(std::move(Symbols)) { @@ -137,8 +177,6 @@ void AsynchronousSymbolQuery::handleComplete() { TmpNotifyComplete(std::move(ResolvedSymbols)); } -bool AsynchronousSymbolQuery::canStillFail() { return !!NotifyComplete; } - void AsynchronousSymbolQuery::handleFailed(Error Err) { assert(QueryRegistrations.empty() && ResolvedSymbols.empty() && OutstandingSymbolsCount == 0 && @@ -181,156 +219,9 @@ void AsynchronousSymbolQuery::detach() { QueryRegistrations.clear(); } -MaterializationResponsibility::~MaterializationResponsibility() { - assert(SymbolFlags.empty() && - "All symbols should have been explicitly materialized or failed"); -} - -SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { - return JD->getRequestedSymbols(SymbolFlags); -} - -Error MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { - LLVM_DEBUG({ - dbgs() << "In " << JD->getName() << " resolving " << Symbols << "\n"; - }); -#ifndef NDEBUG - for (auto &KV : Symbols) { - auto WeakFlags = JITSymbolFlags::Weak | JITSymbolFlags::Common; - auto I = SymbolFlags.find(KV.first); - assert(I != SymbolFlags.end() && - "Resolving symbol outside this responsibility set"); - assert(!I->second.hasMaterializationSideEffectsOnly() && - "Can't resolve materialization-side-effects-only symbol"); - assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) && - "Resolving symbol with incorrect flags"); - } -#endif - - return JD->resolve(Symbols); -} - -Error MaterializationResponsibility::notifyEmitted() { - - LLVM_DEBUG({ - dbgs() << "In " << JD->getName() << " emitting " << SymbolFlags << "\n"; - }); - - if (auto Err = JD->emit(SymbolFlags)) - return Err; - - SymbolFlags.clear(); - return Error::success(); -} - -Error MaterializationResponsibility::defineMaterializing( - SymbolFlagsMap NewSymbolFlags) { - - LLVM_DEBUG({ - dbgs() << "In " << JD->getName() << " defining materializing symbols " - << NewSymbolFlags << "\n"; - }); - if (auto AcceptedDefs = JD->defineMaterializing(std::move(NewSymbolFlags))) { - // Add all newly accepted symbols to this responsibility object. - for (auto &KV : *AcceptedDefs) - SymbolFlags.insert(KV); - return Error::success(); - } else - return AcceptedDefs.takeError(); -} - -void MaterializationResponsibility::failMaterialization() { - - LLVM_DEBUG({ - dbgs() << "In " << JD->getName() << " failing materialization for " - << SymbolFlags << "\n"; - }); - - JITDylib::FailedSymbolsWorklist Worklist; - - for (auto &KV : SymbolFlags) - Worklist.push_back(std::make_pair(JD.get(), KV.first)); - SymbolFlags.clear(); - - JD->notifyFailed(std::move(Worklist)); -} - -void MaterializationResponsibility::replace( - std::unique_ptr<MaterializationUnit> MU) { - - // If the replacement MU is empty then return. - if (MU->getSymbols().empty()) - return; - - for (auto &KV : MU->getSymbols()) { - assert(SymbolFlags.count(KV.first) && - "Replacing definition outside this responsibility set"); - SymbolFlags.erase(KV.first); - } - - if (MU->getInitializerSymbol() == InitSymbol) - InitSymbol = nullptr; - - LLVM_DEBUG(JD->getExecutionSession().runSessionLocked([&]() { - dbgs() << "In " << JD->getName() << " replacing symbols with " << *MU - << "\n"; - });); - - JD->replace(std::move(MU)); -} - -MaterializationResponsibility -MaterializationResponsibility::delegate(const SymbolNameSet &Symbols, - VModuleKey NewKey) { - - if (NewKey == VModuleKey()) - NewKey = K; - - SymbolStringPtr DelegatedInitSymbol; - SymbolFlagsMap DelegatedFlags; - - for (auto &Name : Symbols) { - auto I = SymbolFlags.find(Name); - assert(I != SymbolFlags.end() && - "Symbol is not tracked by this MaterializationResponsibility " - "instance"); - - DelegatedFlags[Name] = std::move(I->second); - if (Name == InitSymbol) - std::swap(InitSymbol, DelegatedInitSymbol); - - SymbolFlags.erase(I); - } - - return MaterializationResponsibility(JD, std::move(DelegatedFlags), - std::move(DelegatedInitSymbol), - std::move(NewKey)); -} - -void MaterializationResponsibility::addDependencies( - const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { - LLVM_DEBUG({ - dbgs() << "Adding dependencies for " << Name << ": " << Dependencies - << "\n"; - }); - assert(SymbolFlags.count(Name) && - "Symbol not covered by this MaterializationResponsibility instance"); - JD->addDependencies(Name, Dependencies); -} - -void MaterializationResponsibility::addDependenciesForAll( - const SymbolDependenceMap &Dependencies) { - LLVM_DEBUG({ - dbgs() << "Adding dependencies for all symbols in " << SymbolFlags << ": " - << Dependencies << "\n"; - }); - for (auto &KV : SymbolFlags) - JD->addDependencies(KV.first, Dependencies); -} - AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( - SymbolMap Symbols, VModuleKey K) - : MaterializationUnit(extractFlags(Symbols), nullptr, std::move(K)), + SymbolMap Symbols) + : MaterializationUnit(extractFlags(Symbols), nullptr), Symbols(std::move(Symbols)) {} StringRef AbsoluteSymbolsMaterializationUnit::getName() const { @@ -338,10 +229,10 @@ StringRef AbsoluteSymbolsMaterializationUnit::getName() const { } void AbsoluteSymbolsMaterializationUnit::materialize( - MaterializationResponsibility R) { + std::unique_ptr<MaterializationResponsibility> R) { // No dependencies, so these calls can't fail. - cantFail(R.notifyResolved(Symbols)); - cantFail(R.notifyEmitted()); + cantFail(R->notifyResolved(Symbols)); + cantFail(R->notifyEmitted()); } void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD, @@ -360,26 +251,25 @@ AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { ReExportsMaterializationUnit::ReExportsMaterializationUnit( JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, - SymbolAliasMap Aliases, VModuleKey K) - : MaterializationUnit(extractFlags(Aliases), nullptr, std::move(K)), - SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), - Aliases(std::move(Aliases)) {} + SymbolAliasMap Aliases) + : MaterializationUnit(extractFlags(Aliases), nullptr), SourceJD(SourceJD), + SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {} StringRef ReExportsMaterializationUnit::getName() const { return "<Reexports>"; } void ReExportsMaterializationUnit::materialize( - MaterializationResponsibility R) { + std::unique_ptr<MaterializationResponsibility> R) { - auto &ES = R.getTargetJITDylib().getExecutionSession(); - JITDylib &TgtJD = R.getTargetJITDylib(); + auto &ES = R->getTargetJITDylib().getExecutionSession(); + JITDylib &TgtJD = R->getTargetJITDylib(); JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD; // Find the set of requested aliases and aliasees. Return any unrequested // aliases back to the JITDylib so as to not prematurely materialize any // aliasees. - auto RequestedSymbols = R.getRequestedSymbols(); + auto RequestedSymbols = R->getRequestedSymbols(); SymbolAliasMap RequestedAliases; for (auto &Name : RequestedSymbols) { @@ -398,19 +288,27 @@ void ReExportsMaterializationUnit::materialize( }); if (!Aliases.empty()) { - if (SourceJD) - R.replace(reexports(*SourceJD, std::move(Aliases), SourceJDLookupFlags)); - else - R.replace(symbolAliases(std::move(Aliases))); + auto Err = SourceJD ? R->replace(reexports(*SourceJD, std::move(Aliases), + SourceJDLookupFlags)) + : R->replace(symbolAliases(std::move(Aliases))); + + if (Err) { + // FIXME: Should this be reported / treated as failure to materialize? + // Or should this be treated as a sanctioned bailing-out? + ES.reportError(std::move(Err)); + R->failMaterialization(); + return; + } } // The OnResolveInfo struct will hold the aliases and responsibilty for each // query in the list. struct OnResolveInfo { - OnResolveInfo(MaterializationResponsibility R, SymbolAliasMap Aliases) + OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R, + SymbolAliasMap Aliases) : R(std::move(R)), Aliases(std::move(Aliases)) {} - MaterializationResponsibility R; + std::unique_ptr<MaterializationResponsibility> R; SymbolAliasMap Aliases; }; @@ -450,8 +348,15 @@ void ReExportsMaterializationUnit::materialize( assert(!QuerySymbols.empty() && "Alias cycle detected!"); - auto QueryInfo = std::make_shared<OnResolveInfo>( - R.delegate(ResponsibilitySymbols), std::move(QueryAliases)); + auto NewR = R->delegate(ResponsibilitySymbols); + if (!NewR) { + ES.reportError(NewR.takeError()); + R->failMaterialization(); + return; + } + + auto QueryInfo = std::make_shared<OnResolveInfo>(std::move(*NewR), + std::move(QueryAliases)); QueryInfos.push_back( make_pair(std::move(QuerySymbols), std::move(QueryInfo))); } @@ -480,12 +385,12 @@ void ReExportsMaterializationUnit::materialize( for (auto &KV : QueryInfo->Aliases) if (SrcJDDeps.count(KV.second.Aliasee)) { PerAliasDeps = {KV.second.Aliasee}; - QueryInfo->R.addDependencies(KV.first, PerAliasDepsMap); + QueryInfo->R->addDependencies(KV.first, PerAliasDepsMap); } }; auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) { - auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession(); + auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession(); if (Result) { SymbolMap ResolutionMap; for (auto &KV : QueryInfo->Aliases) { @@ -499,19 +404,19 @@ void ReExportsMaterializationUnit::materialize( ResolutionMap[KV.first] = JITEvaluatedSymbol( (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); } - if (auto Err = QueryInfo->R.notifyResolved(ResolutionMap)) { + if (auto Err = QueryInfo->R->notifyResolved(ResolutionMap)) { ES.reportError(std::move(Err)); - QueryInfo->R.failMaterialization(); + QueryInfo->R->failMaterialization(); return; } - if (auto Err = QueryInfo->R.notifyEmitted()) { + if (auto Err = QueryInfo->R->notifyEmitted()) { ES.reportError(std::move(Err)); - QueryInfo->R.failMaterialization(); + QueryInfo->R->failMaterialization(); return; } } else { ES.reportError(Result.takeError()); - QueryInfo->R.failMaterialization(); + QueryInfo->R->failMaterialization(); } }; @@ -538,20 +443,16 @@ ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { return SymbolFlags; } -Expected<SymbolAliasMap> -buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { +Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD, + SymbolNameSet Symbols) { SymbolLookupSet LookupSet(Symbols); - auto Flags = SourceJD.lookupFlags( - LookupKind::Static, JITDylibLookupFlags::MatchAllSymbols, LookupSet); + auto Flags = SourceJD.getExecutionSession().lookupFlags( + LookupKind::Static, {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}}, + SymbolLookupSet(std::move(Symbols))); if (!Flags) return Flags.takeError(); - if (!LookupSet.empty()) { - LookupSet.sortByName(); - return make_error<SymbolsNotFound>(LookupSet.getSymbolNames()); - } - SymbolAliasMap Result; for (auto &Name : Symbols) { assert(Flags->count(Name) && "Missing entry in flags map"); @@ -561,19 +462,100 @@ buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { return Result; } +class InProgressLookupState { +public: + InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet LookupSet, SymbolState RequiredState) + : K(K), SearchOrder(std::move(SearchOrder)), + LookupSet(std::move(LookupSet)), RequiredState(RequiredState) { + DefGeneratorCandidates = this->LookupSet; + } + virtual ~InProgressLookupState() {} + virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0; + virtual void fail(Error Err) = 0; + + LookupKind K; + JITDylibSearchOrder SearchOrder; + SymbolLookupSet LookupSet; + SymbolState RequiredState; + + std::unique_lock<std::mutex> GeneratorLock; + size_t CurSearchOrderIndex = 0; + bool NewJITDylib = true; + SymbolLookupSet DefGeneratorCandidates; + SymbolLookupSet DefGeneratorNonCandidates; + std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack; +}; + +class InProgressLookupFlagsState : public InProgressLookupState { +public: + InProgressLookupFlagsState( + LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) + : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet), + SymbolState::NeverSearched), + OnComplete(std::move(OnComplete)) {} + + void complete(std::unique_ptr<InProgressLookupState> IPLS) override { + GeneratorLock = {}; // Unlock and release. + auto &ES = SearchOrder.front().first->getExecutionSession(); + ES.OL_completeLookupFlags(std::move(IPLS), std::move(OnComplete)); + } + + void fail(Error Err) override { + GeneratorLock = {}; // Unlock and release. + OnComplete(std::move(Err)); + } + +private: + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete; +}; + +class InProgressFullLookupState : public InProgressLookupState { +public: + InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet LookupSet, + SymbolState RequiredState, + std::shared_ptr<AsynchronousSymbolQuery> Q, + RegisterDependenciesFunction RegisterDependencies) + : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet), + RequiredState), + Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) { + } + + void complete(std::unique_ptr<InProgressLookupState> IPLS) override { + GeneratorLock = {}; // Unlock and release. + auto &ES = SearchOrder.front().first->getExecutionSession(); + ES.OL_completeLookup(std::move(IPLS), std::move(Q), + std::move(RegisterDependencies)); + } + + void fail(Error Err) override { + GeneratorLock = {}; + Q->detach(); + Q->handleFailed(std::move(Err)); + } + +private: + std::shared_ptr<AsynchronousSymbolQuery> Q; + RegisterDependenciesFunction RegisterDependencies; +}; + ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolPredicate Allow) : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), Allow(std::move(Allow)) {} -Error ReexportsGenerator::tryToGenerate(LookupKind K, JITDylib &JD, +Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K, + JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &LookupSet) { assert(&JD != &SourceJD && "Cannot re-export from the same dylib"); // Use lookupFlags to find the subset of symbols that match our lookup. - auto Flags = SourceJD.lookupFlags(K, JDLookupFlags, LookupSet); + auto Flags = JD.getExecutionSession().lookupFlags( + K, {{&SourceJD, JDLookupFlags}}, LookupSet); if (!Flags) return Flags.takeError(); @@ -590,19 +572,63 @@ Error ReexportsGenerator::tryToGenerate(LookupKind K, JITDylib &JD, return JD.define(reexports(SourceJD, AliasMap, SourceJDLookupFlags)); } -JITDylib::DefinitionGenerator::~DefinitionGenerator() {} +LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS) + : IPLS(std::move(IPLS)) {} -void JITDylib::removeGenerator(DefinitionGenerator &G) { +void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(IPLS); } + +LookupState::LookupState() = default; +LookupState::LookupState(LookupState &&) = default; +LookupState &LookupState::operator=(LookupState &&) = default; +LookupState::~LookupState() = default; + +void LookupState::continueLookup(Error Err) { + assert(IPLS && "Cannot call continueLookup on empty LookupState"); + auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession(); + ES.OL_applyQueryPhase1(std::move(IPLS), std::move(Err)); +} + +DefinitionGenerator::~DefinitionGenerator() {} + +Error JITDylib::clear() { + std::vector<ResourceTrackerSP> TrackersToRemove; ES.runSessionLocked([&]() { - auto I = std::find_if(DefGenerators.begin(), DefGenerators.end(), - [&](const std::unique_ptr<DefinitionGenerator> &H) { - return H.get() == &G; - }); - assert(I != DefGenerators.end() && "Generator not found"); - DefGenerators.erase(I); + for (auto &KV : TrackerSymbols) + TrackersToRemove.push_back(KV.first); + TrackersToRemove.push_back(getDefaultResourceTracker()); + }); + + Error Err = Error::success(); + for (auto &RT : TrackersToRemove) + Err = joinErrors(std::move(Err), RT->remove()); + return Err; +} + +ResourceTrackerSP JITDylib::getDefaultResourceTracker() { + return ES.runSessionLocked([this] { + if (!DefaultTracker) + DefaultTracker = new ResourceTracker(this); + return DefaultTracker; + }); +} + +ResourceTrackerSP JITDylib::createResourceTracker() { + return ES.runSessionLocked([this] { + ResourceTrackerSP RT = new ResourceTracker(this); + return RT; }); } +void JITDylib::removeGenerator(DefinitionGenerator &G) { + std::lock_guard<std::mutex> Lock(GeneratorsMutex); + auto I = llvm::find_if(DefGenerators, + [&](const std::shared_ptr<DefinitionGenerator> &H) { + return H.get() == &G; + }); + assert(I != DefGenerators.end() && "Generator not found"); + DefGenerators.erase(I); +} + Expected<SymbolFlagsMap> JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { @@ -652,11 +678,18 @@ JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { }); } -void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { +Error JITDylib::replace(MaterializationResponsibility &FromMR, + std::unique_ptr<MaterializationUnit> MU) { assert(MU != nullptr && "Can not replace with a null MaterializationUnit"); + std::unique_ptr<MaterializationUnit> MustRunMU; + std::unique_ptr<MaterializationResponsibility> MustRunMR; + + auto Err = + ES.runSessionLocked([&, this]() -> Error { + auto RT = getTracker(FromMR); - auto MustRunMU = - ES.runSessionLocked([&, this]() -> std::unique_ptr<MaterializationUnit> { + if (RT->isDefunct()) + return make_error<ResourceTrackerDefunct>(std::move(RT)); #ifndef NDEBUG for (auto &KV : MU->getSymbols()) { @@ -671,18 +704,27 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { } #endif // NDEBUG + // If the tracker is defunct we need to bail out immediately. + // If any symbol has pending queries against it then we need to // materialize MU immediately. for (auto &KV : MU->getSymbols()) { auto MII = MaterializingInfos.find(KV.first); if (MII != MaterializingInfos.end()) { - if (MII->second.hasQueriesPending()) - return std::move(MU); + if (MII->second.hasQueriesPending()) { + MustRunMR = ES.createMaterializationResponsibility( + *RT, std::move(MU->SymbolFlags), std::move(MU->InitSymbol)); + MustRunMU = std::move(MU); + return Error::success(); + } } } // Otherwise, make MU responsible for all the symbols. - auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); + auto RTI = MRTrackers.find(&FromMR); + assert(RTI != MRTrackers.end() && "No tracker for FromMR"); + auto UMI = + std::make_shared<UnmaterializedInfo>(std::move(MU), RTI->second); for (auto &KV : UMI->MU->getSymbols()) { auto SymI = Symbols.find(KV.first); assert(SymI->second.getState() == SymbolState::Materializing && @@ -700,14 +742,36 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { UMIEntry = UMI; } - return nullptr; + return Error::success(); }); + if (Err) + return Err; + if (MustRunMU) { - auto MR = - MustRunMU->createMaterializationResponsibility(shared_from_this()); - ES.dispatchMaterialization(std::move(MustRunMU), std::move(MR)); + assert(MustRunMR && "MustRunMU set implies MustRunMR set"); + ES.dispatchMaterialization(std::move(MustRunMU), std::move(MustRunMR)); + } else { + assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset"); } + + return Error::success(); +} + +Expected<std::unique_ptr<MaterializationResponsibility>> +JITDylib::delegate(MaterializationResponsibility &FromMR, + SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) { + + return ES.runSessionLocked( + [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> { + auto RT = getTracker(FromMR); + + if (RT->isDefunct()) + return make_error<ResourceTrackerDefunct>(std::move(RT)); + + return ES.createMaterializationResponsibility( + *RT, std::move(SymbolFlags), std::move(InitSymbol)); + }); } SymbolNameSet @@ -808,89 +872,93 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, Symbols[Name].setFlags(Symbols[Name].getFlags() | JITSymbolFlags::HasError); } -Error JITDylib::resolve(const SymbolMap &Resolved) { - SymbolNameSet SymbolsInErrorState; +Error JITDylib::resolve(MaterializationResponsibility &MR, + const SymbolMap &Resolved) { AsynchronousSymbolQuerySet CompletedQueries; - ES.runSessionLocked([&, this]() { - struct WorklistEntry { - SymbolTable::iterator SymI; - JITEvaluatedSymbol ResolvedSym; - }; - - std::vector<WorklistEntry> Worklist; - Worklist.reserve(Resolved.size()); + if (auto Err = ES.runSessionLocked([&, this]() -> Error { + auto RTI = MRTrackers.find(&MR); + assert(RTI != MRTrackers.end() && "No resource tracker for MR?"); + if (RTI->second->isDefunct()) + return make_error<ResourceTrackerDefunct>(RTI->second); - // Build worklist and check for any symbols in the error state. - for (const auto &KV : Resolved) { + struct WorklistEntry { + SymbolTable::iterator SymI; + JITEvaluatedSymbol ResolvedSym; + }; - assert(!KV.second.getFlags().hasError() && - "Resolution result can not have error flag set"); + SymbolNameSet SymbolsInErrorState; + std::vector<WorklistEntry> Worklist; + Worklist.reserve(Resolved.size()); - auto SymI = Symbols.find(KV.first); + // Build worklist and check for any symbols in the error state. + for (const auto &KV : Resolved) { - assert(SymI != Symbols.end() && "Symbol not found"); - assert(!SymI->second.hasMaterializerAttached() && - "Resolving symbol with materializer attached?"); - assert(SymI->second.getState() == SymbolState::Materializing && - "Symbol should be materializing"); - assert(SymI->second.getAddress() == 0 && - "Symbol has already been resolved"); + assert(!KV.second.getFlags().hasError() && + "Resolution result can not have error flag set"); - if (SymI->second.getFlags().hasError()) - SymbolsInErrorState.insert(KV.first); - else { - auto Flags = KV.second.getFlags(); - Flags &= ~(JITSymbolFlags::Weak | JITSymbolFlags::Common); - assert(Flags == (SymI->second.getFlags() & - ~(JITSymbolFlags::Weak | JITSymbolFlags::Common)) && - "Resolved flags should match the declared flags"); - - Worklist.push_back( - {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)}); - } - } + auto SymI = Symbols.find(KV.first); - // If any symbols were in the error state then bail out. - if (!SymbolsInErrorState.empty()) - return; + assert(SymI != Symbols.end() && "Symbol not found"); + assert(!SymI->second.hasMaterializerAttached() && + "Resolving symbol with materializer attached?"); + assert(SymI->second.getState() == SymbolState::Materializing && + "Symbol should be materializing"); + assert(SymI->second.getAddress() == 0 && + "Symbol has already been resolved"); + + if (SymI->second.getFlags().hasError()) + SymbolsInErrorState.insert(KV.first); + else { + auto Flags = KV.second.getFlags(); + Flags &= ~(JITSymbolFlags::Weak | JITSymbolFlags::Common); + assert(Flags == + (SymI->second.getFlags() & + ~(JITSymbolFlags::Weak | JITSymbolFlags::Common)) && + "Resolved flags should match the declared flags"); + + Worklist.push_back( + {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)}); + } + } - while (!Worklist.empty()) { - auto SymI = Worklist.back().SymI; - auto ResolvedSym = Worklist.back().ResolvedSym; - Worklist.pop_back(); + // If any symbols were in the error state then bail out. + if (!SymbolsInErrorState.empty()) { + auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); + return make_error<FailedToMaterialize>( + std::move(FailedSymbolsDepMap)); + } - auto &Name = SymI->first; + while (!Worklist.empty()) { + auto SymI = Worklist.back().SymI; + auto ResolvedSym = Worklist.back().ResolvedSym; + Worklist.pop_back(); - // Resolved symbols can not be weak: discard the weak flag. - JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags(); - SymI->second.setAddress(ResolvedSym.getAddress()); - SymI->second.setFlags(ResolvedFlags); - SymI->second.setState(SymbolState::Resolved); + auto &Name = SymI->first; - auto MII = MaterializingInfos.find(Name); - if (MII == MaterializingInfos.end()) - continue; + // Resolved symbols can not be weak: discard the weak flag. + JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags(); + SymI->second.setAddress(ResolvedSym.getAddress()); + SymI->second.setFlags(ResolvedFlags); + SymI->second.setState(SymbolState::Resolved); - auto &MI = MII->second; - for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { - Q->notifySymbolMetRequiredState(Name, ResolvedSym); - Q->removeQueryDependence(*this, Name); - if (Q->isComplete()) - CompletedQueries.insert(std::move(Q)); - } - } - }); + auto MII = MaterializingInfos.find(Name); + if (MII == MaterializingInfos.end()) + continue; - assert((SymbolsInErrorState.empty() || CompletedQueries.empty()) && - "Can't fail symbols and completed queries at the same time"); + auto &MI = MII->second; + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { + Q->notifySymbolMetRequiredState(Name, ResolvedSym); + Q->removeQueryDependence(*this, Name); + if (Q->isComplete()) + CompletedQueries.insert(std::move(Q)); + } + } - // If we failed any symbols then return an error. - if (!SymbolsInErrorState.empty()) { - auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); - (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); - return make_error<FailedToMaterialize>(std::move(FailedSymbolsDepMap)); - } + return Error::success(); + })) + return Err; // Otherwise notify all the completed queries. for (auto &Q : CompletedQueries) { @@ -901,139 +969,145 @@ Error JITDylib::resolve(const SymbolMap &Resolved) { return Error::success(); } -Error JITDylib::emit(const SymbolFlagsMap &Emitted) { +Error JITDylib::emit(MaterializationResponsibility &MR, + const SymbolFlagsMap &Emitted) { AsynchronousSymbolQuerySet CompletedQueries; - SymbolNameSet SymbolsInErrorState; DenseMap<JITDylib *, SymbolNameVector> ReadySymbols; - ES.runSessionLocked([&, this]() { - std::vector<SymbolTable::iterator> Worklist; + if (auto Err = ES.runSessionLocked([&, this]() -> Error { + auto RTI = MRTrackers.find(&MR); + assert(RTI != MRTrackers.end() && "No resource tracker for MR?"); + if (RTI->second->isDefunct()) + return make_error<ResourceTrackerDefunct>(RTI->second); - // Scan to build worklist, record any symbols in the erorr state. - for (const auto &KV : Emitted) { - auto &Name = KV.first; + SymbolNameSet SymbolsInErrorState; + std::vector<SymbolTable::iterator> Worklist; - auto SymI = Symbols.find(Name); - assert(SymI != Symbols.end() && "No symbol table entry for Name"); + // Scan to build worklist, record any symbols in the erorr state. + for (const auto &KV : Emitted) { + auto &Name = KV.first; - if (SymI->second.getFlags().hasError()) - SymbolsInErrorState.insert(Name); - else - Worklist.push_back(SymI); - } + auto SymI = Symbols.find(Name); + assert(SymI != Symbols.end() && "No symbol table entry for Name"); - // If any symbols were in the error state then bail out. - if (!SymbolsInErrorState.empty()) - return; + if (SymI->second.getFlags().hasError()) + SymbolsInErrorState.insert(Name); + else + Worklist.push_back(SymI); + } - // Otherwise update dependencies and move to the emitted state. - while (!Worklist.empty()) { - auto SymI = Worklist.back(); - Worklist.pop_back(); + // If any symbols were in the error state then bail out. + if (!SymbolsInErrorState.empty()) { + auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); + return make_error<FailedToMaterialize>( + std::move(FailedSymbolsDepMap)); + } - auto &Name = SymI->first; - auto &SymEntry = SymI->second; + // Otherwise update dependencies and move to the emitted state. + while (!Worklist.empty()) { + auto SymI = Worklist.back(); + Worklist.pop_back(); - // Move symbol to the emitted state. - assert(((SymEntry.getFlags().hasMaterializationSideEffectsOnly() && - SymEntry.getState() == SymbolState::Materializing) || - SymEntry.getState() == SymbolState::Resolved) && - "Emitting from state other than Resolved"); - SymEntry.setState(SymbolState::Emitted); + auto &Name = SymI->first; + auto &SymEntry = SymI->second; - auto MII = MaterializingInfos.find(Name); + // Move symbol to the emitted state. + assert(((SymEntry.getFlags().hasMaterializationSideEffectsOnly() && + SymEntry.getState() == SymbolState::Materializing) || + SymEntry.getState() == SymbolState::Resolved) && + "Emitting from state other than Resolved"); + SymEntry.setState(SymbolState::Emitted); - // If this symbol has no MaterializingInfo then it's trivially ready. - // Update its state and continue. - if (MII == MaterializingInfos.end()) { - SymEntry.setState(SymbolState::Ready); - continue; - } + auto MII = MaterializingInfos.find(Name); + + // If this symbol has no MaterializingInfo then it's trivially ready. + // Update its state and continue. + if (MII == MaterializingInfos.end()) { + SymEntry.setState(SymbolState::Ready); + continue; + } + + auto &MI = MII->second; + + // For each dependant, transfer this node's emitted dependencies to + // it. If the dependant node is ready (i.e. has no unemitted + // dependencies) then notify any pending queries. + for (auto &KV : MI.Dependants) { + auto &DependantJD = *KV.first; + auto &DependantJDReadySymbols = ReadySymbols[&DependantJD]; + for (auto &DependantName : KV.second) { + auto DependantMII = + DependantJD.MaterializingInfos.find(DependantName); + assert(DependantMII != DependantJD.MaterializingInfos.end() && + "Dependant should have MaterializingInfo"); + + auto &DependantMI = DependantMII->second; + + // Remove the dependant's dependency on this node. + assert(DependantMI.UnemittedDependencies.count(this) && + "Dependant does not have an unemitted dependencies record " + "for " + "this JITDylib"); + assert(DependantMI.UnemittedDependencies[this].count(Name) && + "Dependant does not count this symbol as a dependency?"); + + DependantMI.UnemittedDependencies[this].erase(Name); + if (DependantMI.UnemittedDependencies[this].empty()) + DependantMI.UnemittedDependencies.erase(this); + + // Transfer unemitted dependencies from this node to the + // dependant. + DependantJD.transferEmittedNodeDependencies(DependantMI, + DependantName, MI); + + auto DependantSymI = DependantJD.Symbols.find(DependantName); + assert(DependantSymI != DependantJD.Symbols.end() && + "Dependant has no entry in the Symbols table"); + auto &DependantSymEntry = DependantSymI->second; + + // If the dependant is emitted and this node was the last of its + // unemitted dependencies then the dependant node is now ready, so + // notify any pending queries on the dependant node. + if (DependantSymEntry.getState() == SymbolState::Emitted && + DependantMI.UnemittedDependencies.empty()) { + assert(DependantMI.Dependants.empty() && + "Dependants should be empty by now"); + + // Since this dependant is now ready, we erase its + // MaterializingInfo and update its materializing state. + DependantSymEntry.setState(SymbolState::Ready); + DependantJDReadySymbols.push_back(DependantName); + + for (auto &Q : + DependantMI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState( + DependantName, DependantSymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); + Q->removeQueryDependence(DependantJD, DependantName); + } + } + } + } - auto &MI = MII->second; - - // For each dependant, transfer this node's emitted dependencies to - // it. If the dependant node is ready (i.e. has no unemitted - // dependencies) then notify any pending queries. - for (auto &KV : MI.Dependants) { - auto &DependantJD = *KV.first; - auto &DependantJDReadySymbols = ReadySymbols[&DependantJD]; - for (auto &DependantName : KV.second) { - auto DependantMII = - DependantJD.MaterializingInfos.find(DependantName); - assert(DependantMII != DependantJD.MaterializingInfos.end() && - "Dependant should have MaterializingInfo"); - - auto &DependantMI = DependantMII->second; - - // Remove the dependant's dependency on this node. - assert(DependantMI.UnemittedDependencies.count(this) && - "Dependant does not have an unemitted dependencies record for " - "this JITDylib"); - assert(DependantMI.UnemittedDependencies[this].count(Name) && - "Dependant does not count this symbol as a dependency?"); - - DependantMI.UnemittedDependencies[this].erase(Name); - if (DependantMI.UnemittedDependencies[this].empty()) - DependantMI.UnemittedDependencies.erase(this); - - // Transfer unemitted dependencies from this node to the dependant. - DependantJD.transferEmittedNodeDependencies(DependantMI, - DependantName, MI); - - auto DependantSymI = DependantJD.Symbols.find(DependantName); - assert(DependantSymI != DependantJD.Symbols.end() && - "Dependant has no entry in the Symbols table"); - auto &DependantSymEntry = DependantSymI->second; - - // If the dependant is emitted and this node was the last of its - // unemitted dependencies then the dependant node is now ready, so - // notify any pending queries on the dependant node. - if (DependantSymEntry.getState() == SymbolState::Emitted && - DependantMI.UnemittedDependencies.empty()) { - assert(DependantMI.Dependants.empty() && - "Dependants should be empty by now"); - - // Since this dependant is now ready, we erase its MaterializingInfo - // and update its materializing state. - DependantSymEntry.setState(SymbolState::Ready); - DependantJDReadySymbols.push_back(DependantName); - - for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { - Q->notifySymbolMetRequiredState( - DependantName, DependantSymI->second.getSymbol()); + auto &ThisJDReadySymbols = ReadySymbols[this]; + MI.Dependants.clear(); + if (MI.UnemittedDependencies.empty()) { + SymI->second.setState(SymbolState::Ready); + ThisJDReadySymbols.push_back(Name); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); if (Q->isComplete()) CompletedQueries.insert(Q); - Q->removeQueryDependence(DependantJD, DependantName); + Q->removeQueryDependence(*this, Name); } } } - } - auto &ThisJDReadySymbols = ReadySymbols[this]; - MI.Dependants.clear(); - if (MI.UnemittedDependencies.empty()) { - SymI->second.setState(SymbolState::Ready); - ThisJDReadySymbols.push_back(Name); - for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { - Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); - if (Q->isComplete()) - CompletedQueries.insert(Q); - Q->removeQueryDependence(*this, Name); - } - } - } - }); - - assert((SymbolsInErrorState.empty() || CompletedQueries.empty()) && - "Can't fail symbols and completed queries at the same time"); - - // If we failed any symbols then return an error. - if (!SymbolsInErrorState.empty()) { - auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); - (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); - return make_error<FailedToMaterialize>(std::move(FailedSymbolsDepMap)); - } + return Error::success(); + })) + return Err; // Otherwise notify all the completed queries. for (auto &Q : CompletedQueries) { @@ -1044,120 +1118,122 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { return Error::success(); } -void JITDylib::notifyFailed(FailedSymbolsWorklist Worklist) { - AsynchronousSymbolQuerySet FailedQueries; - auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); - - // Failing no symbols is a no-op. - if (Worklist.empty()) - return; - - auto &ES = Worklist.front().first->getExecutionSession(); - +void JITDylib::unlinkMaterializationResponsibility( + MaterializationResponsibility &MR) { ES.runSessionLocked([&]() { - while (!Worklist.empty()) { - assert(Worklist.back().first && "Failed JITDylib can not be null"); - auto &JD = *Worklist.back().first; - auto Name = std::move(Worklist.back().second); - Worklist.pop_back(); - - (*FailedSymbolsMap)[&JD].insert(Name); - - assert(JD.Symbols.count(Name) && "No symbol table entry for Name"); - auto &Sym = JD.Symbols[Name]; - - // Move the symbol into the error state. - // Note that this may be redundant: The symbol might already have been - // moved to this state in response to the failure of a dependence. - Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError); - - // FIXME: Come up with a sane mapping of state to - // presence-of-MaterializingInfo so that we can assert presence / absence - // here, rather than testing it. - auto MII = JD.MaterializingInfos.find(Name); + auto I = MRTrackers.find(&MR); + assert(I != MRTrackers.end() && "MaterializationResponsibility not linked"); + MRTrackers.erase(I); + }); +} - if (MII == JD.MaterializingInfos.end()) - continue; +std::pair<JITDylib::AsynchronousSymbolQuerySet, + std::shared_ptr<SymbolDependenceMap>> +JITDylib::failSymbols(FailedSymbolsWorklist Worklist) { + AsynchronousSymbolQuerySet FailedQueries; + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); - auto &MI = MII->second; - - // Move all dependants to the error state and disconnect from them. - for (auto &KV : MI.Dependants) { - auto &DependantJD = *KV.first; - for (auto &DependantName : KV.second) { - assert(DependantJD.Symbols.count(DependantName) && - "No symbol table entry for DependantName"); - auto &DependantSym = DependantJD.Symbols[DependantName]; - DependantSym.setFlags(DependantSym.getFlags() | - JITSymbolFlags::HasError); - - assert(DependantJD.MaterializingInfos.count(DependantName) && - "No MaterializingInfo for dependant"); - auto &DependantMI = DependantJD.MaterializingInfos[DependantName]; - - auto UnemittedDepI = DependantMI.UnemittedDependencies.find(&JD); - assert(UnemittedDepI != DependantMI.UnemittedDependencies.end() && - "No UnemittedDependencies entry for this JITDylib"); - assert(UnemittedDepI->second.count(Name) && - "No UnemittedDependencies entry for this symbol"); - UnemittedDepI->second.erase(Name); - if (UnemittedDepI->second.empty()) - DependantMI.UnemittedDependencies.erase(UnemittedDepI); - - // If this symbol is already in the emitted state then we need to - // take responsibility for failing its queries, so add it to the - // worklist. - if (DependantSym.getState() == SymbolState::Emitted) { - assert(DependantMI.Dependants.empty() && - "Emitted symbol should not have dependants"); - Worklist.push_back(std::make_pair(&DependantJD, DependantName)); - } - } - } - MI.Dependants.clear(); - - // Disconnect from all unemitted depenencies. - for (auto &KV : MI.UnemittedDependencies) { - auto &UnemittedDepJD = *KV.first; - for (auto &UnemittedDepName : KV.second) { - auto UnemittedDepMII = - UnemittedDepJD.MaterializingInfos.find(UnemittedDepName); - assert(UnemittedDepMII != UnemittedDepJD.MaterializingInfos.end() && - "Missing MII for unemitted dependency"); - assert(UnemittedDepMII->second.Dependants.count(&JD) && - "JD not listed as a dependant of unemitted dependency"); - assert(UnemittedDepMII->second.Dependants[&JD].count(Name) && - "Name is not listed as a dependant of unemitted dependency"); - UnemittedDepMII->second.Dependants[&JD].erase(Name); - if (UnemittedDepMII->second.Dependants[&JD].empty()) - UnemittedDepMII->second.Dependants.erase(&JD); + while (!Worklist.empty()) { + assert(Worklist.back().first && "Failed JITDylib can not be null"); + auto &JD = *Worklist.back().first; + auto Name = std::move(Worklist.back().second); + Worklist.pop_back(); + + (*FailedSymbolsMap)[&JD].insert(Name); + + assert(JD.Symbols.count(Name) && "No symbol table entry for Name"); + auto &Sym = JD.Symbols[Name]; + + // Move the symbol into the error state. + // Note that this may be redundant: The symbol might already have been + // moved to this state in response to the failure of a dependence. + Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError); + + // FIXME: Come up with a sane mapping of state to + // presence-of-MaterializingInfo so that we can assert presence / absence + // here, rather than testing it. + auto MII = JD.MaterializingInfos.find(Name); + + if (MII == JD.MaterializingInfos.end()) + continue; + + auto &MI = MII->second; + + // Move all dependants to the error state and disconnect from them. + for (auto &KV : MI.Dependants) { + auto &DependantJD = *KV.first; + for (auto &DependantName : KV.second) { + assert(DependantJD.Symbols.count(DependantName) && + "No symbol table entry for DependantName"); + auto &DependantSym = DependantJD.Symbols[DependantName]; + DependantSym.setFlags(DependantSym.getFlags() | + JITSymbolFlags::HasError); + + assert(DependantJD.MaterializingInfos.count(DependantName) && + "No MaterializingInfo for dependant"); + auto &DependantMI = DependantJD.MaterializingInfos[DependantName]; + + auto UnemittedDepI = DependantMI.UnemittedDependencies.find(&JD); + assert(UnemittedDepI != DependantMI.UnemittedDependencies.end() && + "No UnemittedDependencies entry for this JITDylib"); + assert(UnemittedDepI->second.count(Name) && + "No UnemittedDependencies entry for this symbol"); + UnemittedDepI->second.erase(Name); + if (UnemittedDepI->second.empty()) + DependantMI.UnemittedDependencies.erase(UnemittedDepI); + + // If this symbol is already in the emitted state then we need to + // take responsibility for failing its queries, so add it to the + // worklist. + if (DependantSym.getState() == SymbolState::Emitted) { + assert(DependantMI.Dependants.empty() && + "Emitted symbol should not have dependants"); + Worklist.push_back(std::make_pair(&DependantJD, DependantName)); } } - MI.UnemittedDependencies.clear(); - - // Collect queries to be failed for this MII. - AsynchronousSymbolQueryList ToDetach; - for (auto &Q : MII->second.pendingQueries()) { - // Add the query to the list to be failed and detach it. - FailedQueries.insert(Q); - ToDetach.push_back(Q); + } + MI.Dependants.clear(); + + // Disconnect from all unemitted depenencies. + for (auto &KV : MI.UnemittedDependencies) { + auto &UnemittedDepJD = *KV.first; + for (auto &UnemittedDepName : KV.second) { + auto UnemittedDepMII = + UnemittedDepJD.MaterializingInfos.find(UnemittedDepName); + assert(UnemittedDepMII != UnemittedDepJD.MaterializingInfos.end() && + "Missing MII for unemitted dependency"); + assert(UnemittedDepMII->second.Dependants.count(&JD) && + "JD not listed as a dependant of unemitted dependency"); + assert(UnemittedDepMII->second.Dependants[&JD].count(Name) && + "Name is not listed as a dependant of unemitted dependency"); + UnemittedDepMII->second.Dependants[&JD].erase(Name); + if (UnemittedDepMII->second.Dependants[&JD].empty()) + UnemittedDepMII->second.Dependants.erase(&JD); } - for (auto &Q : ToDetach) - Q->detach(); - - assert(MI.Dependants.empty() && - "Can not delete MaterializingInfo with dependants still attached"); - assert(MI.UnemittedDependencies.empty() && - "Can not delete MaterializingInfo with unemitted dependencies " - "still attached"); - assert(!MI.hasQueriesPending() && - "Can not delete MaterializingInfo with queries pending"); - JD.MaterializingInfos.erase(MII); } - }); + MI.UnemittedDependencies.clear(); + + // Collect queries to be failed for this MII. + AsynchronousSymbolQueryList ToDetach; + for (auto &Q : MII->second.pendingQueries()) { + // Add the query to the list to be failed and detach it. + FailedQueries.insert(Q); + ToDetach.push_back(Q); + } + for (auto &Q : ToDetach) + Q->detach(); - for (auto &Q : FailedQueries) - Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbolsMap)); + assert(MI.Dependants.empty() && + "Can not delete MaterializingInfo with dependants still attached"); + assert(MI.UnemittedDependencies.empty() && + "Can not delete MaterializingInfo with unemitted dependencies " + "still attached"); + assert(!MI.hasQueriesPending() && + "Can not delete MaterializingInfo with queries pending"); + JD.MaterializingInfos.erase(MII); + } + + return std::make_pair(std::move(FailedQueries), std::move(FailedSymbolsMap)); } void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder, @@ -1168,8 +1244,7 @@ void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder, if (NewLinkOrder.empty() || NewLinkOrder.front().first != this) LinkOrder.push_back( std::make_pair(this, JITDylibLookupFlags::MatchAllSymbols)); - LinkOrder.insert(LinkOrder.end(), NewLinkOrder.begin(), - NewLinkOrder.end()); + llvm::append_range(LinkOrder, NewLinkOrder); } else LinkOrder = std::move(NewLinkOrder); }); @@ -1192,10 +1267,10 @@ void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD, void JITDylib::removeFromLinkOrder(JITDylib &JD) { ES.runSessionLocked([&]() { - auto I = std::find_if(LinkOrder.begin(), LinkOrder.end(), - [&](const JITDylibSearchOrder::value_type &KV) { - return KV.first == &JD; - }); + auto I = llvm::find_if(LinkOrder, + [&](const JITDylibSearchOrder::value_type &KV) { + return KV.first == &JD; + }); if (I != LinkOrder.end()) LinkOrder.erase(I); }); @@ -1257,279 +1332,6 @@ Error JITDylib::remove(const SymbolNameSet &Names) { }); } -Expected<SymbolFlagsMap> -JITDylib::lookupFlags(LookupKind K, JITDylibLookupFlags JDLookupFlags, - SymbolLookupSet LookupSet) { - return ES.runSessionLocked([&, this]() -> Expected<SymbolFlagsMap> { - SymbolFlagsMap Result; - lookupFlagsImpl(Result, K, JDLookupFlags, LookupSet); - - // Run any definition generators. - for (auto &DG : DefGenerators) { - - // Bail out early if we found everything. - if (LookupSet.empty()) - break; - - // Run this generator. - if (auto Err = DG->tryToGenerate(K, *this, JDLookupFlags, LookupSet)) - return std::move(Err); - - // Re-try the search. - lookupFlagsImpl(Result, K, JDLookupFlags, LookupSet); - } - - return Result; - }); -} - -void JITDylib::lookupFlagsImpl(SymbolFlagsMap &Result, LookupKind K, - JITDylibLookupFlags JDLookupFlags, - SymbolLookupSet &LookupSet) { - - LookupSet.forEachWithRemoval( - [&](const SymbolStringPtr &Name, SymbolLookupFlags Flags) -> bool { - auto I = Symbols.find(Name); - if (I == Symbols.end()) - return false; - assert(!Result.count(Name) && "Symbol already present in Flags map"); - Result[Name] = I->second.getFlags(); - return true; - }); -} - -Error JITDylib::lodgeQuery(MaterializationUnitList &MUs, - std::shared_ptr<AsynchronousSymbolQuery> &Q, - LookupKind K, JITDylibLookupFlags JDLookupFlags, - SymbolLookupSet &Unresolved) { - assert(Q && "Query can not be null"); - - if (auto Err = lodgeQueryImpl(MUs, Q, K, JDLookupFlags, Unresolved)) - return Err; - - // Run any definition generators. - for (auto &DG : DefGenerators) { - - // Bail out early if we have resolved everything. - if (Unresolved.empty()) - break; - - // Run the generator. - if (auto Err = DG->tryToGenerate(K, *this, JDLookupFlags, Unresolved)) - return Err; - - // Lodge query. This can not fail as any new definitions were added - // by the generator under the session locked. Since they can't have - // started materializing yet they can not have failed. - cantFail(lodgeQueryImpl(MUs, Q, K, JDLookupFlags, Unresolved)); - } - - return Error::success(); -} - -Error JITDylib::lodgeQueryImpl(MaterializationUnitList &MUs, - std::shared_ptr<AsynchronousSymbolQuery> &Q, - LookupKind K, JITDylibLookupFlags JDLookupFlags, - SymbolLookupSet &Unresolved) { - - return Unresolved.forEachWithRemoval( - [&](const SymbolStringPtr &Name, - SymbolLookupFlags SymLookupFlags) -> Expected<bool> { - // Search for name in symbols. If not found then continue without - // removal. - auto SymI = Symbols.find(Name); - if (SymI == Symbols.end()) - return false; - - // If we match against a materialization-side-effects only symbol then - // make sure it is weakly-referenced. Otherwise bail out with an error. - if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && - SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) - return make_error<SymbolsNotFound>(SymbolNameVector({Name})); - - // If this is a non exported symbol and we're matching exported symbols - // only then skip this symbol without removal. - if (!SymI->second.getFlags().isExported() && - JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) - return false; - - // If we matched against this symbol but it is in the error state then - // bail out and treat it as a failure to materialize. - if (SymI->second.getFlags().hasError()) { - auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); - (*FailedSymbolsMap)[this] = {Name}; - return make_error<FailedToMaterialize>(std::move(FailedSymbolsMap)); - } - - // If this symbol already meets the required state for then notify the - // query, then remove the symbol and continue. - if (SymI->second.getState() >= Q->getRequiredState()) { - Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); - return true; - } - - // Otherwise this symbol does not yet meet the required state. Check - // whether it has a materializer attached, and if so prepare to run it. - if (SymI->second.hasMaterializerAttached()) { - assert(SymI->second.getAddress() == 0 && - "Symbol not resolved but already has address?"); - auto UMII = UnmaterializedInfos.find(Name); - assert(UMII != UnmaterializedInfos.end() && - "Lazy symbol should have UnmaterializedInfo"); - auto MU = std::move(UMII->second->MU); - assert(MU != nullptr && "Materializer should not be null"); - - // Move all symbols associated with this MaterializationUnit into - // materializing state. - for (auto &KV : MU->getSymbols()) { - auto SymK = Symbols.find(KV.first); - SymK->second.setMaterializerAttached(false); - SymK->second.setState(SymbolState::Materializing); - UnmaterializedInfos.erase(KV.first); - } - - // Add MU to the list of MaterializationUnits to be materialized. - MUs.push_back(std::move(MU)); - } - - // Add the query to the PendingQueries list and continue, deleting the - // element. - assert(SymI->second.getState() != SymbolState::NeverSearched && - SymI->second.getState() != SymbolState::Ready && - "By this line the symbol should be materializing"); - auto &MI = MaterializingInfos[Name]; - MI.addQuery(Q); - Q->addQueryDependence(*this, Name); - return true; - }); -} - -Expected<SymbolNameSet> -JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, - SymbolNameSet Names) { - assert(Q && "Query can not be null"); - - ES.runOutstandingMUs(); - - bool QueryComplete = false; - std::vector<std::unique_ptr<MaterializationUnit>> MUs; - - SymbolLookupSet Unresolved(Names); - auto Err = ES.runSessionLocked([&, this]() -> Error { - QueryComplete = lookupImpl(Q, MUs, Unresolved); - - // Run any definition generators. - for (auto &DG : DefGenerators) { - - // Bail out early if we have resolved everything. - if (Unresolved.empty()) - break; - - assert(!QueryComplete && "query complete but unresolved symbols remain?"); - if (auto Err = DG->tryToGenerate(LookupKind::Static, *this, - JITDylibLookupFlags::MatchAllSymbols, - Unresolved)) - return Err; - - if (!Unresolved.empty()) - QueryComplete = lookupImpl(Q, MUs, Unresolved); - } - return Error::success(); - }); - - if (Err) - return std::move(Err); - - assert((MUs.empty() || !QueryComplete) && - "If action flags are set, there should be no work to do (so no MUs)"); - - if (QueryComplete) - Q->handleComplete(); - - // FIXME: Swap back to the old code below once RuntimeDyld works with - // callbacks from asynchronous queries. - // Add MUs to the OutstandingMUs list. - { - std::lock_guard<std::recursive_mutex> Lock(ES.OutstandingMUsMutex); - auto ThisJD = shared_from_this(); - for (auto &MU : MUs) { - auto MR = MU->createMaterializationResponsibility(ThisJD); - ES.OutstandingMUs.push_back(make_pair(std::move(MU), std::move(MR))); - } - } - ES.runOutstandingMUs(); - - // Dispatch any required MaterializationUnits for materialization. - // for (auto &MU : MUs) - // ES.dispatchMaterialization(*this, std::move(MU)); - - SymbolNameSet RemainingSymbols; - for (auto &KV : Unresolved) - RemainingSymbols.insert(KV.first); - - return RemainingSymbols; -} - -bool JITDylib::lookupImpl( - std::shared_ptr<AsynchronousSymbolQuery> &Q, - std::vector<std::unique_ptr<MaterializationUnit>> &MUs, - SymbolLookupSet &Unresolved) { - bool QueryComplete = false; - - std::vector<SymbolStringPtr> ToRemove; - Unresolved.forEachWithRemoval( - [&](const SymbolStringPtr &Name, SymbolLookupFlags Flags) -> bool { - // Search for the name in Symbols. Skip without removing if not found. - auto SymI = Symbols.find(Name); - if (SymI == Symbols.end()) - return false; - - // If the symbol is already in the required state then notify the query - // and remove. - if (SymI->second.getState() >= Q->getRequiredState()) { - Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); - if (Q->isComplete()) - QueryComplete = true; - return true; - } - - // If the symbol is lazy, get the MaterialiaztionUnit for it. - if (SymI->second.hasMaterializerAttached()) { - assert(SymI->second.getAddress() == 0 && - "Lazy symbol should not have a resolved address"); - auto UMII = UnmaterializedInfos.find(Name); - assert(UMII != UnmaterializedInfos.end() && - "Lazy symbol should have UnmaterializedInfo"); - auto MU = std::move(UMII->second->MU); - assert(MU != nullptr && "Materializer should not be null"); - - // Kick all symbols associated with this MaterializationUnit into - // materializing state. - for (auto &KV : MU->getSymbols()) { - auto SymK = Symbols.find(KV.first); - assert(SymK != Symbols.end() && "Missing symbol table entry"); - SymK->second.setState(SymbolState::Materializing); - SymK->second.setMaterializerAttached(false); - UnmaterializedInfos.erase(KV.first); - } - - // Add MU to the list of MaterializationUnits to be materialized. - MUs.push_back(std::move(MU)); - } - - // Add the query to the PendingQueries list. - assert(SymI->second.getState() != SymbolState::NeverSearched && - SymI->second.getState() != SymbolState::Ready && - "By this line the symbol should be materializing"); - auto &MI = MaterializingInfos[Name]; - MI.addQuery(Q); - Q->addQueryDependence(*this, Name); - return true; - }); - - return QueryComplete; -} - void JITDylib::dump(raw_ostream &OS) { ES.runSessionLocked([&, this]() { OS << "JITDylib \"" << JITDylibName << "\" (ES: " @@ -1589,11 +1391,10 @@ void JITDylib::MaterializingInfo::addQuery( void JITDylib::MaterializingInfo::removeQuery( const AsynchronousSymbolQuery &Q) { // FIXME: Implement 'find_as' for shared_ptr<T>/T*. - auto I = - std::find_if(PendingQueries.begin(), PendingQueries.end(), - [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) { - return V.get() == &Q; - }); + auto I = llvm::find_if( + PendingQueries, [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) { + return V.get() == &Q; + }); assert(I != PendingQueries.end() && "Query is not attached to this MaterializingInfo"); PendingQueries.erase(I); @@ -1618,6 +1419,137 @@ JITDylib::JITDylib(ExecutionSession &ES, std::string Name) LinkOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols}); } +ResourceTrackerSP JITDylib::getTracker(MaterializationResponsibility &MR) { + auto I = MRTrackers.find(&MR); + assert(I != MRTrackers.end() && "MR is not linked"); + assert(I->second && "Linked tracker is null"); + return I->second; +} + +std::pair<JITDylib::AsynchronousSymbolQuerySet, + std::shared_ptr<SymbolDependenceMap>> +JITDylib::removeTracker(ResourceTracker &RT) { + // Note: Should be called under the session lock. + + SymbolNameVector SymbolsToRemove; + std::vector<std::pair<JITDylib *, SymbolStringPtr>> SymbolsToFail; + + if (&RT == DefaultTracker.get()) { + SymbolNameSet TrackedSymbols; + for (auto &KV : TrackerSymbols) + for (auto &Sym : KV.second) + TrackedSymbols.insert(Sym); + + for (auto &KV : Symbols) { + auto &Sym = KV.first; + if (!TrackedSymbols.count(Sym)) + SymbolsToRemove.push_back(Sym); + } + + DefaultTracker.reset(); + } else { + /// Check for a non-default tracker. + auto I = TrackerSymbols.find(&RT); + if (I != TrackerSymbols.end()) { + SymbolsToRemove = std::move(I->second); + TrackerSymbols.erase(I); + } + // ... if not found this tracker was already defunct. Nothing to do. + } + + for (auto &Sym : SymbolsToRemove) { + assert(Symbols.count(Sym) && "Symbol not in symbol table"); + + // If there is a MaterializingInfo then collect any queries to fail. + auto MII = MaterializingInfos.find(Sym); + if (MII != MaterializingInfos.end()) + SymbolsToFail.push_back({this, Sym}); + } + + AsynchronousSymbolQuerySet QueriesToFail; + auto Result = failSymbols(std::move(SymbolsToFail)); + + // Removed symbols should be taken out of the table altogether. + for (auto &Sym : SymbolsToRemove) { + auto I = Symbols.find(Sym); + assert(I != Symbols.end() && "Symbol not present in table"); + + // Remove Materializer if present. + if (I->second.hasMaterializerAttached()) { + // FIXME: Should this discard the symbols? + UnmaterializedInfos.erase(Sym); + } else { + assert(!UnmaterializedInfos.count(Sym) && + "Symbol has materializer attached"); + } + + Symbols.erase(I); + } + + return Result; +} + +void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) { + assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker"); + assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib"); + assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib"); + + // Update trackers for any not-yet materialized units. + for (auto &KV : UnmaterializedInfos) { + if (KV.second->RT == &SrcRT) + KV.second->RT = &DstRT; + } + + // Update trackers for any active materialization responsibilities. + for (auto &KV : MRTrackers) { + if (KV.second == &SrcRT) + KV.second = &DstRT; + } + + // If we're transfering to the default tracker we just need to delete the + // tracked symbols for the source tracker. + if (&DstRT == DefaultTracker.get()) { + TrackerSymbols.erase(&SrcRT); + return; + } + + // If we're transferring from the default tracker we need to find all + // currently untracked symbols. + if (&SrcRT == DefaultTracker.get()) { + assert(!TrackerSymbols.count(&SrcRT) && + "Default tracker should not appear in TrackerSymbols"); + + SymbolNameVector SymbolsToTrack; + + SymbolNameSet CurrentlyTrackedSymbols; + for (auto &KV : TrackerSymbols) + for (auto &Sym : KV.second) + CurrentlyTrackedSymbols.insert(Sym); + + for (auto &KV : Symbols) { + auto &Sym = KV.first; + if (!CurrentlyTrackedSymbols.count(Sym)) + SymbolsToTrack.push_back(Sym); + } + + TrackerSymbols[&DstRT] = std::move(SymbolsToTrack); + return; + } + + auto &DstTrackedSymbols = TrackerSymbols[&DstRT]; + + // Finally if neither SrtRT or DstRT are the default tracker then + // just append DstRT's tracked symbols to SrtRT's. + auto SI = TrackerSymbols.find(&SrcRT); + if (SI == TrackerSymbols.end()) + return; + + DstTrackedSymbols.reserve(DstTrackedSymbols.size() + SI->second.size()); + for (auto &Sym : SI->second) + DstTrackedSymbols.push_back(std::move(Sym)); + TrackerSymbols.erase(SI); +} + Error JITDylib::defineImpl(MaterializationUnit &MU) { LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; }); @@ -1685,6 +1617,22 @@ Error JITDylib::defineImpl(MaterializationUnit &MU) { return Error::success(); } +void JITDylib::installMaterializationUnit( + std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) { + + /// defineImpl succeeded. + if (&RT != DefaultTracker.get()) { + auto &TS = TrackerSymbols[&RT]; + TS.reserve(TS.size() + MU->getSymbols().size()); + for (auto &KV : MU->getSymbols()) + TS.push_back(KV.first); + } + + auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU), &RT); + for (auto &KV : UMI->MU->getSymbols()) + UnmaterializedInfos[KV.first] = UMI; +} + void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q, const SymbolNameSet &QuerySymbols) { for (auto &QuerySymbol : QuerySymbols) { @@ -1773,7 +1721,39 @@ Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols( } ExecutionSession::ExecutionSession(std::shared_ptr<SymbolStringPool> SSP) - : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) { + : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) {} + +Error ExecutionSession::endSession() { + LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n"); + + std::vector<JITDylibSP> JITDylibsToClose = runSessionLocked([&] { + SessionOpen = false; + return std::move(JDs); + }); + + // TODO: notifiy platform? run static deinits? + + Error Err = Error::success(); + for (auto &JD : JITDylibsToClose) + Err = joinErrors(std::move(Err), JD->clear()); + return Err; +} + +void ExecutionSession::registerResourceManager(ResourceManager &RM) { + runSessionLocked([&] { ResourceManagers.push_back(&RM); }); +} + +void ExecutionSession::deregisterResourceManager(ResourceManager &RM) { + runSessionLocked([&] { + assert(!ResourceManagers.empty() && "No managers registered"); + if (ResourceManagers.back() == &RM) + ResourceManagers.pop_back(); + else { + auto I = llvm::find(ResourceManagers, &RM); + assert(I != ResourceManagers.end() && "RM not registered"); + ResourceManagers.erase(I); + } + }); } JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { @@ -1788,8 +1768,7 @@ JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { JITDylib &ExecutionSession::createBareJITDylib(std::string Name) { assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { - JDs.push_back( - std::shared_ptr<JITDylib>(new JITDylib(*this, std::move(Name)))); + JDs.push_back(new JITDylib(*this, std::move(Name))); return *JDs.back(); }); } @@ -1802,86 +1781,80 @@ Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) { return JD; } -void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { - assert(!!Err && "Error should be in failure state"); +std::vector<JITDylibSP> JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) { + if (JDs.empty()) + return {}; - bool SendErrorToQuery; - runSessionLocked([&]() { - Q.detach(); - SendErrorToQuery = Q.canStillFail(); + auto &ES = JDs.front()->getExecutionSession(); + return ES.runSessionLocked([&]() { + DenseSet<JITDylib *> Visited; + std::vector<JITDylibSP> Result; + + for (auto &JD : JDs) { + + if (Visited.count(JD.get())) + continue; + + SmallVector<JITDylibSP, 64> WorkStack; + WorkStack.push_back(JD); + Visited.insert(JD.get()); + + while (!WorkStack.empty()) { + Result.push_back(std::move(WorkStack.back())); + WorkStack.pop_back(); + + for (auto &KV : llvm::reverse(Result.back()->LinkOrder)) { + auto &JD = *KV.first; + if (Visited.count(&JD)) + continue; + Visited.insert(&JD); + WorkStack.push_back(&JD); + } + } + } + return Result; }); +} - if (SendErrorToQuery) - Q.handleFailed(std::move(Err)); - else - reportError(std::move(Err)); +std::vector<JITDylibSP> +JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) { + auto Tmp = getDFSLinkOrder(JDs); + std::reverse(Tmp.begin(), Tmp.end()); + return Tmp; } -Expected<SymbolMap> ExecutionSession::legacyLookup( - LegacyAsyncLookupFunction AsyncLookup, SymbolNameSet Names, - SymbolState RequiredState, - RegisterDependenciesFunction RegisterDependencies) { -#if LLVM_ENABLE_THREADS - // In the threaded case we use promises to return the results. - std::promise<SymbolMap> PromisedResult; - Error ResolutionError = Error::success(); - auto NotifyComplete = [&](Expected<SymbolMap> R) { - if (R) - PromisedResult.set_value(std::move(*R)); - else { - ErrorAsOutParameter _(&ResolutionError); - ResolutionError = R.takeError(); - PromisedResult.set_value(SymbolMap()); - } - }; -#else - SymbolMap Result; - Error ResolutionError = Error::success(); +std::vector<JITDylibSP> JITDylib::getDFSLinkOrder() { + return getDFSLinkOrder({this}); +} - auto NotifyComplete = [&](Expected<SymbolMap> R) { - ErrorAsOutParameter _(&ResolutionError); - if (R) - Result = std::move(*R); - else - ResolutionError = R.takeError(); - }; -#endif +std::vector<JITDylibSP> JITDylib::getReverseDFSLinkOrder() { + return getReverseDFSLinkOrder({this}); +} - auto Query = std::make_shared<AsynchronousSymbolQuery>( - SymbolLookupSet(Names), RequiredState, std::move(NotifyComplete)); - // FIXME: This should be run session locked along with the registration code - // and error reporting below. - SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); - - // If the query was lodged successfully then register the dependencies, - // otherwise fail it with an error. - if (UnresolvedSymbols.empty()) - RegisterDependencies(Query->QueryRegistrations); - else { - bool DeliverError = runSessionLocked([&]() { - Query->detach(); - return Query->canStillFail(); - }); - auto Err = make_error<SymbolsNotFound>(std::move(UnresolvedSymbols)); - if (DeliverError) - Query->handleFailed(std::move(Err)); - else - reportError(std::move(Err)); - } +void ExecutionSession::lookupFlags( + LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) { -#if LLVM_ENABLE_THREADS - auto ResultFuture = PromisedResult.get_future(); - auto Result = ResultFuture.get(); - if (ResolutionError) - return std::move(ResolutionError); - return std::move(Result); + OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>( + K, std::move(SearchOrder), std::move(LookupSet), + std::move(OnComplete)), + Error::success()); +} -#else - if (ResolutionError) - return std::move(ResolutionError); +Expected<SymbolFlagsMap> +ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet LookupSet) { - return Result; -#endif + std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP; + OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>( + K, std::move(SearchOrder), std::move(LookupSet), + [&ResultP](Expected<SymbolFlagsMap> Result) { + ResultP.set_value(std::move(Result)); + }), + Error::success()); + + auto ResultF = ResultP.get_future(); + return ResultF.get(); } void ExecutionSession::lookup( @@ -1900,94 +1873,17 @@ void ExecutionSession::lookup( // lookup can be re-entered recursively if running on a single thread. Run any // outstanding MUs in case this query depends on them, otherwise this lookup // will starve waiting for a result from an MU that is stuck in the queue. - runOutstandingMUs(); + dispatchOutstandingMUs(); auto Unresolved = std::move(Symbols); - std::map<JITDylib *, MaterializationUnitList> CollectedMUsMap; auto Q = std::make_shared<AsynchronousSymbolQuery>(Unresolved, RequiredState, std::move(NotifyComplete)); - bool QueryComplete = false; - - auto LodgingErr = runSessionLocked([&]() -> Error { - auto LodgeQuery = [&]() -> Error { - for (auto &KV : SearchOrder) { - assert(KV.first && "JITDylibList entries must not be null"); - assert(!CollectedMUsMap.count(KV.first) && - "JITDylibList should not contain duplicate entries"); - - auto &JD = *KV.first; - auto JDLookupFlags = KV.second; - if (auto Err = JD.lodgeQuery(CollectedMUsMap[&JD], Q, K, JDLookupFlags, - Unresolved)) - return Err; - } - - // Strip any weakly referenced symbols that were not found. - Unresolved.forEachWithRemoval( - [&](const SymbolStringPtr &Name, SymbolLookupFlags Flags) { - if (Flags == SymbolLookupFlags::WeaklyReferencedSymbol) { - Q->dropSymbol(Name); - return true; - } - return false; - }); - - if (!Unresolved.empty()) - return make_error<SymbolsNotFound>(Unresolved.getSymbolNames()); - - return Error::success(); - }; - if (auto Err = LodgeQuery()) { - // Query failed. + auto IPLS = std::make_unique<InProgressFullLookupState>( + K, SearchOrder, std::move(Unresolved), RequiredState, std::move(Q), + std::move(RegisterDependencies)); - // Disconnect the query from its dependencies. - Q->detach(); - - // Replace the MUs. - for (auto &KV : CollectedMUsMap) - for (auto &MU : KV.second) - KV.first->replace(std::move(MU)); - - return Err; - } - - // Query lodged successfully. - - // Record whether this query is fully ready / resolved. We will use - // this to call handleFullyResolved/handleFullyReady outside the session - // lock. - QueryComplete = Q->isComplete(); - - // Call the register dependencies function. - if (RegisterDependencies && !Q->QueryRegistrations.empty()) - RegisterDependencies(Q->QueryRegistrations); - - return Error::success(); - }); - - if (LodgingErr) { - Q->handleFailed(std::move(LodgingErr)); - return; - } - - if (QueryComplete) - Q->handleComplete(); - - // Move the MUs to the OutstandingMUs list, then materialize. - { - std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); - - for (auto &KV : CollectedMUsMap) { - auto JD = KV.first->shared_from_this(); - for (auto &MU : KV.second) { - auto MR = MU->createMaterializationResponsibility(JD); - OutstandingMUs.push_back(std::make_pair(std::move(MU), std::move(MR))); - } - } - } - - runOutstandingMUs(); + OL_applyQueryPhase1(std::move(IPLS), Error::success()); } Expected<SymbolMap> @@ -2077,10 +1973,11 @@ void ExecutionSession::dump(raw_ostream &OS) { }); } -void ExecutionSession::runOutstandingMUs() { +void ExecutionSession::dispatchOutstandingMUs() { + LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n"); while (1) { Optional<std::pair<std::unique_ptr<MaterializationUnit>, - MaterializationResponsibility>> + std::unique_ptr<MaterializationResponsibility>>> JMU; { @@ -2095,8 +1992,777 @@ void ExecutionSession::runOutstandingMUs() { break; assert(JMU->first && "No MU?"); + LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n"); dispatchMaterialization(std::move(JMU->first), std::move(JMU->second)); } + LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n"); +} + +Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) { + LLVM_DEBUG({ + dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker " + << formatv("{0:x}", RT.getKeyUnsafe()) << "\n"; + }); + std::vector<ResourceManager *> CurrentResourceManagers; + + JITDylib::AsynchronousSymbolQuerySet QueriesToFail; + std::shared_ptr<SymbolDependenceMap> FailedSymbols; + + runSessionLocked([&] { + CurrentResourceManagers = ResourceManagers; + RT.makeDefunct(); + std::tie(QueriesToFail, FailedSymbols) = RT.getJITDylib().removeTracker(RT); + }); + + Error Err = Error::success(); + + for (auto *L : reverse(CurrentResourceManagers)) + Err = + joinErrors(std::move(Err), L->handleRemoveResources(RT.getKeyUnsafe())); + + for (auto &Q : QueriesToFail) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); + + return Err; +} + +void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT, + ResourceTracker &SrcRT) { + LLVM_DEBUG({ + dbgs() << "In " << SrcRT.getJITDylib().getName() + << " transfering resources from tracker " + << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker " + << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n"; + }); + + // No-op transfers are allowed and do not invalidate the source. + if (&DstRT == &SrcRT) + return; + + assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() && + "Can't transfer resources between JITDylibs"); + runSessionLocked([&]() { + SrcRT.makeDefunct(); + auto &JD = DstRT.getJITDylib(); + JD.transferTracker(DstRT, SrcRT); + for (auto *L : reverse(ResourceManagers)) + L->handleTransferResources(DstRT.getKeyUnsafe(), SrcRT.getKeyUnsafe()); + }); +} + +void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) { + runSessionLocked([&]() { + LLVM_DEBUG({ + dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker " + << formatv("{0:x}", RT.getKeyUnsafe()) << "\n"; + }); + if (!RT.isDefunct()) + transferResourceTracker(*RT.getJITDylib().getDefaultResourceTracker(), + RT); + }); +} + +Error ExecutionSession::IL_updateCandidatesFor( + JITDylib &JD, JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) { + return Candidates.forEachWithRemoval( + [&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) -> Expected<bool> { + /// Search for the symbol. If not found then continue without + /// removal. + auto SymI = JD.Symbols.find(Name); + if (SymI == JD.Symbols.end()) + return false; + + // If this is a non-exported symbol and we're matching exported + // symbols only then remove this symbol from the candidates list. + // + // If we're tracking non-candidates then add this to the non-candidate + // list. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) { + if (NonCandidates) + NonCandidates->add(Name, SymLookupFlags); + return true; + } + + // If we match against a materialization-side-effects only symbol + // then make sure it is weakly-referenced. Otherwise bail out with + // an error. + // FIXME: Use a "materialization-side-effects-only symbols must be + // weakly referenced" specific error here to reduce confusion. + if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && + SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) + return make_error<SymbolsNotFound>(SymbolNameVector({Name})); + + // If we matched against this symbol but it is in the error state + // then bail out and treat it as a failure to materialize. + if (SymI->second.getFlags().hasError()) { + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsMap)[&JD] = {Name}; + return make_error<FailedToMaterialize>(std::move(FailedSymbolsMap)); + } + + // Otherwise this is a match. Remove it from the candidate set. + return true; + }); +} + +void ExecutionSession::OL_applyQueryPhase1( + std::unique_ptr<InProgressLookupState> IPLS, Error Err) { + + LLVM_DEBUG({ + dbgs() << "Entering OL_applyQueryPhase1:\n" + << " Lookup kind: " << IPLS->K << "\n" + << " Search order: " << IPLS->SearchOrder + << ", Current index = " << IPLS->CurSearchOrderIndex + << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n" + << " Lookup set: " << IPLS->LookupSet << "\n" + << " Definition generator candidates: " + << IPLS->DefGeneratorCandidates << "\n" + << " Definition generator non-candidates: " + << IPLS->DefGeneratorNonCandidates << "\n"; + }); + + // FIXME: We should attach the query as we go: This provides a result in a + // single pass in the common case where all symbols have already reached the + // required state. The query could be detached again in the 'fail' method on + // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs. + + while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) { + + // If we've been handed an error or received one back from a generator then + // fail the query. We don't need to unlink: At this stage the query hasn't + // actually been lodged. + if (Err) + return IPLS->fail(std::move(Err)); + + // Get the next JITDylib and lookup flags. + auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex]; + auto &JD = *KV.first; + auto JDLookupFlags = KV.second; + + LLVM_DEBUG({ + dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags + << ") with lookup set " << IPLS->LookupSet << ":\n"; + }); + + // If we've just reached a new JITDylib then perform some setup. + if (IPLS->NewJITDylib) { + + // Acquire the generator lock for this JITDylib. + IPLS->GeneratorLock = std::unique_lock<std::mutex>(JD.GeneratorsMutex); + + // Add any non-candidates from the last JITDylib (if any) back on to the + // list of definition candidates for this JITDylib, reset definition + // non-candiates to the empty set. + SymbolLookupSet Tmp; + std::swap(IPLS->DefGeneratorNonCandidates, Tmp); + IPLS->DefGeneratorCandidates.append(std::move(Tmp)); + + LLVM_DEBUG({ + dbgs() << " First time visiting " << JD.getName() + << ", resetting candidate sets and building generator stack\n"; + }); + + // Build the definition generator stack for this JITDylib. + for (auto &DG : reverse(JD.DefGenerators)) + IPLS->CurDefGeneratorStack.push_back(DG); + + // Flag that we've done our initialization. + IPLS->NewJITDylib = false; + } + + // Remove any generation candidates that are already defined (and match) in + // this JITDylib. + runSessionLocked([&] { + // Update the list of candidates (and non-candidates) for definition + // generation. + LLVM_DEBUG(dbgs() << " Updating candidate set...\n"); + Err = IL_updateCandidatesFor( + JD, JDLookupFlags, IPLS->DefGeneratorCandidates, + JD.DefGenerators.empty() ? nullptr + : &IPLS->DefGeneratorNonCandidates); + LLVM_DEBUG({ + dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates + << "\n"; + }); + }); + + // If we encountered an error while filtering generation candidates then + // bail out. + if (Err) + return IPLS->fail(std::move(Err)); + + /// Apply any definition generators on the stack. + LLVM_DEBUG({ + if (IPLS->CurDefGeneratorStack.empty()) + LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n"); + else if (IPLS->DefGeneratorCandidates.empty()) + LLVM_DEBUG(dbgs() << " No candidates to generate.\n"); + else + dbgs() << " Running " << IPLS->CurDefGeneratorStack.size() + << " remaining generators for " + << IPLS->DefGeneratorCandidates.size() << " candidates\n"; + }); + while (!IPLS->CurDefGeneratorStack.empty() && + !IPLS->DefGeneratorCandidates.empty()) { + auto DG = IPLS->CurDefGeneratorStack.back().lock(); + IPLS->CurDefGeneratorStack.pop_back(); + + if (!DG) + return IPLS->fail(make_error<StringError>( + "DefinitionGenerator removed while lookup in progress", + inconvertibleErrorCode())); + + auto K = IPLS->K; + auto &LookupSet = IPLS->DefGeneratorCandidates; + + // Run the generator. If the generator takes ownership of QA then this + // will break the loop. + { + LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n"); + LookupState LS(std::move(IPLS)); + Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet); + IPLS = std::move(LS.IPLS); + } + + // If there was an error then fail the query. + if (Err) { + LLVM_DEBUG({ + dbgs() << " Error attempting to generate " << LookupSet << "\n"; + }); + assert(IPLS && "LS cannot be retained if error is returned"); + return IPLS->fail(std::move(Err)); + } + + // Otherwise if QA was captured then break the loop. + if (!IPLS) { + LLVM_DEBUG( + { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; }); + return; + } + + // Otherwise if we're continuing around the loop then update candidates + // for the next round. + runSessionLocked([&] { + LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n"); + Err = IL_updateCandidatesFor( + JD, JDLookupFlags, IPLS->DefGeneratorCandidates, + JD.DefGenerators.empty() ? nullptr + : &IPLS->DefGeneratorNonCandidates); + }); + + // If updating candidates failed then fail the query. + if (Err) { + LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n"); + return IPLS->fail(std::move(Err)); + } + } + + // If we get here then we've moved on to the next JITDylib. + LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n"); + ++IPLS->CurSearchOrderIndex; + IPLS->NewJITDylib = true; + } + + // Remove any weakly referenced candidates that could not be found/generated. + IPLS->DefGeneratorCandidates.remove_if( + [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { + return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol; + }); + + // If we get here then we've finished searching all JITDylibs. + // If we matched all symbols then move to phase 2, otherwise fail the query + // with a SymbolsNotFound error. + if (IPLS->DefGeneratorCandidates.empty()) { + LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n"); + IPLS->complete(std::move(IPLS)); + } else { + LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n"); + IPLS->fail(make_error<SymbolsNotFound>( + IPLS->DefGeneratorCandidates.getSymbolNames())); + } +} + +void ExecutionSession::OL_completeLookup( + std::unique_ptr<InProgressLookupState> IPLS, + std::shared_ptr<AsynchronousSymbolQuery> Q, + RegisterDependenciesFunction RegisterDependencies) { + + LLVM_DEBUG({ + dbgs() << "Entering OL_completeLookup:\n" + << " Lookup kind: " << IPLS->K << "\n" + << " Search order: " << IPLS->SearchOrder + << ", Current index = " << IPLS->CurSearchOrderIndex + << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n" + << " Lookup set: " << IPLS->LookupSet << "\n" + << " Definition generator candidates: " + << IPLS->DefGeneratorCandidates << "\n" + << " Definition generator non-candidates: " + << IPLS->DefGeneratorNonCandidates << "\n"; + }); + + bool QueryComplete = false; + DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs; + + auto LodgingErr = runSessionLocked([&]() -> Error { + for (auto &KV : IPLS->SearchOrder) { + auto &JD = *KV.first; + auto JDLookupFlags = KV.second; + LLVM_DEBUG({ + dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags + << ") with lookup set " << IPLS->LookupSet << ":\n"; + }); + + auto Err = IPLS->LookupSet.forEachWithRemoval( + [&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) -> Expected<bool> { + LLVM_DEBUG({ + dbgs() << " Attempting to match \"" << Name << "\" (" + << SymLookupFlags << ")... "; + }); + + /// Search for the symbol. If not found then continue without + /// removal. + auto SymI = JD.Symbols.find(Name); + if (SymI == JD.Symbols.end()) { + LLVM_DEBUG(dbgs() << "skipping: not present\n"); + return false; + } + + // If this is a non-exported symbol and we're matching exported + // symbols only then skip this symbol without removal. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == + JITDylibLookupFlags::MatchExportedSymbolsOnly) { + LLVM_DEBUG(dbgs() << "skipping: not exported\n"); + return false; + } + + // If we match against a materialization-side-effects only symbol + // then make sure it is weakly-referenced. Otherwise bail out with + // an error. + // FIXME: Use a "materialization-side-effects-only symbols must be + // weakly referenced" specific error here to reduce confusion. + if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && + SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) { + LLVM_DEBUG({ + dbgs() << "error: " + "required, but symbol is has-side-effects-only\n"; + }); + return make_error<SymbolsNotFound>(SymbolNameVector({Name})); + } + + // If we matched against this symbol but it is in the error state + // then bail out and treat it as a failure to materialize. + if (SymI->second.getFlags().hasError()) { + LLVM_DEBUG(dbgs() << "error: symbol is in error state\n"); + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsMap)[&JD] = {Name}; + return make_error<FailedToMaterialize>( + std::move(FailedSymbolsMap)); + } + + // Otherwise this is a match. + + // If this symbol is already in the requried state then notify the + // query, remove the symbol and continue. + if (SymI->second.getState() >= Q->getRequiredState()) { + LLVM_DEBUG(dbgs() + << "matched, symbol already in required state\n"); + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + return true; + } + + // Otherwise this symbol does not yet meet the required state. Check + // whether it has a materializer attached, and if so prepare to run + // it. + if (SymI->second.hasMaterializerAttached()) { + assert(SymI->second.getAddress() == 0 && + "Symbol not resolved but already has address?"); + auto UMII = JD.UnmaterializedInfos.find(Name); + assert(UMII != JD.UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + + auto UMI = UMII->second; + assert(UMI->MU && "Materializer should not be null"); + assert(UMI->RT && "Tracker should not be null"); + LLVM_DEBUG({ + dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get() + << " (" << UMI->MU->getName() << ")\n"; + }); + + // Move all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : UMI->MU->getSymbols()) { + auto SymK = JD.Symbols.find(KV.first); + assert(SymK != JD.Symbols.end() && + "No entry for symbol covered by MaterializationUnit"); + SymK->second.setMaterializerAttached(false); + SymK->second.setState(SymbolState::Materializing); + JD.UnmaterializedInfos.erase(KV.first); + } + + // Add MU to the list of MaterializationUnits to be materialized. + CollectedUMIs[&JD].push_back(std::move(UMI)); + } else + LLVM_DEBUG(dbgs() << "matched, registering query"); + + // Add the query to the PendingQueries list and continue, deleting + // the element from the lookup set. + assert(SymI->second.getState() != SymbolState::NeverSearched && + SymI->second.getState() != SymbolState::Ready && + "By this line the symbol should be materializing"); + auto &MI = JD.MaterializingInfos[Name]; + MI.addQuery(Q); + Q->addQueryDependence(JD, Name); + + return true; + }); + + // Handle failure. + if (Err) { + + LLVM_DEBUG({ + dbgs() << "Lookup failed. Detaching query and replacing MUs.\n"; + }); + + // Detach the query. + Q->detach(); + + // Replace the MUs. + for (auto &KV : CollectedUMIs) { + auto &JD = *KV.first; + for (auto &UMI : KV.second) + for (auto &KV2 : UMI->MU->getSymbols()) { + assert(!JD.UnmaterializedInfos.count(KV2.first) && + "Unexpected materializer in map"); + auto SymI = JD.Symbols.find(KV2.first); + assert(SymI != JD.Symbols.end() && "Missing symbol entry"); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "MaterializerAttached flag should not be set"); + SymI->second.setMaterializerAttached(true); + JD.UnmaterializedInfos[KV2.first] = UMI; + } + } + + return Err; + } + } + + LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-refererced symbols\n"); + IPLS->LookupSet.forEachWithRemoval( + [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { + if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) { + Q->dropSymbol(Name); + return true; + } else + return false; + }); + + if (!IPLS->LookupSet.empty()) { + LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n"); + return make_error<SymbolsNotFound>(IPLS->LookupSet.getSymbolNames()); + } + + // Record whether the query completed. + QueryComplete = Q->isComplete(); + + LLVM_DEBUG({ + dbgs() << "Query successfully " + << (QueryComplete ? "completed" : "lodged") << "\n"; + }); + + // Move the collected MUs to the OutstandingMUs list. + if (!CollectedUMIs.empty()) { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + + LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n"); + for (auto &KV : CollectedUMIs) { + auto &JD = *KV.first; + LLVM_DEBUG({ + dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size() + << " MUs.\n"; + }); + for (auto &UMI : KV.second) { + std::unique_ptr<MaterializationResponsibility> MR( + new MaterializationResponsibility( + &JD, std::move(UMI->MU->SymbolFlags), + std::move(UMI->MU->InitSymbol))); + JD.MRTrackers[MR.get()] = UMI->RT; + OutstandingMUs.push_back( + std::make_pair(std::move(UMI->MU), std::move(MR))); + } + } + } else + LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n"); + + if (RegisterDependencies && !Q->QueryRegistrations.empty()) { + LLVM_DEBUG(dbgs() << "Registering dependencies\n"); + RegisterDependencies(Q->QueryRegistrations); + } else + LLVM_DEBUG(dbgs() << "No dependencies to register\n"); + + return Error::success(); + }); + + if (LodgingErr) { + LLVM_DEBUG(dbgs() << "Failing query\n"); + Q->detach(); + Q->handleFailed(std::move(LodgingErr)); + return; + } + + if (QueryComplete) { + LLVM_DEBUG(dbgs() << "Completing query\n"); + Q->handleComplete(); + } + + dispatchOutstandingMUs(); +} + +void ExecutionSession::OL_completeLookupFlags( + std::unique_ptr<InProgressLookupState> IPLS, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) { + + auto Result = runSessionLocked([&]() -> Expected<SymbolFlagsMap> { + LLVM_DEBUG({ + dbgs() << "Entering OL_completeLookupFlags:\n" + << " Lookup kind: " << IPLS->K << "\n" + << " Search order: " << IPLS->SearchOrder + << ", Current index = " << IPLS->CurSearchOrderIndex + << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n" + << " Lookup set: " << IPLS->LookupSet << "\n" + << " Definition generator candidates: " + << IPLS->DefGeneratorCandidates << "\n" + << " Definition generator non-candidates: " + << IPLS->DefGeneratorNonCandidates << "\n"; + }); + + SymbolFlagsMap Result; + + // Attempt to find flags for each symbol. + for (auto &KV : IPLS->SearchOrder) { + auto &JD = *KV.first; + auto JDLookupFlags = KV.second; + LLVM_DEBUG({ + dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags + << ") with lookup set " << IPLS->LookupSet << ":\n"; + }); + + IPLS->LookupSet.forEachWithRemoval([&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) { + LLVM_DEBUG({ + dbgs() << " Attempting to match \"" << Name << "\" (" + << SymLookupFlags << ")... "; + }); + + // Search for the symbol. If not found then continue without removing + // from the lookup set. + auto SymI = JD.Symbols.find(Name); + if (SymI == JD.Symbols.end()) { + LLVM_DEBUG(dbgs() << "skipping: not present\n"); + return false; + } + + // If this is a non-exported symbol then it doesn't match. Skip it. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) { + LLVM_DEBUG(dbgs() << "skipping: not exported\n"); + return false; + } + + LLVM_DEBUG({ + dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags() + << "\n"; + }); + Result[Name] = SymI->second.getFlags(); + return true; + }); + } + + // Remove any weakly referenced symbols that haven't been resolved. + IPLS->LookupSet.remove_if( + [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { + return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol; + }); + + if (!IPLS->LookupSet.empty()) { + LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n"); + return make_error<SymbolsNotFound>(IPLS->LookupSet.getSymbolNames()); + } + + LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n"); + return Result; + }); + + // Run the callback on the result. + LLVM_DEBUG(dbgs() << "Sending result to handler.\n"); + OnComplete(std::move(Result)); +} + +void ExecutionSession::OL_destroyMaterializationResponsibility( + MaterializationResponsibility &MR) { + + assert(MR.SymbolFlags.empty() && + "All symbols should have been explicitly materialized or failed"); + MR.JD->unlinkMaterializationResponsibility(MR); +} + +SymbolNameSet ExecutionSession::OL_getRequestedSymbols( + const MaterializationResponsibility &MR) { + return MR.JD->getRequestedSymbols(MR.SymbolFlags); +} + +Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR, + const SymbolMap &Symbols) { + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " resolving " << Symbols << "\n"; + }); +#ifndef NDEBUG + for (auto &KV : Symbols) { + auto WeakFlags = JITSymbolFlags::Weak | JITSymbolFlags::Common; + auto I = MR.SymbolFlags.find(KV.first); + assert(I != MR.SymbolFlags.end() && + "Resolving symbol outside this responsibility set"); + assert(!I->second.hasMaterializationSideEffectsOnly() && + "Can't resolve materialization-side-effects-only symbol"); + assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) && + "Resolving symbol with incorrect flags"); + } +#endif + + return MR.JD->resolve(MR, Symbols); +} + +Error ExecutionSession::OL_notifyEmitted(MaterializationResponsibility &MR) { + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " emitting " << MR.SymbolFlags << "\n"; + }); + + if (auto Err = MR.JD->emit(MR, MR.SymbolFlags)) + return Err; + + MR.SymbolFlags.clear(); + return Error::success(); +} + +Error ExecutionSession::OL_defineMaterializing( + MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) { + + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " defining materializing symbols " + << NewSymbolFlags << "\n"; + }); + if (auto AcceptedDefs = MR.JD->defineMaterializing(std::move(NewSymbolFlags))) { + // Add all newly accepted symbols to this responsibility object. + for (auto &KV : *AcceptedDefs) + MR.SymbolFlags.insert(KV); + return Error::success(); + } else + return AcceptedDefs.takeError(); +} + +void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) { + + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " failing materialization for " + << MR.SymbolFlags << "\n"; + }); + + JITDylib::FailedSymbolsWorklist Worklist; + + for (auto &KV : MR.SymbolFlags) + Worklist.push_back(std::make_pair(MR.JD.get(), KV.first)); + MR.SymbolFlags.clear(); + + if (Worklist.empty()) + return; + + JITDylib::AsynchronousSymbolQuerySet FailedQueries; + std::shared_ptr<SymbolDependenceMap> FailedSymbols; + + runSessionLocked([&]() { + auto RTI = MR.JD->MRTrackers.find(&MR); + assert(RTI != MR.JD->MRTrackers.end() && "No tracker for this"); + if (RTI->second->isDefunct()) + return; + + std::tie(FailedQueries, FailedSymbols) = + JITDylib::failSymbols(std::move(Worklist)); + }); + + for (auto &Q : FailedQueries) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); +} + +Error ExecutionSession::OL_replace(MaterializationResponsibility &MR, + std::unique_ptr<MaterializationUnit> MU) { + for (auto &KV : MU->getSymbols()) { + assert(MR.SymbolFlags.count(KV.first) && + "Replacing definition outside this responsibility set"); + MR.SymbolFlags.erase(KV.first); + } + + if (MU->getInitializerSymbol() == MR.InitSymbol) + MR.InitSymbol = nullptr; + + LLVM_DEBUG(MR.JD->getExecutionSession().runSessionLocked([&]() { + dbgs() << "In " << MR.JD->getName() << " replacing symbols with " << *MU + << "\n"; + });); + + return MR.JD->replace(MR, std::move(MU)); +} + +Expected<std::unique_ptr<MaterializationResponsibility>> +ExecutionSession::OL_delegate(MaterializationResponsibility &MR, + const SymbolNameSet &Symbols) { + + SymbolStringPtr DelegatedInitSymbol; + SymbolFlagsMap DelegatedFlags; + + for (auto &Name : Symbols) { + auto I = MR.SymbolFlags.find(Name); + assert(I != MR.SymbolFlags.end() && + "Symbol is not tracked by this MaterializationResponsibility " + "instance"); + + DelegatedFlags[Name] = std::move(I->second); + if (Name == MR.InitSymbol) + std::swap(MR.InitSymbol, DelegatedInitSymbol); + + MR.SymbolFlags.erase(I); + } + + return MR.JD->delegate(MR, std::move(DelegatedFlags), + std::move(DelegatedInitSymbol)); +} + +void ExecutionSession::OL_addDependencies( + MaterializationResponsibility &MR, const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies) { + LLVM_DEBUG({ + dbgs() << "Adding dependencies for " << Name << ": " << Dependencies + << "\n"; + }); + assert(MR.SymbolFlags.count(Name) && + "Symbol not covered by this MaterializationResponsibility instance"); + MR.JD->addDependencies(Name, Dependencies); +} + +void ExecutionSession::OL_addDependenciesForAll( + MaterializationResponsibility &MR, + const SymbolDependenceMap &Dependencies) { + LLVM_DEBUG({ + dbgs() << "Adding dependencies for all symbols in " << MR.SymbolFlags << ": " + << Dependencies << "\n"; + }); + for (auto &KV : MR.SymbolFlags) + MR.JD->addDependencies(KV.first, Dependencies); } #ifndef NDEBUG diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 4d255cd66c1b..6a1a41a13a1b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -21,32 +21,6 @@ namespace llvm { namespace orc { -int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args, - Optional<StringRef> ProgramName) { - std::vector<std::unique_ptr<char[]>> ArgVStorage; - std::vector<char *> ArgV; - - ArgVStorage.reserve(Args.size() + (ProgramName ? 1 : 0)); - ArgV.reserve(Args.size() + 1 + (ProgramName ? 1 : 0)); - - if (ProgramName) { - ArgVStorage.push_back(std::make_unique<char[]>(ProgramName->size() + 1)); - llvm::copy(*ProgramName, &ArgVStorage.back()[0]); - ArgVStorage.back()[ProgramName->size()] = '\0'; - ArgV.push_back(ArgVStorage.back().get()); - } - - for (auto &Arg : Args) { - ArgVStorage.push_back(std::make_unique<char[]>(Arg.size() + 1)); - llvm::copy(Arg, &ArgVStorage.back()[0]); - ArgVStorage.back()[Arg.size()] = '\0'; - ArgV.push_back(ArgVStorage.back().get()); - } - ArgV.push_back(nullptr); - - return Main(Args.size() + !!ProgramName, ArgV.data()); -} - CtorDtorIterator::CtorDtorIterator(const GlobalVariable *GV, bool End) : InitList( GV ? dyn_cast_or_null<ConstantArray>(GV->getInitializer()) : nullptr), @@ -261,8 +235,8 @@ DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, } Error DynamicLibrarySearchGenerator::tryToGenerate( - LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, - const SymbolLookupSet &Symbols) { + LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { orc::SymbolMap NewSymbols; bool HasGlobalPrefix = (GlobalPrefix != '\0'); @@ -322,7 +296,8 @@ StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName, auto ObjTT = Obj.getTriple(); if (ObjTT.getArch() == TT.getArch() && ObjTT.getSubArch() == TT.getSubArch() && - ObjTT.getVendor() == TT.getVendor()) { + (TT.getVendor() == Triple::UnknownVendor || + ObjTT.getVendor() == TT.getVendor())) { // We found a match. Create an instance from a buffer covering this // slice. auto SliceBuffer = MemoryBuffer::getFileSlice(FileName, Obj.getSize(), @@ -364,8 +339,8 @@ StaticLibraryDefinitionGenerator::Create( } Error StaticLibraryDefinitionGenerator::tryToGenerate( - LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, - const SymbolLookupSet &Symbols) { + LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { // Don't materialize symbols from static archives unless this is a static // lookup. @@ -396,8 +371,7 @@ Error StaticLibraryDefinitionGenerator::tryToGenerate( MemoryBufferRef ChildBufferRef(ChildBufferInfo.first, ChildBufferInfo.second); - if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef, false), - VModuleKey())) + if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef, false))) return Err; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp index 023940dc8298..aadc437c80c4 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -25,7 +25,7 @@ void IRCompileLayer::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) { this->NotifyCompiled = std::move(NotifyCompiled); } -void IRCompileLayer::emit(MaterializationResponsibility R, +void IRCompileLayer::emit(std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM) { assert(TSM && "Module must not be null"); @@ -33,13 +33,13 @@ void IRCompileLayer::emit(MaterializationResponsibility R, { std::lock_guard<std::mutex> Lock(IRLayerMutex); if (NotifyCompiled) - NotifyCompiled(R.getVModuleKey(), std::move(TSM)); + NotifyCompiled(*R, std::move(TSM)); else TSM = ThreadSafeModule(); } BaseLayer.emit(std::move(R), std::move(*Obj)); } else { - R.failMaterialization(); + R->failMaterialization(); getExecutionSession().reportError(Obj.takeError()); } } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp index 511248f83b25..d5b11349277c 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -17,14 +17,14 @@ IRTransformLayer::IRTransformLayer(ExecutionSession &ES, IRLayer &BaseLayer, : IRLayer(ES, BaseLayer.getManglingOptions()), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} -void IRTransformLayer::emit(MaterializationResponsibility R, +void IRTransformLayer::emit(std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM) { assert(TSM && "Module must not be null"); - if (auto TransformedTSM = Transform(std::move(TSM), R)) + if (auto TransformedTSM = Transform(std::move(TSM), *R)) BaseLayer.emit(std::move(R), std::move(*TransformedTSM)); else { - R.failMaterialization(); + R->failMaterialization(); getExecutionSession().reportError(TransformedTSM.takeError()); } } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 031b1afefc9d..1cfcf8ae943d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -25,20 +25,20 @@ public: using CompileFunction = JITCompileCallbackManager::CompileFunction; CompileCallbackMaterializationUnit(SymbolStringPtr Name, - CompileFunction Compile, VModuleKey K) + CompileFunction Compile) : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), - nullptr, std::move(K)), + nullptr), Name(std::move(Name)), Compile(std::move(Compile)) {} StringRef getName() const override { return "<Compile Callbacks>"; } private: - void materialize(MaterializationResponsibility R) override { + void materialize(std::unique_ptr<MaterializationResponsibility> R) override { SymbolMap Result; Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); // No dependencies, so these calls cannot fail. - cantFail(R.notifyResolved(Result)); - cantFail(R.notifyEmitted()); + cantFail(R->notifyResolved(Result)); + cantFail(R->notifyEmitted()); } void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { @@ -54,8 +54,8 @@ private: namespace llvm { namespace orc { +TrampolinePool::~TrampolinePool() {} void IndirectStubsManager::anchor() {} -void TrampolinePool::anchor() {} Expected<JITTargetAddress> JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { @@ -65,10 +65,9 @@ JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { std::lock_guard<std::mutex> Lock(CCMgrMutex); AddrToSymbol[*TrampolineAddr] = CallbackName; - cantFail(CallbacksJD.define( - std::make_unique<CompileCallbackMaterializationUnit>( - std::move(CallbackName), std::move(Compile), - ES.allocateVModule()))); + cantFail( + CallbacksJD.define(std::make_unique<CompileCallbackMaterializationUnit>( + std::move(CallbackName), std::move(Compile)))); return *TrampolineAddr; } else return TrampolineAddr.takeError(); @@ -149,7 +148,7 @@ createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, } case Triple::x86_64: { - if ( T.getOS() == Triple::OSType::Win32 ) { + if (T.getOS() == Triple::OSType::Win32) { typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; return CCMgrT::Create(ES, ErrorHandlerAddress); } else { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index 713a48fbf3eb..c368c1e37134 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -11,8 +11,10 @@ #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" -#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" @@ -87,8 +89,9 @@ class GenericLLVMIRPlatform : public Platform { public: GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {} Error setupJITDylib(JITDylib &JD) override; - Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) override; - Error notifyRemoving(JITDylib &JD, VModuleKey K) override { + Error notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) override; + Error notifyRemoving(ResourceTracker &RT) override { // Noop -- Nothing to do (yet). return Error::success(); } @@ -186,7 +189,8 @@ public: return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx))); } - Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) { + Error notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) { + auto &JD = RT.getJITDylib(); if (auto &InitSym = MU.getInitializerSymbol()) InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol); else { @@ -235,7 +239,7 @@ public: }); for (auto DeinitFnAddr : *Deinitializers) { LLVM_DEBUG({ - dbgs() << " Running init " << formatv("{0:x16}", DeinitFnAddr) + dbgs() << " Running deinit " << formatv("{0:x16}", DeinitFnAddr) << "...\n"; }); auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr); @@ -260,23 +264,23 @@ private: return std::move(Err); DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols; - std::vector<JITDylib *> DFSLinkOrder; + std::vector<JITDylibSP> DFSLinkOrder; getExecutionSession().runSessionLocked([&]() { - DFSLinkOrder = getDFSLinkOrder(JD); - - for (auto *NextJD : DFSLinkOrder) { - auto IFItr = InitFunctions.find(NextJD); - if (IFItr != InitFunctions.end()) { - LookupSymbols[NextJD] = std::move(IFItr->second); - InitFunctions.erase(IFItr); - } + DFSLinkOrder = JD.getDFSLinkOrder(); + + for (auto &NextJD : DFSLinkOrder) { + auto IFItr = InitFunctions.find(NextJD.get()); + if (IFItr != InitFunctions.end()) { + LookupSymbols[NextJD.get()] = std::move(IFItr->second); + InitFunctions.erase(IFItr); } - }); + } + }); LLVM_DEBUG({ dbgs() << "JITDylib init order is [ "; - for (auto *JD : llvm::reverse(DFSLinkOrder)) + for (auto &JD : llvm::reverse(DFSLinkOrder)) dbgs() << "\"" << JD->getName() << "\" "; dbgs() << "]\n"; dbgs() << "Looking up init functions:\n"; @@ -310,26 +314,26 @@ private: auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits"); DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols; - std::vector<JITDylib *> DFSLinkOrder; + std::vector<JITDylibSP> DFSLinkOrder; ES.runSessionLocked([&]() { - DFSLinkOrder = getDFSLinkOrder(JD); - - for (auto *NextJD : DFSLinkOrder) { - auto &JDLookupSymbols = LookupSymbols[NextJD]; - auto DIFItr = DeInitFunctions.find(NextJD); - if (DIFItr != DeInitFunctions.end()) { - LookupSymbols[NextJD] = std::move(DIFItr->second); - DeInitFunctions.erase(DIFItr); - } - JDLookupSymbols.add(LLJITRunAtExits, + DFSLinkOrder = JD.getDFSLinkOrder(); + + for (auto &NextJD : DFSLinkOrder) { + auto &JDLookupSymbols = LookupSymbols[NextJD.get()]; + auto DIFItr = DeInitFunctions.find(NextJD.get()); + if (DIFItr != DeInitFunctions.end()) { + LookupSymbols[NextJD.get()] = std::move(DIFItr->second); + DeInitFunctions.erase(DIFItr); + } + JDLookupSymbols.add(LLJITRunAtExits, SymbolLookupFlags::WeaklyReferencedSymbol); } }); LLVM_DEBUG({ dbgs() << "JITDylib deinit order is [ "; - for (auto *JD : DFSLinkOrder) + for (auto &JD : DFSLinkOrder) dbgs() << "\"" << JD->getName() << "\" "; dbgs() << "]\n"; dbgs() << "Looking up deinit functions:\n"; @@ -343,8 +347,8 @@ private: return LookupResult.takeError(); std::vector<JITTargetAddress> DeInitializers; - for (auto *NextJD : DFSLinkOrder) { - auto DeInitsItr = LookupResult->find(NextJD); + for (auto &NextJD : DFSLinkOrder) { + auto DeInitsItr = LookupResult->find(NextJD.get()); assert(DeInitsItr != LookupResult->end() && "Every JD should have at least __lljit_run_atexits"); @@ -360,46 +364,23 @@ private: return DeInitializers; } - // Returns a DFS traversal order of the JITDylibs reachable (via - // links-against edges) from JD, starting with JD itself. - static std::vector<JITDylib *> getDFSLinkOrder(JITDylib &JD) { - std::vector<JITDylib *> DFSLinkOrder; - std::vector<JITDylib *> WorkStack({&JD}); - DenseSet<JITDylib *> Visited; - - while (!WorkStack.empty()) { - auto &NextJD = *WorkStack.back(); - WorkStack.pop_back(); - if (Visited.count(&NextJD)) - continue; - Visited.insert(&NextJD); - DFSLinkOrder.push_back(&NextJD); - NextJD.withLinkOrderDo([&](const JITDylibSearchOrder &LinkOrder) { - for (auto &KV : LinkOrder) - WorkStack.push_back(KV.first); - }); - } - - return DFSLinkOrder; - } - /// Issue lookups for all init symbols required to initialize JD (and any /// JITDylibs that it depends on). Error issueInitLookups(JITDylib &JD) { DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols; - std::vector<JITDylib *> DFSLinkOrder; + std::vector<JITDylibSP> DFSLinkOrder; getExecutionSession().runSessionLocked([&]() { - DFSLinkOrder = getDFSLinkOrder(JD); - - for (auto *NextJD : DFSLinkOrder) { - auto ISItr = InitSymbols.find(NextJD); - if (ISItr != InitSymbols.end()) { - RequiredInitSymbols[NextJD] = std::move(ISItr->second); - InitSymbols.erase(ISItr); - } + DFSLinkOrder = JD.getDFSLinkOrder(); + + for (auto &NextJD : DFSLinkOrder) { + auto ISItr = InitSymbols.find(NextJD.get()); + if (ISItr != InitSymbols.end()) { + RequiredInitSymbols[NextJD.get()] = std::move(ISItr->second); + InitSymbols.erase(ISItr); } - }); + } + }); return Platform::lookupInitSymbols(getExecutionSession(), RequiredInitSymbols) @@ -468,9 +449,9 @@ Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) { return S.setupJITDylib(JD); } -Error GenericLLVMIRPlatform::notifyAdding(JITDylib &JD, +Error GenericLLVMIRPlatform::notifyAdding(ResourceTracker &RT, const MaterializationUnit &MU) { - return S.notifyAdding(JD, MU); + return S.notifyAdding(RT, MU); } Expected<ThreadSafeModule> @@ -927,7 +908,7 @@ LLJIT::PlatformSupport::~PlatformSupport() {} Error LLJITBuilderState::prepareForConstruction() { - LLVM_DEBUG(dbgs() << "Preparing to create LLIT instance...\n"); + LLVM_DEBUG(dbgs() << "Preparing to create LLJIT instance...\n"); if (!JTMB) { LLVM_DEBUG({ @@ -973,12 +954,18 @@ Error LLJITBuilderState::prepareForConstruction() { JTMB->setRelocationModel(Reloc::PIC_); JTMB->setCodeModel(CodeModel::Small); CreateObjectLinkingLayer = - [](ExecutionSession &ES, - const Triple &) -> std::unique_ptr<ObjectLayer> { - auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>( - ES, std::make_unique<jitlink::InProcessMemoryManager>()); + [TPC = this->TPC]( + ExecutionSession &ES, + const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> { + std::unique_ptr<ObjectLinkingLayer> ObjLinkingLayer; + if (TPC) + ObjLinkingLayer = + std::make_unique<ObjectLinkingLayer>(ES, TPC->getMemMgr()); + else + ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>( + ES, std::make_unique<jitlink::InProcessMemoryManager>()); ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>( - jitlink::InProcessEHFrameRegistrar::getInstance())); + ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>())); return std::move(ObjLinkingLayer); }; } @@ -990,23 +977,33 @@ Error LLJITBuilderState::prepareForConstruction() { LLJIT::~LLJIT() { if (CompileThreads) CompileThreads->wait(); + if (auto Err = ES->endSession()) + ES->reportError(std::move(Err)); } -Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { +Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) { assert(TSM && "Can not add null module"); if (auto Err = TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); })) return Err; - return InitHelperTransformLayer->add(JD, std::move(TSM), - ES->allocateVModule()); + return InitHelperTransformLayer->add(std::move(RT), std::move(TSM)); } -Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { +Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { + return addIRModule(JD.getDefaultResourceTracker(), std::move(TSM)); +} + +Error LLJIT::addObjectFile(ResourceTrackerSP RT, + std::unique_ptr<MemoryBuffer> Obj) { assert(Obj && "Can not add null object"); - return ObjTransformLayer.add(JD, std::move(Obj), ES->allocateVModule()); + return ObjTransformLayer->add(std::move(RT), std::move(Obj)); +} + +Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { + return addObjectFile(JD.getDefaultResourceTracker(), std::move(Obj)); } Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, @@ -1015,7 +1012,7 @@ Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name); } -std::unique_ptr<ObjectLayer> +Expected<std::unique_ptr<ObjectLayer>> LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { // If the config state provided an ObjectLinkingLayer factory then use it. @@ -1061,9 +1058,7 @@ LLJIT::createCompileFunction(LLJITBuilderState &S, LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), Main(), - DL(""), TT(S.JTMB->getTargetTriple()), - ObjLinkingLayer(createObjectLinkingLayer(S, *ES)), - ObjTransformLayer(*this->ES, *ObjLinkingLayer) { + DL(""), TT(S.JTMB->getTargetTriple()) { ErrorAsOutParameter _(&Err); @@ -1083,6 +1078,15 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) return; } + auto ObjLayer = createObjectLinkingLayer(S, *ES); + if (!ObjLayer) { + Err = ObjLayer.takeError(); + return; + } + ObjLinkingLayer = std::move(*ObjLayer); + ObjTransformLayer = + std::make_unique<ObjectTransformLayer>(*ES, *ObjLinkingLayer); + { auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB)); if (!CompileFunction) { @@ -1090,7 +1094,7 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) return; } CompileLayer = std::make_unique<IRCompileLayer>( - *ES, ObjTransformLayer, std::move(*CompileFunction)); + *ES, *ObjTransformLayer, std::move(*CompileFunction)); TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer); InitHelperTransformLayer = std::make_unique<IRTransformLayer>(*ES, *TransformLayer); @@ -1102,15 +1106,17 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads)); ES->setDispatchMaterialization( [this](std::unique_ptr<MaterializationUnit> MU, - MaterializationResponsibility MR) { - // FIXME: Switch to move capture once ThreadPool uses unique_function. - auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU)); - auto SharedMR = - std::make_shared<MaterializationResponsibility>(std::move(MR)); - auto Work = [SharedMU, SharedMR]() mutable { - SharedMU->materialize(std::move(*SharedMR)); - }; - CompileThreads->async(std::move(Work)); + std::unique_ptr<MaterializationResponsibility> MR) { + // FIXME: We should be able to use move-capture here, but ThreadPool's + // AsyncTaskTys are std::functions rather than unique_functions + // (because MSVC's std::packaged_tasks don't support move-only types). + // Fix this when all the above gets sorted out. + CompileThreads->async( + [UnownedMU = MU.release(), UnownedMR = MR.release()]() mutable { + std::unique_ptr<MaterializationUnit> MU(UnownedMU); + std::unique_ptr<MaterializationResponsibility> MR(UnownedMR); + MU->materialize(std::move(MR)); + }); }); } @@ -1172,7 +1178,7 @@ Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { [&](Module &M) -> Error { return applyDataLayout(M); })) return Err; - return CODLayer->add(JD, std::move(TSM), ES->allocateVModule()); + return CODLayer->add(JD, std::move(TSM)); } LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp index 61e7ab5ae68b..5e27e343d23b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -22,16 +22,18 @@ namespace orc { IRLayer::~IRLayer() {} -Error IRLayer::add(JITDylib &JD, ThreadSafeModule TSM, VModuleKey K) { +Error IRLayer::add(ResourceTrackerSP RT, ThreadSafeModule TSM) { + assert(RT && "RT can not be null"); + auto &JD = RT->getJITDylib(); return JD.define(std::make_unique<BasicIRLayerMaterializationUnit>( - *this, *getManglingOptions(), std::move(TSM), std::move(K))); + *this, *getManglingOptions(), std::move(TSM)), + std::move(RT)); } IRMaterializationUnit::IRMaterializationUnit( ExecutionSession &ES, const IRSymbolMapper::ManglingOptions &MO, - ThreadSafeModule TSM, VModuleKey K) - : MaterializationUnit(SymbolFlagsMap(), nullptr, std::move(K)), - TSM(std::move(TSM)) { + ThreadSafeModule TSM) + : MaterializationUnit(SymbolFlagsMap(), nullptr), TSM(std::move(TSM)) { assert(this->TSM && "Module must not be null"); @@ -83,26 +85,22 @@ IRMaterializationUnit::IRMaterializationUnit( if (!llvm::empty(getStaticInitGVs(M))) { size_t Counter = 0; - while (true) { + do { std::string InitSymbolName; raw_string_ostream(InitSymbolName) << "$." << M.getModuleIdentifier() << ".__inits." << Counter++; InitSymbol = ES.intern(InitSymbolName); - if (SymbolFlags.count(InitSymbol)) - continue; - SymbolFlags[InitSymbol] = - JITSymbolFlags::MaterializationSideEffectsOnly; - break; - } + } while (SymbolFlags.count(InitSymbol)); + + SymbolFlags[InitSymbol] = JITSymbolFlags::MaterializationSideEffectsOnly; } }); } IRMaterializationUnit::IRMaterializationUnit( - ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, + ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition) - : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol), - std::move(K)), + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol)), TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {} StringRef IRMaterializationUnit::getName() const { @@ -129,14 +127,12 @@ void IRMaterializationUnit::discard(const JITDylib &JD, } BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit( - IRLayer &L, const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, - VModuleKey K) - : IRMaterializationUnit(L.getExecutionSession(), MO, std::move(TSM), - std::move(K)), - L(L), K(std::move(K)) {} + IRLayer &L, const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM) + : IRMaterializationUnit(L.getExecutionSession(), MO, std::move(TSM)), L(L) { +} void BasicIRLayerMaterializationUnit::materialize( - MaterializationResponsibility R) { + std::unique_ptr<MaterializationResponsibility> R) { // Throw away the SymbolToDefinition map: it's not usable after we hand // off the module. @@ -147,8 +143,8 @@ void BasicIRLayerMaterializationUnit::materialize( TSM = cloneToNewContext(TSM); #ifndef NDEBUG - auto &ES = R.getTargetJITDylib().getExecutionSession(); - auto &N = R.getTargetJITDylib().getName(); + auto &ES = R->getTargetJITDylib().getExecutionSession(); + auto &N = R->getTargetJITDylib().getName(); #endif // NDEBUG LLVM_DEBUG(ES.runSessionLocked( @@ -163,17 +159,17 @@ ObjectLayer::ObjectLayer(ExecutionSession &ES) : ES(ES) {} ObjectLayer::~ObjectLayer() {} -Error ObjectLayer::add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O, - VModuleKey K) { - auto ObjMU = BasicObjectLayerMaterializationUnit::Create(*this, std::move(K), - std::move(O)); +Error ObjectLayer::add(ResourceTrackerSP RT, std::unique_ptr<MemoryBuffer> O) { + assert(RT && "RT can not be null"); + auto ObjMU = BasicObjectLayerMaterializationUnit::Create(*this, std::move(O)); if (!ObjMU) return ObjMU.takeError(); - return JD.define(std::move(*ObjMU)); + auto &JD = RT->getJITDylib(); + return JD.define(std::move(*ObjMU), std::move(RT)); } Expected<std::unique_ptr<BasicObjectLayerMaterializationUnit>> -BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L, VModuleKey K, +BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> O) { auto ObjSymInfo = getObjectSymbolInfo(L.getExecutionSession(), O->getMemBufferRef()); @@ -186,15 +182,14 @@ BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L, VModuleKey K, return std::unique_ptr<BasicObjectLayerMaterializationUnit>( new BasicObjectLayerMaterializationUnit( - L, K, std::move(O), std::move(SymbolFlags), std::move(InitSymbol))); + L, std::move(O), std::move(SymbolFlags), std::move(InitSymbol))); } BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit( - ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O, - SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) - : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol), - std::move(K)), - L(L), O(std::move(O)) {} + ObjectLayer &L, std::unique_ptr<MemoryBuffer> O, SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol) + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol)), L(L), + O(std::move(O)) {} StringRef BasicObjectLayerMaterializationUnit::getName() const { if (O) @@ -203,7 +198,7 @@ StringRef BasicObjectLayerMaterializationUnit::getName() const { } void BasicObjectLayerMaterializationUnit::materialize( - MaterializationResponsibility R) { + std::unique_ptr<MaterializationResponsibility> R) { L.emit(std::move(R), std::move(O)); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp index 153f6b80784f..e1f494415e86 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -75,27 +75,31 @@ void LazyCallThroughManager::resolveTrampolineLandingAddress( if (!Entry) return NotifyLandingResolved(reportCallThroughError(Entry.takeError())); - ES.lookup( - LookupKind::Static, - makeJITDylibSearchOrder(Entry->SourceJD, - JITDylibLookupFlags::MatchAllSymbols), - SymbolLookupSet({Entry->SymbolName}), SymbolState::Ready, - [this, TrampolineAddr, SymbolName = Entry->SymbolName, - NotifyLandingResolved = std::move(NotifyLandingResolved)]( - Expected<SymbolMap> Result) mutable { - if (Result) { - assert(Result->size() == 1 && "Unexpected result size"); - assert(Result->count(SymbolName) && "Unexpected result value"); - JITTargetAddress LandingAddr = (*Result)[SymbolName].getAddress(); - - if (auto Err = notifyResolved(TrampolineAddr, LandingAddr)) - NotifyLandingResolved(reportCallThroughError(std::move(Err))); - else - NotifyLandingResolved(LandingAddr); - } else - NotifyLandingResolved(reportCallThroughError(Result.takeError())); - }, - NoDependenciesToRegister); + // Declaring SLS and the callback outside of the call to ES.lookup is a + // workaround to fix build failures on AIX and on z/OS platforms. + SymbolLookupSet SLS({Entry->SymbolName}); + auto Callback = [this, TrampolineAddr, SymbolName = Entry->SymbolName, + NotifyLandingResolved = std::move(NotifyLandingResolved)]( + Expected<SymbolMap> Result) mutable { + if (Result) { + assert(Result->size() == 1 && "Unexpected result size"); + assert(Result->count(SymbolName) && "Unexpected result value"); + JITTargetAddress LandingAddr = (*Result)[SymbolName].getAddress(); + + if (auto Err = notifyResolved(TrampolineAddr, LandingAddr)) + NotifyLandingResolved(reportCallThroughError(std::move(Err))); + else + NotifyLandingResolved(LandingAddr); + } else { + NotifyLandingResolved(reportCallThroughError(Result.takeError())); + } + }; + + ES.lookup(LookupKind::Static, + makeJITDylibSearchOrder(Entry->SourceJD, + JITDylibLookupFlags::MatchAllSymbols), + std::move(SLS), SymbolState::Ready, std::move(Callback), + NoDependenciesToRegister); } Expected<std::unique_ptr<LazyCallThroughManager>> @@ -139,9 +143,8 @@ createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit( LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, - JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc, - VModuleKey K) - : MaterializationUnit(extractFlags(CallableAliases), nullptr, std::move(K)), + JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc) + : MaterializationUnit(extractFlags(CallableAliases), nullptr), LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD), CallableAliases(std::move(CallableAliases)), AliaseeTable(SrcJDLoc) {} @@ -150,8 +153,8 @@ StringRef LazyReexportsMaterializationUnit::getName() const { } void LazyReexportsMaterializationUnit::materialize( - MaterializationResponsibility R) { - auto RequestedSymbols = R.getRequestedSymbols(); + std::unique_ptr<MaterializationResponsibility> R) { + auto RequestedSymbols = R->getRequestedSymbols(); SymbolAliasMap RequestedAliases; for (auto &RequestedSymbol : RequestedSymbols) { @@ -162,8 +165,13 @@ void LazyReexportsMaterializationUnit::materialize( } if (!CallableAliases.empty()) - R.replace(lazyReexports(LCTManager, ISManager, SourceJD, - std::move(CallableAliases), AliaseeTable)); + if (auto Err = R->replace(lazyReexports(LCTManager, ISManager, SourceJD, + std::move(CallableAliases), + AliaseeTable))) { + R->getExecutionSession().reportError(std::move(Err)); + R->failMaterialization(); + return; + } IndirectStubsManager::StubInitsMap StubInits; for (auto &Alias : RequestedAliases) { @@ -178,7 +186,7 @@ void LazyReexportsMaterializationUnit::materialize( if (!CallThroughTrampoline) { SourceJD.getExecutionSession().reportError( CallThroughTrampoline.takeError()); - R.failMaterialization(); + R->failMaterialization(); return; } @@ -191,7 +199,7 @@ void LazyReexportsMaterializationUnit::materialize( if (auto Err = ISManager.createStubs(StubInits)) { SourceJD.getExecutionSession().reportError(std::move(Err)); - R.failMaterialization(); + R->failMaterialization(); return; } @@ -200,8 +208,8 @@ void LazyReexportsMaterializationUnit::materialize( Stubs[Alias.first] = ISManager.findStub(*Alias.first, false); // No registered dependencies, so these calls cannot fail. - cantFail(R.notifyResolved(Stubs)); - cantFail(R.notifyEmitted()); + cantFail(R->notifyResolved(Stubs)); + cantFail(R->notifyEmitted()); } void LazyReexportsMaterializationUnit::discard(const JITDylib &JD, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Legacy.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Legacy.cpp deleted file mode 100644 index 67b804c37287..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Legacy.cpp +++ /dev/null @@ -1,68 +0,0 @@ -//===------- Legacy.cpp - Adapters for ExecutionEngine API interop --------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/Orc/Legacy.h" - -namespace llvm { -namespace orc { - -void SymbolResolver::anchor() {} - -JITSymbolResolverAdapter::JITSymbolResolverAdapter( - ExecutionSession &ES, SymbolResolver &R, MaterializationResponsibility *MR) - : ES(ES), R(R), MR(MR) {} - -void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols, - OnResolvedFunction OnResolved) { - SymbolNameSet InternedSymbols; - for (auto &S : Symbols) - InternedSymbols.insert(ES.intern(S)); - - auto OnResolvedWithUnwrap = [OnResolved = std::move(OnResolved)]( - Expected<SymbolMap> InternedResult) mutable { - if (!InternedResult) { - OnResolved(InternedResult.takeError()); - return; - } - - LookupResult Result; - for (auto &KV : *InternedResult) - Result[*KV.first] = std::move(KV.second); - OnResolved(Result); - }; - - auto Q = std::make_shared<AsynchronousSymbolQuery>( - SymbolLookupSet(InternedSymbols), SymbolState::Resolved, - std::move(OnResolvedWithUnwrap)); - - auto Unresolved = R.lookup(Q, InternedSymbols); - if (Unresolved.empty()) { - if (MR) - MR->addDependenciesForAll(Q->QueryRegistrations); - } else - ES.legacyFailQuery(*Q, make_error<SymbolsNotFound>(std::move(Unresolved))); -} - -Expected<JITSymbolResolverAdapter::LookupSet> -JITSymbolResolverAdapter::getResponsibilitySet(const LookupSet &Symbols) { - SymbolNameSet InternedSymbols; - for (auto &S : Symbols) - InternedSymbols.insert(ES.intern(S)); - - auto InternedResult = R.getResponsibilitySet(InternedSymbols); - LookupSet Result; - for (auto &S : InternedResult) { - ResolvedStrings.insert(S); - Result.insert(*S); - } - - return Result; -} - -} // End namespace orc. -} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 15c3aa79a2a8..17b9465a0541 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -159,7 +159,9 @@ Error MachOPlatform::setupJITDylib(JITDylib &JD) { return ObjLinkingLayer.add(JD, std::move(ObjBuffer)); } -Error MachOPlatform::notifyAdding(JITDylib &JD, const MaterializationUnit &MU) { +Error MachOPlatform::notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) { + auto &JD = RT.getJITDylib(); const auto &InitSym = MU.getInitializerSymbol(); if (!InitSym) return Error::success(); @@ -173,7 +175,7 @@ Error MachOPlatform::notifyAdding(JITDylib &JD, const MaterializationUnit &MU) { return Error::success(); } -Error MachOPlatform::notifyRemoving(JITDylib &JD, VModuleKey K) { +Error MachOPlatform::notifyRemoving(ResourceTracker &RT) { llvm_unreachable("Not supported yet"); } @@ -185,19 +187,19 @@ MachOPlatform::getInitializerSequence(JITDylib &JD) { << JD.getName() << "\n"; }); - std::vector<JITDylib *> DFSLinkOrder; + std::vector<JITDylibSP> DFSLinkOrder; while (true) { DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; ES.runSessionLocked([&]() { - DFSLinkOrder = getDFSLinkOrder(JD); + DFSLinkOrder = JD.getDFSLinkOrder(); - for (auto *InitJD : DFSLinkOrder) { - auto RISItr = RegisteredInitSymbols.find(InitJD); + for (auto &InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD.get()); if (RISItr != RegisteredInitSymbols.end()) { - NewInitSymbols[InitJD] = std::move(RISItr->second); + NewInitSymbols[InitJD.get()] = std::move(RISItr->second); RegisteredInitSymbols.erase(RISItr); } } @@ -229,14 +231,14 @@ MachOPlatform::getInitializerSequence(JITDylib &JD) { InitializerSequence FullInitSeq; { std::lock_guard<std::mutex> Lock(InitSeqsMutex); - for (auto *InitJD : reverse(DFSLinkOrder)) { + for (auto &InitJD : reverse(DFSLinkOrder)) { LLVM_DEBUG({ dbgs() << "MachOPlatform: Appending inits for \"" << InitJD->getName() << "\" to sequence\n"; }); - auto ISItr = InitSeqs.find(InitJD); + auto ISItr = InitSeqs.find(InitJD.get()); if (ISItr != InitSeqs.end()) { - FullInitSeq.emplace_back(InitJD, std::move(ISItr->second)); + FullInitSeq.emplace_back(InitJD.get(), std::move(ISItr->second)); InitSeqs.erase(ISItr); } } @@ -247,39 +249,19 @@ MachOPlatform::getInitializerSequence(JITDylib &JD) { Expected<MachOPlatform::DeinitializerSequence> MachOPlatform::getDeinitializerSequence(JITDylib &JD) { - std::vector<JITDylib *> DFSLinkOrder = getDFSLinkOrder(JD); + std::vector<JITDylibSP> DFSLinkOrder = JD.getDFSLinkOrder(); DeinitializerSequence FullDeinitSeq; { std::lock_guard<std::mutex> Lock(InitSeqsMutex); - for (auto *DeinitJD : DFSLinkOrder) { - FullDeinitSeq.emplace_back(DeinitJD, MachOJITDylibDeinitializers()); + for (auto &DeinitJD : DFSLinkOrder) { + FullDeinitSeq.emplace_back(DeinitJD.get(), MachOJITDylibDeinitializers()); } } return FullDeinitSeq; } -std::vector<JITDylib *> MachOPlatform::getDFSLinkOrder(JITDylib &JD) { - std::vector<JITDylib *> Result, WorkStack({&JD}); - DenseSet<JITDylib *> Visited; - - while (!WorkStack.empty()) { - auto *NextJD = WorkStack.back(); - WorkStack.pop_back(); - if (Visited.count(NextJD)) - continue; - Visited.insert(NextJD); - Result.push_back(NextJD); - NextJD->withLinkOrderDo([&](const JITDylibSearchOrder &LO) { - for (auto &KV : LO) - WorkStack.push_back(KV.first); - }); - } - - return Result; -} - void MachOPlatform::registerInitInfo( JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, MachOJITDylibInitializers::SectionExtent ModInits, @@ -319,13 +301,16 @@ void MachOPlatform::InitScraperPlugin::modifyPassConfig( MaterializationResponsibility &MR, const Triple &TT, jitlink::PassConfiguration &Config) { + if (!MR.getInitializerSymbol()) + return; + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { JITLinkSymbolVector InitSectionSymbols; preserveInitSectionIfPresent(InitSectionSymbols, G, "__mod_init_func"); preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_selrefs"); preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_classlist"); - if (!InitSymbolDeps.empty()) { + if (!InitSectionSymbols.empty()) { std::lock_guard<std::mutex> Lock(InitScraperMutex); InitSymbolDeps[&MR] = std::move(InitSectionSymbols); } @@ -343,10 +328,8 @@ void MachOPlatform::InitScraperPlugin::modifyPassConfig( JITTargetAddress ObjCImageInfoAddr = 0; if (auto *ObjCImageInfoSec = G.findSectionByName("__objc_image_info")) { - if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) { + if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) ObjCImageInfoAddr = Addr; - dbgs() << "Recorded __objc_imageinfo @ " << formatv("{0:x16}", Addr); - } } // Record __mod_init_func. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp deleted file mode 100644 index 5b4345b870bb..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/NullResolver.cpp +++ /dev/null @@ -1,37 +0,0 @@ -//===---------- NullResolver.cpp - Reject symbol lookup requests ----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/Orc/NullResolver.h" - -#include "llvm/Support/ErrorHandling.h" - -namespace llvm { -namespace orc { - -SymbolNameSet NullResolver::getResponsibilitySet(const SymbolNameSet &Symbols) { - return Symbols; -} - -SymbolNameSet -NullResolver::lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, - SymbolNameSet Symbols) { - assert(Symbols.empty() && "Null resolver: Symbols must be empty"); - return Symbols; -} - -JITSymbol NullLegacyResolver::findSymbol(const std::string &Name) { - llvm_unreachable("Unexpected cross-object symbol reference"); -} - -JITSymbol -NullLegacyResolver::findSymbolInLogicalDylib(const std::string &Name) { - llvm_unreachable("Unexpected cross-object symbol reference"); -} - -} // End namespace orc. -} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 02066b458dfc..26f77acd91fc 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -24,34 +24,34 @@ namespace orc { class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { public: - ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer &Layer, - MaterializationResponsibility MR, - std::unique_ptr<MemoryBuffer> ObjBuffer) - : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} + ObjectLinkingLayerJITLinkContext( + ObjectLinkingLayer &Layer, + std::unique_ptr<MaterializationResponsibility> MR, + std::unique_ptr<MemoryBuffer> ObjBuffer) + : JITLinkContext(&MR->getTargetJITDylib()), Layer(Layer), + MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} ~ObjectLinkingLayerJITLinkContext() { // If there is an object buffer return function then use it to // return ownership of the buffer. - if (Layer.ReturnObjectBuffer) + if (Layer.ReturnObjectBuffer && ObjBuffer) Layer.ReturnObjectBuffer(std::move(ObjBuffer)); } - JITLinkMemoryManager &getMemoryManager() override { return *Layer.MemMgr; } - - MemoryBufferRef getObjectBuffer() const override { - return ObjBuffer->getMemBufferRef(); - } + JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } void notifyFailed(Error Err) override { + for (auto &P : Layer.Plugins) + Err = joinErrors(std::move(Err), P->notifyFailed(*MR)); Layer.getExecutionSession().reportError(std::move(Err)); - MR.failMaterialization(); + MR->failMaterialization(); } void lookup(const LookupMap &Symbols, std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { JITDylibSearchOrder LinkOrder; - MR.getTargetJITDylib().withLinkOrderDo( + MR->getTargetJITDylib().withLinkOrderDo( [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); auto &ES = Layer.getExecutionSession(); @@ -71,9 +71,8 @@ public: } // OnResolve -- De-intern the symbols and pass the result to the linker. - auto OnResolve = [this, LookupContinuation = std::move(LC)]( - Expected<SymbolMap> Result) mutable { - auto Main = Layer.getExecutionSession().intern("_main"); + auto OnResolve = [LookupContinuation = + std::move(LC)](Expected<SymbolMap> Result) mutable { if (!Result) LookupContinuation->run(Result.takeError()); else { @@ -86,8 +85,8 @@ public: for (auto &KV : InternalNamedSymbolDeps) { SymbolDependenceMap InternalDeps; - InternalDeps[&MR.getTargetJITDylib()] = std::move(KV.second); - MR.addDependencies(KV.first, InternalDeps); + InternalDeps[&MR->getTargetJITDylib()] = std::move(KV.second); + MR->addDependencies(KV.first, InternalDeps); } ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet), @@ -97,7 +96,7 @@ public: }); } - void notifyResolved(LinkGraph &G) override { + Error notifyResolved(LinkGraph &G) override { auto &ES = Layer.getExecutionSession(); SymbolFlagsMap ExtraSymbolsToClaim; @@ -116,7 +115,7 @@ public: InternedResult[InternedName] = JITEvaluatedSymbol(Sym->getAddress(), Flags); - if (AutoClaim && !MR.getSymbols().count(InternedName)) { + if (AutoClaim && !MR->getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); ExtraSymbolsToClaim[InternedName] = Flags; @@ -134,7 +133,7 @@ public: Flags |= JITSymbolFlags::Weak; InternedResult[InternedName] = JITEvaluatedSymbol(Sym->getAddress(), Flags); - if (AutoClaim && !MR.getSymbols().count(InternedName)) { + if (AutoClaim && !MR->getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); ExtraSymbolsToClaim[InternedName] = Flags; @@ -142,19 +141,19 @@ public: } if (!ExtraSymbolsToClaim.empty()) - if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) - return notifyFailed(std::move(Err)); + if (auto Err = MR->defineMaterializing(ExtraSymbolsToClaim)) + return Err; { - // Check that InternedResult matches up with MR.getSymbols(). + // Check that InternedResult matches up with MR->getSymbols(). // This guards against faulty transformations / compilers / object caches. // First check that there aren't any missing symbols. size_t NumMaterializationSideEffectsOnlySymbols = 0; SymbolNameVector ExtraSymbols; SymbolNameVector MissingSymbols; - for (auto &KV : MR.getSymbols()) { + for (auto &KV : MR->getSymbols()) { // If this is a materialization-side-effects only symbol then bump // the counter and make sure it's *not* defined, otherwise make @@ -169,49 +168,42 @@ public: } // If there were missing symbols then report the error. - if (!MissingSymbols.empty()) { - ES.reportError(make_error<MissingSymbolDefinitions>( - G.getName(), std::move(MissingSymbols))); - MR.failMaterialization(); - return; - } + if (!MissingSymbols.empty()) + return make_error<MissingSymbolDefinitions>(G.getName(), + std::move(MissingSymbols)); // If there are more definitions than expected, add them to the // ExtraSymbols vector. if (InternedResult.size() > - MR.getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) { + MR->getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) { for (auto &KV : InternedResult) - if (!MR.getSymbols().count(KV.first)) + if (!MR->getSymbols().count(KV.first)) ExtraSymbols.push_back(KV.first); } // If there were extra definitions then report the error. - if (!ExtraSymbols.empty()) { - ES.reportError(make_error<UnexpectedSymbolDefinitions>( - G.getName(), std::move(ExtraSymbols))); - MR.failMaterialization(); - return; - } + if (!ExtraSymbols.empty()) + return make_error<UnexpectedSymbolDefinitions>(G.getName(), + std::move(ExtraSymbols)); } - if (auto Err = MR.notifyResolved(InternedResult)) { - Layer.getExecutionSession().reportError(std::move(Err)); - MR.failMaterialization(); - return; - } - Layer.notifyLoaded(MR); + if (auto Err = MR->notifyResolved(InternedResult)) + return Err; + + Layer.notifyLoaded(*MR); + return Error::success(); } void notifyFinalized( std::unique_ptr<JITLinkMemoryManager::Allocation> A) override { - if (auto Err = Layer.notifyEmitted(MR, std::move(A))) { + if (auto Err = Layer.notifyEmitted(*MR, std::move(A))) { Layer.getExecutionSession().reportError(std::move(Err)); - MR.failMaterialization(); + MR->failMaterialization(); return; } - if (auto Err = MR.notifyEmitted()) { + if (auto Err = MR->notifyEmitted()) { Layer.getExecutionSession().reportError(std::move(Err)); - MR.failMaterialization(); + MR->failMaterialization(); } } @@ -222,10 +214,11 @@ public: Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { // Add passes to mark duplicate defs as should-discard, and to walk the // link graph to build the symbol dependence graph. - Config.PrePrunePasses.push_back( - [this](LinkGraph &G) { return externalizeWeakAndCommonSymbols(G); }); + Config.PrePrunePasses.push_back([this](LinkGraph &G) { + return claimOrExternalizeWeakAndCommonSymbols(G); + }); - Layer.modifyPassConfig(MR, TT, Config); + Layer.modifyPassConfig(*MR, TT, Config); Config.PostPrunePasses.push_back( [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); }); @@ -241,19 +234,38 @@ private: using LocalSymbolNamedDependenciesMap = DenseMap<const Symbol *, LocalSymbolNamedDependencies>; - Error externalizeWeakAndCommonSymbols(LinkGraph &G) { + Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) { auto &ES = Layer.getExecutionSession(); - for (auto *Sym : G.defined_symbols()) + + SymbolFlagsMap NewSymbolsToClaim; + std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym; + + auto ProcessSymbol = [&](Symbol *Sym) { if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) { - if (!MR.getSymbols().count(ES.intern(Sym->getName()))) - G.makeExternal(*Sym); + auto Name = ES.intern(Sym->getName()); + if (!MR->getSymbols().count(ES.intern(Sym->getName()))) { + JITSymbolFlags SF = JITSymbolFlags::Weak; + if (Sym->getScope() == Scope::Default) + SF |= JITSymbolFlags::Exported; + NewSymbolsToClaim[Name] = SF; + NameToSym.push_back(std::make_pair(std::move(Name), Sym)); + } } + }; + for (auto *Sym : G.defined_symbols()) + ProcessSymbol(Sym); for (auto *Sym : G.absolute_symbols()) - if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) { - if (!MR.getSymbols().count(ES.intern(Sym->getName()))) - G.makeExternal(*Sym); - } + ProcessSymbol(Sym); + + // Attempt to claim all weak defs that we're not already responsible for. + // This cannot fail -- any clashes will just result in rejection of our + // claim, at which point we'll externalize that symbol. + cantFail(MR->defineMaterializing(std::move(NewSymbolsToClaim))); + + for (auto &KV : NameToSym) + if (!MR->getSymbols().count(KV.first)) + G.makeExternal(*KV.second); return Error::success(); } @@ -261,13 +273,13 @@ private: Error markResponsibilitySymbolsLive(LinkGraph &G) const { auto &ES = Layer.getExecutionSession(); for (auto *Sym : G.defined_symbols()) - if (Sym->hasName() && MR.getSymbols().count(ES.intern(Sym->getName()))) + if (Sym->hasName() && MR->getSymbols().count(ES.intern(Sym->getName()))) Sym->setLive(true); return Error::success(); } Error computeNamedSymbolDependencies(LinkGraph &G) { - auto &ES = MR.getTargetJITDylib().getExecutionSession(); + auto &ES = MR->getTargetJITDylib().getExecutionSession(); auto LocalDeps = computeLocalDeps(G); // Compute dependencies for symbols defined in the JITLink graph. @@ -314,7 +326,7 @@ private: } for (auto &P : Layer.Plugins) { - auto SyntheticLocalDeps = P->getSyntheticSymbolLocalDependencies(MR); + auto SyntheticLocalDeps = P->getSyntheticSymbolLocalDependencies(*MR); if (SyntheticLocalDeps.empty()) continue; @@ -434,12 +446,12 @@ private: SymbolDeps.erase(&SourceJD); } - MR.addDependencies(Name, SymbolDeps); + MR->addDependencies(Name, SymbolDeps); } } ObjectLinkingLayer &Layer; - MaterializationResponsibility MR; + std::unique_ptr<MaterializationResponsibility> MR; std::unique_ptr<MemoryBuffer> ObjBuffer; DenseMap<SymbolStringPtr, SymbolNameSet> ExternalNamedSymbolDeps; DenseMap<SymbolStringPtr, SymbolNameSet> InternalNamedSymbolDeps; @@ -447,20 +459,39 @@ private: ObjectLinkingLayer::Plugin::~Plugin() {} +ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, + JITLinkMemoryManager &MemMgr) + : ObjectLayer(ES), MemMgr(MemMgr) { + ES.registerResourceManager(*this); +} + ObjectLinkingLayer::ObjectLinkingLayer( ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr) - : ObjectLayer(ES), MemMgr(std::move(MemMgr)) {} + : ObjectLayer(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) { + ES.registerResourceManager(*this); +} ObjectLinkingLayer::~ObjectLinkingLayer() { - if (auto Err = removeAllModules()) - getExecutionSession().reportError(std::move(Err)); + assert(Allocs.empty() && "Layer destroyed with resources still attached"); + getExecutionSession().deregisterResourceManager(*this); } -void ObjectLinkingLayer::emit(MaterializationResponsibility R, +void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, std::unique_ptr<MemoryBuffer> O) { assert(O && "Object must not be null"); - jitLink(std::make_unique<ObjectLinkingLayerJITLinkContext>( - *this, std::move(R), std::move(O))); + auto ObjBuffer = O->getMemBufferRef(); + auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( + *this, std::move(R), std::move(O)); + if (auto G = createLinkGraphFromObject(std::move(ObjBuffer))) + link(std::move(*G), std::move(Ctx)); + else + Ctx->notifyFailed(G.takeError()); +} + +void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<LinkGraph> G) { + link(std::move(G), std::make_unique<ObjectLinkingLayerJITLinkContext>( + *this, std::move(R), nullptr)); } void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, @@ -484,63 +515,56 @@ Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, if (Err) return Err; - { - std::lock_guard<std::mutex> Lock(LayerMutex); - UntrackedAllocs.push_back(std::move(Alloc)); - } - - return Error::success(); + return MR.withResourceKeyDo( + [&](ResourceKey K) { Allocs[K].push_back(std::move(Alloc)); }); } -Error ObjectLinkingLayer::removeModule(VModuleKey K) { +Error ObjectLinkingLayer::handleRemoveResources(ResourceKey K) { + Error Err = Error::success(); for (auto &P : Plugins) - Err = joinErrors(std::move(Err), P->notifyRemovingModule(K)); - - AllocPtr Alloc; + Err = joinErrors(std::move(Err), P->notifyRemovingResources(K)); + + std::vector<AllocPtr> AllocsToRemove; + getExecutionSession().runSessionLocked([&] { + auto I = Allocs.find(K); + if (I != Allocs.end()) { + std::swap(AllocsToRemove, I->second); + Allocs.erase(I); + } + }); - { - std::lock_guard<std::mutex> Lock(LayerMutex); - auto AllocItr = TrackedAllocs.find(K); - Alloc = std::move(AllocItr->second); - TrackedAllocs.erase(AllocItr); + while (!AllocsToRemove.empty()) { + Err = joinErrors(std::move(Err), AllocsToRemove.back()->deallocate()); + AllocsToRemove.pop_back(); } - assert(Alloc && "No allocation for key K"); - - return joinErrors(std::move(Err), Alloc->deallocate()); + return Err; } -Error ObjectLinkingLayer::removeAllModules() { - - Error Err = Error::success(); - - for (auto &P : Plugins) - Err = joinErrors(std::move(Err), P->notifyRemovingAllModules()); - - std::vector<AllocPtr> Allocs; - { - std::lock_guard<std::mutex> Lock(LayerMutex); - Allocs = std::move(UntrackedAllocs); - - for (auto &KV : TrackedAllocs) - Allocs.push_back(std::move(KV.second)); - - TrackedAllocs.clear(); +void ObjectLinkingLayer::handleTransferResources(ResourceKey DstKey, + ResourceKey SrcKey) { + auto I = Allocs.find(SrcKey); + if (I != Allocs.end()) { + auto &SrcAllocs = I->second; + auto &DstAllocs = Allocs[DstKey]; + DstAllocs.reserve(DstAllocs.size() + SrcAllocs.size()); + for (auto &Alloc : SrcAllocs) + DstAllocs.push_back(std::move(Alloc)); + + // Erase SrcKey entry using value rather than iterator I: I may have been + // invalidated when we looked up DstKey. + Allocs.erase(SrcKey); } - while (!Allocs.empty()) { - Err = joinErrors(std::move(Err), Allocs.back()->deallocate()); - Allocs.pop_back(); - } - - return Err; + for (auto &P : Plugins) + P->notifyTransferringResources(DstKey, SrcKey); } EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( - EHFrameRegistrar &Registrar) - : Registrar(Registrar) {} + ExecutionSession &ES, std::unique_ptr<EHFrameRegistrar> Registrar) + : ES(ES), Registrar(std::move(Registrar)) {} void EHFrameRegistrationPlugin::modifyPassConfig( MaterializationResponsibility &MR, const Triple &TT, @@ -559,65 +583,70 @@ void EHFrameRegistrationPlugin::modifyPassConfig( Error EHFrameRegistrationPlugin::notifyEmitted( MaterializationResponsibility &MR) { - std::lock_guard<std::mutex> Lock(EHFramePluginMutex); - - auto EHFrameRangeItr = InProcessLinks.find(&MR); - if (EHFrameRangeItr == InProcessLinks.end()) - return Error::success(); - - auto EHFrameRange = EHFrameRangeItr->second; - assert(EHFrameRange.Addr && - "eh-frame addr to register can not be null"); - - InProcessLinks.erase(EHFrameRangeItr); - if (auto Key = MR.getVModuleKey()) - TrackedEHFrameRanges[Key] = EHFrameRange; - else - UntrackedEHFrameRanges.push_back(EHFrameRange); - return Registrar.registerEHFrames(EHFrameRange.Addr, EHFrameRange.Size); -} + EHFrameRange EmittedRange; + { + std::lock_guard<std::mutex> Lock(EHFramePluginMutex); -Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) { - std::lock_guard<std::mutex> Lock(EHFramePluginMutex); + auto EHFrameRangeItr = InProcessLinks.find(&MR); + if (EHFrameRangeItr == InProcessLinks.end()) + return Error::success(); - auto EHFrameRangeItr = TrackedEHFrameRanges.find(K); - if (EHFrameRangeItr == TrackedEHFrameRanges.end()) - return Error::success(); - - auto EHFrameRange = EHFrameRangeItr->second; - assert(EHFrameRange.Addr && "Tracked eh-frame range must not be null"); + EmittedRange = EHFrameRangeItr->second; + assert(EmittedRange.Addr && "eh-frame addr to register can not be null"); + InProcessLinks.erase(EHFrameRangeItr); + } - TrackedEHFrameRanges.erase(EHFrameRangeItr); + if (auto Err = MR.withResourceKeyDo( + [&](ResourceKey K) { EHFrameRanges[K].push_back(EmittedRange); })) + return Err; - return Registrar.deregisterEHFrames(EHFrameRange.Addr, EHFrameRange.Size); + return Registrar->registerEHFrames(EmittedRange.Addr, EmittedRange.Size); } -Error EHFrameRegistrationPlugin::notifyRemovingAllModules() { +Error EHFrameRegistrationPlugin::notifyFailed( + MaterializationResponsibility &MR) { std::lock_guard<std::mutex> Lock(EHFramePluginMutex); + InProcessLinks.erase(&MR); + return Error::success(); +} - std::vector<EHFrameRange> EHFrameRanges = - std::move(UntrackedEHFrameRanges); - EHFrameRanges.reserve(EHFrameRanges.size() + TrackedEHFrameRanges.size()); - - for (auto &KV : TrackedEHFrameRanges) - EHFrameRanges.push_back(KV.second); +Error EHFrameRegistrationPlugin::notifyRemovingResources(ResourceKey K) { + std::vector<EHFrameRange> RangesToRemove; - TrackedEHFrameRanges.clear(); + ES.runSessionLocked([&] { + auto I = EHFrameRanges.find(K); + if (I != EHFrameRanges.end()) { + RangesToRemove = std::move(I->second); + EHFrameRanges.erase(I); + } + }); Error Err = Error::success(); - - while (!EHFrameRanges.empty()) { - auto EHFrameRange = EHFrameRanges.back(); - assert(EHFrameRange.Addr && "Untracked eh-frame range must not be null"); - EHFrameRanges.pop_back(); - Err = joinErrors(std::move(Err), - Registrar.deregisterEHFrames(EHFrameRange.Addr, - EHFrameRange.Size)); + while (!RangesToRemove.empty()) { + auto RangeToRemove = RangesToRemove.back(); + RangesToRemove.pop_back(); + assert(RangeToRemove.Addr && "Untracked eh-frame range must not be null"); + Err = joinErrors( + std::move(Err), + Registrar->deregisterEHFrames(RangeToRemove.Addr, RangeToRemove.Size)); } return Err; } +void EHFrameRegistrationPlugin::notifyTransferringResources( + ResourceKey DstKey, ResourceKey SrcKey) { + auto SI = EHFrameRanges.find(SrcKey); + if (SI != EHFrameRanges.end()) { + auto &SrcRanges = SI->second; + auto &DstRanges = EHFrameRanges[DstKey]; + DstRanges.reserve(DstRanges.size() + SrcRanges.size()); + for (auto &SrcRange : SrcRanges) + DstRanges.push_back(std::move(SrcRange)); + EHFrameRanges.erase(SI); + } +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp index d18eb38a4142..a57662e10a79 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -17,8 +17,9 @@ ObjectTransformLayer::ObjectTransformLayer(ExecutionSession &ES, TransformFunction Transform) : ObjectLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} -void ObjectTransformLayer::emit(MaterializationResponsibility R, - std::unique_ptr<MemoryBuffer> O) { +void ObjectTransformLayer::emit( + std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<MemoryBuffer> O) { assert(O && "Module must not be null"); // If there is a transform set then apply it. @@ -26,7 +27,7 @@ void ObjectTransformLayer::emit(MaterializationResponsibility R, if (auto TransformedObj = Transform(std::move(O))) O = std::move(*TransformedObj); else { - R.failMaterialization(); + R->failMaterialization(); getExecutionSession().reportError(TransformedObj.takeError()); return; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp deleted file mode 100644 index 28c8479abba4..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ /dev/null @@ -1,158 +0,0 @@ -//===----------- OrcCBindings.cpp - C bindings for the Orc APIs -----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "OrcCBindingsStack.h" -#include "llvm-c/OrcBindings.h" -#include "llvm/ExecutionEngine/JITEventListener.h" - -using namespace llvm; - -LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM) { - TargetMachine *TM2(unwrap(TM)); - - Triple T(TM2->getTargetTriple()); - - auto IndirectStubsMgrBuilder = - orc::createLocalIndirectStubsManagerBuilder(T); - - OrcCBindingsStack *JITStack = - new OrcCBindingsStack(*TM2, std::move(IndirectStubsMgrBuilder)); - - return wrap(JITStack); -} - -const char *LLVMOrcGetErrorMsg(LLVMOrcJITStackRef JITStack) { - OrcCBindingsStack &J = *unwrap(JITStack); - return J.getErrorMessage().c_str(); -} - -void LLVMOrcGetMangledSymbol(LLVMOrcJITStackRef JITStack, char **MangledName, - const char *SymbolName) { - OrcCBindingsStack &J = *unwrap(JITStack); - std::string Mangled = J.mangle(SymbolName); - *MangledName = new char[Mangled.size() + 1]; - strcpy(*MangledName, Mangled.c_str()); -} - -void LLVMOrcDisposeMangledSymbol(char *MangledName) { delete[] MangledName; } - -LLVMErrorRef LLVMOrcCreateLazyCompileCallback( - LLVMOrcJITStackRef JITStack, LLVMOrcTargetAddress *RetAddr, - LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { - OrcCBindingsStack &J = *unwrap(JITStack); - if (auto Addr = J.createLazyCompileCallback(Callback, CallbackCtx)) { - *RetAddr = *Addr; - return LLVMErrorSuccess; - } else - return wrap(Addr.takeError()); -} - -LLVMErrorRef LLVMOrcCreateIndirectStub(LLVMOrcJITStackRef JITStack, - const char *StubName, - LLVMOrcTargetAddress InitAddr) { - OrcCBindingsStack &J = *unwrap(JITStack); - return wrap(J.createIndirectStub(StubName, InitAddr)); -} - -LLVMErrorRef LLVMOrcSetIndirectStubPointer(LLVMOrcJITStackRef JITStack, - const char *StubName, - LLVMOrcTargetAddress NewAddr) { - OrcCBindingsStack &J = *unwrap(JITStack); - return wrap(J.setIndirectStubPointer(StubName, NewAddr)); -} - -LLVMErrorRef LLVMOrcAddEagerlyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, - LLVMModuleRef Mod, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx) { - OrcCBindingsStack &J = *unwrap(JITStack); - std::unique_ptr<Module> M(unwrap(Mod)); - if (auto Handle = - J.addIRModuleEager(std::move(M), SymbolResolver, SymbolResolverCtx)) { - *RetHandle = *Handle; - return LLVMErrorSuccess; - } else - return wrap(Handle.takeError()); -} - -LLVMErrorRef LLVMOrcAddLazilyCompiledIR(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, - LLVMModuleRef Mod, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx) { - OrcCBindingsStack &J = *unwrap(JITStack); - std::unique_ptr<Module> M(unwrap(Mod)); - if (auto Handle = - J.addIRModuleLazy(std::move(M), SymbolResolver, SymbolResolverCtx)) { - *RetHandle = *Handle; - return LLVMErrorSuccess; - } else - return wrap(Handle.takeError()); -} - -LLVMErrorRef LLVMOrcAddObjectFile(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle *RetHandle, - LLVMMemoryBufferRef Obj, - LLVMOrcSymbolResolverFn SymbolResolver, - void *SymbolResolverCtx) { - OrcCBindingsStack &J = *unwrap(JITStack); - std::unique_ptr<MemoryBuffer> O(unwrap(Obj)); - if (auto Handle = - J.addObject(std::move(O), SymbolResolver, SymbolResolverCtx)) { - *RetHandle = *Handle; - return LLVMErrorSuccess; - } else - return wrap(Handle.takeError()); -} - -LLVMErrorRef LLVMOrcRemoveModule(LLVMOrcJITStackRef JITStack, - LLVMOrcModuleHandle H) { - OrcCBindingsStack &J = *unwrap(JITStack); - return wrap(J.removeModule(H)); -} - -LLVMErrorRef LLVMOrcGetSymbolAddress(LLVMOrcJITStackRef JITStack, - LLVMOrcTargetAddress *RetAddr, - const char *SymbolName) { - OrcCBindingsStack &J = *unwrap(JITStack); - if (auto Addr = J.findSymbolAddress(SymbolName, true)) { - *RetAddr = *Addr; - return LLVMErrorSuccess; - } else - return wrap(Addr.takeError()); -} - -LLVMErrorRef LLVMOrcGetSymbolAddressIn(LLVMOrcJITStackRef JITStack, - LLVMOrcTargetAddress *RetAddr, - LLVMOrcModuleHandle H, - const char *SymbolName) { - OrcCBindingsStack &J = *unwrap(JITStack); - if (auto Addr = J.findSymbolAddressIn(H, SymbolName, true)) { - *RetAddr = *Addr; - return LLVMErrorSuccess; - } else - return wrap(Addr.takeError()); -} - -LLVMErrorRef LLVMOrcDisposeInstance(LLVMOrcJITStackRef JITStack) { - auto *J = unwrap(JITStack); - auto Err = J->shutdown(); - delete J; - return wrap(std::move(Err)); -} - -void LLVMOrcRegisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L) -{ - unwrap(JITStack)->RegisterJITEventListener(unwrap(L)); -} - -void LLVMOrcUnregisterJITEventListener(LLVMOrcJITStackRef JITStack, LLVMJITEventListenerRef L) -{ - unwrap(JITStack)->UnregisterJITEventListener(unwrap(L)); -} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h deleted file mode 100644 index 87bb4398765d..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ /dev/null @@ -1,534 +0,0 @@ -//===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H -#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H - -#include "llvm-c/OrcBindings.h" -#include "llvm-c/TargetMachine.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/ExecutionEngine/JITSymbol.h" -#include "llvm/ExecutionEngine/JITEventListener.h" -#include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" -#include "llvm/ExecutionEngine/Orc/CompileUtils.h" -#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" -#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" -#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" -#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" -#include "llvm/ExecutionEngine/RuntimeDyld.h" -#include "llvm/ExecutionEngine/SectionMemoryManager.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/Mangler.h" -#include "llvm/IR/Module.h" -#include "llvm/Support/CBindingWrapping.h" -#include "llvm/Support/Error.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetMachine.h" -#include <algorithm> -#include <cstdint> -#include <functional> -#include <map> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace llvm { - -class OrcCBindingsStack; - -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcCBindingsStack, LLVMOrcJITStackRef) -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) - -namespace detail { - -// FIXME: Kill this off once the Layer concept becomes an interface. -class GenericLayer { -public: - virtual ~GenericLayer() = default; - - virtual JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, - bool ExportedSymbolsOnly) = 0; - virtual Error removeModule(orc::VModuleKey K) = 0; - }; - - template <typename LayerT> class GenericLayerImpl : public GenericLayer { - public: - GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} - - JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, - bool ExportedSymbolsOnly) override { - return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); - } - - Error removeModule(orc::VModuleKey K) override { - return Layer.removeModule(K); - } - - private: - LayerT &Layer; - }; - - template <> - class GenericLayerImpl<orc::LegacyRTDyldObjectLinkingLayer> : public GenericLayer { - private: - using LayerT = orc::LegacyRTDyldObjectLinkingLayer; - public: - GenericLayerImpl(LayerT &Layer) : Layer(Layer) {} - - JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, - bool ExportedSymbolsOnly) override { - return Layer.findSymbolIn(K, Name, ExportedSymbolsOnly); - } - - Error removeModule(orc::VModuleKey K) override { - return Layer.removeObject(K); - } - - private: - LayerT &Layer; - }; - - template <typename LayerT> - std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) { - return std::make_unique<GenericLayerImpl<LayerT>>(Layer); - } - -} // end namespace detail - -class OrcCBindingsStack { -public: - - using CompileCallbackMgr = orc::JITCompileCallbackManager; - using ObjLayerT = orc::LegacyRTDyldObjectLinkingLayer; - using CompileLayerT = orc::LegacyIRCompileLayer<ObjLayerT, orc::SimpleCompiler>; - using CODLayerT = - orc::LegacyCompileOnDemandLayer<CompileLayerT, CompileCallbackMgr>; - - using CallbackManagerBuilder = - std::function<std::unique_ptr<CompileCallbackMgr>()>; - - using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT; - -private: - - using OwningObject = object::OwningBinary<object::ObjectFile>; - - class CBindingsResolver : public orc::SymbolResolver { - public: - CBindingsResolver(OrcCBindingsStack &Stack, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) - : Stack(Stack), ExternalResolver(std::move(ExternalResolver)), - ExternalResolverCtx(std::move(ExternalResolverCtx)) {} - - orc::SymbolNameSet - getResponsibilitySet(const orc::SymbolNameSet &Symbols) override { - orc::SymbolNameSet Result; - - for (auto &S : Symbols) { - if (auto Sym = findSymbol(std::string(*S))) { - if (!Sym.getFlags().isStrong()) - Result.insert(S); - } else if (auto Err = Sym.takeError()) { - Stack.reportError(std::move(Err)); - return orc::SymbolNameSet(); - } - } - - return Result; - } - - orc::SymbolNameSet - lookup(std::shared_ptr<orc::AsynchronousSymbolQuery> Query, - orc::SymbolNameSet Symbols) override { - orc::SymbolNameSet UnresolvedSymbols; - - for (auto &S : Symbols) { - if (auto Sym = findSymbol(std::string(*S))) { - if (auto Addr = Sym.getAddress()) { - Query->notifySymbolMetRequiredState( - S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); - } else { - Stack.ES.legacyFailQuery(*Query, Addr.takeError()); - return orc::SymbolNameSet(); - } - } else if (auto Err = Sym.takeError()) { - Stack.ES.legacyFailQuery(*Query, std::move(Err)); - return orc::SymbolNameSet(); - } else - UnresolvedSymbols.insert(S); - } - - if (Query->isComplete()) - Query->handleComplete(); - - return UnresolvedSymbols; - } - - private: - JITSymbol findSymbol(const std::string &Name) { - // Search order: - // 1. JIT'd symbols. - // 2. Runtime overrides. - // 3. External resolver (if present). - - if (Stack.CODLayer) { - if (auto Sym = Stack.CODLayer->findSymbol(Name, true)) - return Sym; - else if (auto Err = Sym.takeError()) - return Sym.takeError(); - } else { - if (auto Sym = Stack.CompileLayer.findSymbol(Name, true)) - return Sym; - else if (auto Err = Sym.takeError()) - return Sym.takeError(); - } - - if (auto Sym = Stack.CXXRuntimeOverrides.searchOverrides(Name)) - return Sym; - - if (ExternalResolver) - return JITSymbol(ExternalResolver(Name.c_str(), ExternalResolverCtx), - JITSymbolFlags::Exported); - - return JITSymbol(nullptr); - } - - OrcCBindingsStack &Stack; - LLVMOrcSymbolResolverFn ExternalResolver; - void *ExternalResolverCtx = nullptr; - }; - -public: - OrcCBindingsStack(TargetMachine &TM, - IndirectStubsManagerBuilder IndirectStubsMgrBuilder) - : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()), - IndirectStubsMgr(IndirectStubsMgrBuilder()), - ObjectLayer( - AcknowledgeORCv1Deprecation, ES, - [this](orc::VModuleKey K) { - auto ResolverI = Resolvers.find(K); - assert(ResolverI != Resolvers.end() && - "No resolver for module K"); - auto Resolver = std::move(ResolverI->second); - Resolvers.erase(ResolverI); - return ObjLayerT::Resources{ - std::make_shared<SectionMemoryManager>(), Resolver}; - }, - nullptr, - [this](orc::VModuleKey K, const object::ObjectFile &Obj, - const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { - this->notifyFinalized(K, Obj, LoadedObjInfo); - }, - [this](orc::VModuleKey K, const object::ObjectFile &Obj) { - this->notifyFreed(K, Obj); - }), - CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, - orc::SimpleCompiler(TM)), - CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(), - std::move(IndirectStubsMgrBuilder), Resolvers)), - CXXRuntimeOverrides( - AcknowledgeORCv1Deprecation, - [this](const std::string &S) { return mangle(S); }) {} - - Error shutdown() { - // Run any destructors registered with __cxa_atexit. - CXXRuntimeOverrides.runDestructors(); - // Run any IR destructors. - for (auto &DtorRunner : IRStaticDestructorRunners) - if (auto Err = DtorRunner.runViaLayer(*this)) - return Err; - return Error::success(); - } - - std::string mangle(StringRef Name) { - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, DL); - } - return MangledName; - } - - template <typename PtrTy> - static PtrTy fromTargetAddress(JITTargetAddress Addr) { - return reinterpret_cast<PtrTy>(static_cast<uintptr_t>(Addr)); - } - - Expected<JITTargetAddress> - createLazyCompileCallback(LLVMOrcLazyCompileCallbackFn Callback, - void *CallbackCtx) { - auto WrappedCallback = [=]() -> JITTargetAddress { - return Callback(wrap(this), CallbackCtx); - }; - - return CCMgr->getCompileCallback(std::move(WrappedCallback)); - } - - Error createIndirectStub(StringRef StubName, JITTargetAddress Addr) { - return IndirectStubsMgr->createStub(StubName, Addr, - JITSymbolFlags::Exported); - } - - Error setIndirectStubPointer(StringRef Name, JITTargetAddress Addr) { - return IndirectStubsMgr->updatePointer(Name, Addr); - } - - template <typename LayerT> - Expected<orc::VModuleKey> - addIRModule(LayerT &Layer, std::unique_ptr<Module> M, - std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - - // Attach a data-layout if one isn't already present. - if (M->getDataLayout().isDefault()) - M->setDataLayout(DL); - - // Record the static constructors and destructors. We have to do this before - // we hand over ownership of the module to the JIT. - std::vector<std::string> CtorNames, DtorNames; - for (auto Ctor : orc::getConstructors(*M)) - CtorNames.push_back(mangle(Ctor.Func->getName())); - for (auto Dtor : orc::getDestructors(*M)) - DtorNames.push_back(mangle(Dtor.Func->getName())); - - // Add the module to the JIT. - auto K = ES.allocateVModule(); - Resolvers[K] = std::make_shared<CBindingsResolver>(*this, ExternalResolver, - ExternalResolverCtx); - if (auto Err = Layer.addModule(K, std::move(M))) - return std::move(Err); - - KeyLayers[K] = detail::createGenericLayer(Layer); - - // Run the static constructors, and save the static destructor runner for - // execution when the JIT is torn down. - orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner( - AcknowledgeORCv1Deprecation, std::move(CtorNames), K); - if (auto Err = CtorRunner.runViaLayer(*this)) - return std::move(Err); - - IRStaticDestructorRunners.emplace_back(AcknowledgeORCv1Deprecation, - std::move(DtorNames), K); - - return K; - } - - Expected<orc::VModuleKey> - addIRModuleEager(std::unique_ptr<Module> M, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - return addIRModule(CompileLayer, std::move(M), - std::make_unique<SectionMemoryManager>(), - std::move(ExternalResolver), ExternalResolverCtx); - } - - Expected<orc::VModuleKey> - addIRModuleLazy(std::unique_ptr<Module> M, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - if (!CODLayer) - return make_error<StringError>("Can not add lazy module: No compile " - "callback manager available", - inconvertibleErrorCode()); - - return addIRModule(*CODLayer, std::move(M), - std::make_unique<SectionMemoryManager>(), - std::move(ExternalResolver), ExternalResolverCtx); - } - - Error removeModule(orc::VModuleKey K) { - // FIXME: Should error release the module key? - if (auto Err = KeyLayers[K]->removeModule(K)) - return Err; - ES.releaseVModule(K); - KeyLayers.erase(K); - return Error::success(); - } - - Expected<orc::VModuleKey> addObject(std::unique_ptr<MemoryBuffer> ObjBuffer, - LLVMOrcSymbolResolverFn ExternalResolver, - void *ExternalResolverCtx) { - if (auto Obj = object::ObjectFile::createObjectFile( - ObjBuffer->getMemBufferRef())) { - - auto K = ES.allocateVModule(); - Resolvers[K] = std::make_shared<CBindingsResolver>( - *this, ExternalResolver, ExternalResolverCtx); - - if (auto Err = ObjectLayer.addObject(K, std::move(ObjBuffer))) - return std::move(Err); - - KeyLayers[K] = detail::createGenericLayer(ObjectLayer); - - return K; - } else - return Obj.takeError(); - } - - JITSymbol findSymbol(const std::string &Name, - bool ExportedSymbolsOnly) { - if (auto Sym = IndirectStubsMgr->findStub(Name, ExportedSymbolsOnly)) - return Sym; - if (CODLayer) - return CODLayer->findSymbol(mangle(Name), ExportedSymbolsOnly); - return CompileLayer.findSymbol(mangle(Name), ExportedSymbolsOnly); - } - - JITSymbol findSymbolIn(orc::VModuleKey K, const std::string &Name, - bool ExportedSymbolsOnly) { - assert(KeyLayers.count(K) && "looking up symbol in unknown module"); - return KeyLayers[K]->findSymbolIn(K, mangle(Name), ExportedSymbolsOnly); - } - - Expected<JITTargetAddress> findSymbolAddress(const std::string &Name, - bool ExportedSymbolsOnly) { - if (auto Sym = findSymbol(Name, ExportedSymbolsOnly)) { - // Successful lookup, non-null symbol: - if (auto AddrOrErr = Sym.getAddress()) - return *AddrOrErr; - else - return AddrOrErr.takeError(); - } else if (auto Err = Sym.takeError()) { - // Lookup failure - report error. - return std::move(Err); - } - - // No symbol not found. Return 0. - return 0; - } - - Expected<JITTargetAddress> findSymbolAddressIn(orc::VModuleKey K, - const std::string &Name, - bool ExportedSymbolsOnly) { - if (auto Sym = findSymbolIn(K, Name, ExportedSymbolsOnly)) { - // Successful lookup, non-null symbol: - if (auto AddrOrErr = Sym.getAddress()) - return *AddrOrErr; - else - return AddrOrErr.takeError(); - } else if (auto Err = Sym.takeError()) { - // Lookup failure - report error. - return std::move(Err); - } - - // Symbol not found. Return 0. - return 0; - } - - const std::string &getErrorMessage() const { return ErrMsg; } - - void RegisterJITEventListener(JITEventListener *L) { - if (!L) - return; - EventListeners.push_back(L); - } - - void UnregisterJITEventListener(JITEventListener *L) { - if (!L) - return; - - auto I = find(reverse(EventListeners), L); - if (I != EventListeners.rend()) { - std::swap(*I, EventListeners.back()); - EventListeners.pop_back(); - } - } - -private: - using ResolverMap = - std::map<orc::VModuleKey, std::shared_ptr<orc::SymbolResolver>>; - - static std::unique_ptr<CompileCallbackMgr> - createCompileCallbackManager(TargetMachine &TM, orc::ExecutionSession &ES) { - auto CCMgr = createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0); - if (!CCMgr) { - // FIXME: It would be good if we could report this somewhere, but we do - // have an instance yet. - logAllUnhandledErrors(CCMgr.takeError(), errs(), "ORC error: "); - return nullptr; - } - return std::move(*CCMgr); - } - - static std::unique_ptr<CODLayerT> - createCODLayer(orc::ExecutionSession &ES, CompileLayerT &CompileLayer, - CompileCallbackMgr *CCMgr, - IndirectStubsManagerBuilder IndirectStubsMgrBuilder, - ResolverMap &Resolvers) { - // If there is no compile callback manager available we can not create a - // compile on demand layer. - if (!CCMgr) - return nullptr; - - return std::make_unique<CODLayerT>( - AcknowledgeORCv1Deprecation, ES, CompileLayer, - [&Resolvers](orc::VModuleKey K) { - auto ResolverI = Resolvers.find(K); - assert(ResolverI != Resolvers.end() && "No resolver for module K"); - return ResolverI->second; - }, - [&Resolvers](orc::VModuleKey K, - std::shared_ptr<orc::SymbolResolver> Resolver) { - assert(!Resolvers.count(K) && "Resolver already present"); - Resolvers[K] = std::move(Resolver); - }, - [](Function &F) { return std::set<Function *>({&F}); }, *CCMgr, - std::move(IndirectStubsMgrBuilder), false); - } - - void reportError(Error Err) { - // FIXME: Report errors on the execution session. - logAllUnhandledErrors(std::move(Err), errs(), "ORC error: "); - }; - - void notifyFinalized(orc::VModuleKey K, - const object::ObjectFile &Obj, - const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { - uint64_t Key = static_cast<uint64_t>( - reinterpret_cast<uintptr_t>(Obj.getData().data())); - for (auto &Listener : EventListeners) - Listener->notifyObjectLoaded(Key, Obj, LoadedObjInfo); - } - - void notifyFreed(orc::VModuleKey K, const object::ObjectFile &Obj) { - uint64_t Key = static_cast<uint64_t>( - reinterpret_cast<uintptr_t>(Obj.getData().data())); - for (auto &Listener : EventListeners) - Listener->notifyFreeingObject(Key); - } - - orc::ExecutionSession ES; - std::unique_ptr<CompileCallbackMgr> CCMgr; - - std::vector<JITEventListener *> EventListeners; - - DataLayout DL; - SectionMemoryManager CCMgrMemMgr; - - std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr; - - ObjLayerT ObjectLayer; - CompileLayerT CompileLayer; - std::unique_ptr<CODLayerT> CODLayer; - - std::map<orc::VModuleKey, std::unique_ptr<detail::GenericLayer>> KeyLayers; - - orc::LegacyLocalCXXRuntimeOverrides CXXRuntimeOverrides; - std::vector<orc::LegacyCtorDtorRunner<OrcCBindingsStack>> IRStaticDestructorRunners; - std::string ErrMsg; - - ResolverMap Resolvers; -}; - -} // end namespace llvm - -#endif // LLVM_LIB_EXECUTIONENGINE_ORC_ORCCBINDINGSSTACK_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp deleted file mode 100644 index 772a9c2c4ab2..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp +++ /dev/null @@ -1,138 +0,0 @@ -//===-------- OrcMCJITReplacement.cpp - Orc-based MCJIT replacement -------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "OrcMCJITReplacement.h" -#include "llvm/ExecutionEngine/GenericValue.h" - -namespace { - -static struct RegisterJIT { - RegisterJIT() { llvm::orc::OrcMCJITReplacement::Register(); } -} JITRegistrator; - -} - -extern "C" void LLVMLinkInOrcMCJITReplacement() {} - -namespace llvm { -namespace orc { - -GenericValue -OrcMCJITReplacement::runFunction(Function *F, - ArrayRef<GenericValue> ArgValues) { - assert(F && "Function *F was null at entry to run()"); - - void *FPtr = getPointerToFunction(F); - assert(FPtr && "Pointer to fn's code was null after getPointerToFunction"); - FunctionType *FTy = F->getFunctionType(); - Type *RetTy = FTy->getReturnType(); - - assert((FTy->getNumParams() == ArgValues.size() || - (FTy->isVarArg() && FTy->getNumParams() <= ArgValues.size())) && - "Wrong number of arguments passed into function!"); - assert(FTy->getNumParams() == ArgValues.size() && - "This doesn't support passing arguments through varargs (yet)!"); - - // Handle some common cases first. These cases correspond to common `main' - // prototypes. - if (RetTy->isIntegerTy(32) || RetTy->isVoidTy()) { - switch (ArgValues.size()) { - case 3: - if (FTy->getParamType(0)->isIntegerTy(32) && - FTy->getParamType(1)->isPointerTy() && - FTy->getParamType(2)->isPointerTy()) { - int (*PF)(int, char **, const char **) = - (int (*)(int, char **, const char **))(intptr_t)FPtr; - - // Call the function. - GenericValue rv; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), - (char **)GVTOP(ArgValues[1]), - (const char **)GVTOP(ArgValues[2]))); - return rv; - } - break; - case 2: - if (FTy->getParamType(0)->isIntegerTy(32) && - FTy->getParamType(1)->isPointerTy()) { - int (*PF)(int, char **) = (int (*)(int, char **))(intptr_t)FPtr; - - // Call the function. - GenericValue rv; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue(), - (char **)GVTOP(ArgValues[1]))); - return rv; - } - break; - case 1: - if (FTy->getNumParams() == 1 && FTy->getParamType(0)->isIntegerTy(32)) { - GenericValue rv; - int (*PF)(int) = (int (*)(int))(intptr_t)FPtr; - rv.IntVal = APInt(32, PF(ArgValues[0].IntVal.getZExtValue())); - return rv; - } - break; - } - } - - // Handle cases where no arguments are passed first. - if (ArgValues.empty()) { - GenericValue rv; - switch (RetTy->getTypeID()) { - default: - llvm_unreachable("Unknown return type for function call!"); - case Type::IntegerTyID: { - unsigned BitWidth = cast<IntegerType>(RetTy)->getBitWidth(); - if (BitWidth == 1) - rv.IntVal = APInt(BitWidth, ((bool (*)())(intptr_t)FPtr)()); - else if (BitWidth <= 8) - rv.IntVal = APInt(BitWidth, ((char (*)())(intptr_t)FPtr)()); - else if (BitWidth <= 16) - rv.IntVal = APInt(BitWidth, ((short (*)())(intptr_t)FPtr)()); - else if (BitWidth <= 32) - rv.IntVal = APInt(BitWidth, ((int (*)())(intptr_t)FPtr)()); - else if (BitWidth <= 64) - rv.IntVal = APInt(BitWidth, ((int64_t (*)())(intptr_t)FPtr)()); - else - llvm_unreachable("Integer types > 64 bits not supported"); - return rv; - } - case Type::VoidTyID: - rv.IntVal = APInt(32, ((int (*)())(intptr_t)FPtr)()); - return rv; - case Type::FloatTyID: - rv.FloatVal = ((float (*)())(intptr_t)FPtr)(); - return rv; - case Type::DoubleTyID: - rv.DoubleVal = ((double (*)())(intptr_t)FPtr)(); - return rv; - case Type::X86_FP80TyID: - case Type::FP128TyID: - case Type::PPC_FP128TyID: - llvm_unreachable("long double not supported yet"); - case Type::PointerTyID: - return PTOGV(((void *(*)())(intptr_t)FPtr)()); - } - } - - llvm_unreachable("Full-featured argument passing not supported yet!"); -} - -void OrcMCJITReplacement::runStaticConstructorsDestructors(bool isDtors) { - auto &CtorDtorsMap = isDtors ? UnexecutedDestructors : UnexecutedConstructors; - - for (auto &KV : CtorDtorsMap) - cantFail(LegacyCtorDtorRunner<LazyEmitLayerT>( - AcknowledgeORCv1Deprecation, std::move(KV.second), KV.first) - .runViaLayer(LazyEmitLayer)); - - CtorDtorsMap.clear(); -} - -} // End namespace orc. -} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h deleted file mode 100644 index 139572bd6977..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ /dev/null @@ -1,502 +0,0 @@ -//===- OrcMCJITReplacement.h - Orc based MCJIT replacement ------*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Orc based MCJIT replacement. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H -#define LLVM_LIB_EXECUTIONENGINE_ORC_ORCMCJITREPLACEMENT_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/ExecutionEngine/ExecutionEngine.h" -#include "llvm/ExecutionEngine/GenericValue.h" -#include "llvm/ExecutionEngine/JITSymbol.h" -#include "llvm/ExecutionEngine/Orc/CompileUtils.h" -#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" -#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" -#include "llvm/ExecutionEngine/Orc/LazyEmittingLayer.h" -#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" -#include "llvm/ExecutionEngine/RTDyldMemoryManager.h" -#include "llvm/ExecutionEngine/RuntimeDyld.h" -#include "llvm/IR/DataLayout.h" -#include "llvm/IR/Function.h" -#include "llvm/IR/Mangler.h" -#include "llvm/IR/Module.h" -#include "llvm/Object/Archive.h" -#include "llvm/Object/Binary.h" -#include "llvm/Object/ObjectFile.h" -#include "llvm/Support/Error.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetMachine.h" -#include <algorithm> -#include <cassert> -#include <cstddef> -#include <cstdint> -#include <map> -#include <memory> -#include <set> -#include <string> -#include <vector> - -namespace llvm { - -class ObjectCache; - -namespace orc { - -class OrcMCJITReplacement : public ExecutionEngine { - - // OrcMCJITReplacement needs to do a little extra book-keeping to ensure that - // Orc's automatic finalization doesn't kick in earlier than MCJIT clients are - // expecting - see finalizeMemory. - class MCJITReplacementMemMgr : public MCJITMemoryManager { - public: - MCJITReplacementMemMgr(OrcMCJITReplacement &M, - std::shared_ptr<MCJITMemoryManager> ClientMM) - : M(M), ClientMM(std::move(ClientMM)) {} - - uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, - StringRef SectionName) override { - uint8_t *Addr = - ClientMM->allocateCodeSection(Size, Alignment, SectionID, - SectionName); - M.SectionsAllocatedSinceLastLoad.insert(Addr); - return Addr; - } - - uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, - unsigned SectionID, StringRef SectionName, - bool IsReadOnly) override { - uint8_t *Addr = ClientMM->allocateDataSection(Size, Alignment, SectionID, - SectionName, IsReadOnly); - M.SectionsAllocatedSinceLastLoad.insert(Addr); - return Addr; - } - - void reserveAllocationSpace(uintptr_t CodeSize, uint32_t CodeAlign, - uintptr_t RODataSize, uint32_t RODataAlign, - uintptr_t RWDataSize, - uint32_t RWDataAlign) override { - return ClientMM->reserveAllocationSpace(CodeSize, CodeAlign, - RODataSize, RODataAlign, - RWDataSize, RWDataAlign); - } - - bool needsToReserveAllocationSpace() override { - return ClientMM->needsToReserveAllocationSpace(); - } - - void registerEHFrames(uint8_t *Addr, uint64_t LoadAddr, - size_t Size) override { - return ClientMM->registerEHFrames(Addr, LoadAddr, Size); - } - - void deregisterEHFrames() override { - return ClientMM->deregisterEHFrames(); - } - - void notifyObjectLoaded(RuntimeDyld &RTDyld, - const object::ObjectFile &O) override { - return ClientMM->notifyObjectLoaded(RTDyld, O); - } - - void notifyObjectLoaded(ExecutionEngine *EE, - const object::ObjectFile &O) override { - return ClientMM->notifyObjectLoaded(EE, O); - } - - bool finalizeMemory(std::string *ErrMsg = nullptr) override { - // Each set of objects loaded will be finalized exactly once, but since - // symbol lookup during relocation may recursively trigger the - // loading/relocation of other modules, and since we're forwarding all - // finalizeMemory calls to a single underlying memory manager, we need to - // defer forwarding the call on until all necessary objects have been - // loaded. Otherwise, during the relocation of a leaf object, we will end - // up finalizing memory, causing a crash further up the stack when we - // attempt to apply relocations to finalized memory. - // To avoid finalizing too early, look at how many objects have been - // loaded but not yet finalized. This is a bit of a hack that relies on - // the fact that we're lazily emitting object files: The only way you can - // get more than one set of objects loaded but not yet finalized is if - // they were loaded during relocation of another set. - if (M.UnfinalizedSections.size() == 1) - return ClientMM->finalizeMemory(ErrMsg); - return false; - } - - private: - OrcMCJITReplacement &M; - std::shared_ptr<MCJITMemoryManager> ClientMM; - }; - - class LinkingORCResolver : public orc::SymbolResolver { - public: - LinkingORCResolver(OrcMCJITReplacement &M) : M(M) {} - - SymbolNameSet getResponsibilitySet(const SymbolNameSet &Symbols) override { - SymbolNameSet Result; - - for (auto &S : Symbols) { - if (auto Sym = M.findMangledSymbol(*S)) { - if (!Sym.getFlags().isStrong()) - Result.insert(S); - } else if (auto Err = Sym.takeError()) { - M.reportError(std::move(Err)); - return SymbolNameSet(); - } else { - if (auto Sym2 = - M.ClientResolver->findSymbolInLogicalDylib(std::string(*S))) { - if (!Sym2.getFlags().isStrong()) - Result.insert(S); - } else if (auto Err = Sym2.takeError()) { - M.reportError(std::move(Err)); - return SymbolNameSet(); - } else - Result.insert(S); - } - } - - return Result; - } - - SymbolNameSet lookup(std::shared_ptr<AsynchronousSymbolQuery> Query, - SymbolNameSet Symbols) override { - SymbolNameSet UnresolvedSymbols; - bool NewSymbolsResolved = false; - - for (auto &S : Symbols) { - if (auto Sym = M.findMangledSymbol(*S)) { - if (auto Addr = Sym.getAddress()) { - Query->notifySymbolMetRequiredState( - S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); - NewSymbolsResolved = true; - } else { - M.ES.legacyFailQuery(*Query, Addr.takeError()); - return SymbolNameSet(); - } - } else if (auto Err = Sym.takeError()) { - M.ES.legacyFailQuery(*Query, std::move(Err)); - return SymbolNameSet(); - } else { - if (auto Sym2 = M.ClientResolver->findSymbol(std::string(*S))) { - if (auto Addr = Sym2.getAddress()) { - Query->notifySymbolMetRequiredState( - S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); - NewSymbolsResolved = true; - } else { - M.ES.legacyFailQuery(*Query, Addr.takeError()); - return SymbolNameSet(); - } - } else if (auto Err = Sym2.takeError()) { - M.ES.legacyFailQuery(*Query, std::move(Err)); - return SymbolNameSet(); - } else - UnresolvedSymbols.insert(S); - } - } - - if (NewSymbolsResolved && Query->isComplete()) - Query->handleComplete(); - - return UnresolvedSymbols; - } - - private: - OrcMCJITReplacement &M; - }; - -private: - static ExecutionEngine * - createOrcMCJITReplacement(std::string *ErrorMsg, - std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<LegacyJITSymbolResolver> Resolver, - std::unique_ptr<TargetMachine> TM) { - return new OrcMCJITReplacement(std::move(MemMgr), std::move(Resolver), - std::move(TM)); - } - - void reportError(Error Err) { - logAllUnhandledErrors(std::move(Err), errs(), "MCJIT error: "); - } - -public: - OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr, - std::shared_ptr<LegacyJITSymbolResolver> ClientResolver, - std::unique_ptr<TargetMachine> TM) - : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), - MemMgr( - std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))), - Resolver(std::make_shared<LinkingORCResolver>(*this)), - ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), - NotifyFinalized(*this), - ObjectLayer( - AcknowledgeORCv1Deprecation, ES, - [this](VModuleKey K) { - return ObjectLayerT::Resources{this->MemMgr, this->Resolver}; - }, - NotifyObjectLoaded, NotifyFinalized), - CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, - SimpleCompiler(*this->TM), - [this](VModuleKey K, std::unique_ptr<Module> M) { - Modules.push_back(std::move(M)); - }), - LazyEmitLayer(AcknowledgeORCv1Deprecation, CompileLayer) {} - - static void Register() { - OrcMCJITReplacementCtor = createOrcMCJITReplacement; - } - - void addModule(std::unique_ptr<Module> M) override { - // If this module doesn't have a DataLayout attached then attach the - // default. - if (M->getDataLayout().isDefault()) { - M->setDataLayout(getDataLayout()); - } else { - assert(M->getDataLayout() == getDataLayout() && "DataLayout Mismatch"); - } - - // Rename, bump linkage and record static constructors and destructors. - // We have to do this before we hand over ownership of the module to the - // JIT. - std::vector<std::string> CtorNames, DtorNames; - { - unsigned CtorId = 0, DtorId = 0; - for (auto Ctor : orc::getConstructors(*M)) { - std::string NewCtorName = ("__ORCstatic_ctor." + Twine(CtorId++)).str(); - Ctor.Func->setName(NewCtorName); - Ctor.Func->setLinkage(GlobalValue::ExternalLinkage); - Ctor.Func->setVisibility(GlobalValue::HiddenVisibility); - CtorNames.push_back(mangle(NewCtorName)); - } - for (auto Dtor : orc::getDestructors(*M)) { - std::string NewDtorName = ("__ORCstatic_dtor." + Twine(DtorId++)).str(); - dbgs() << "Found dtor: " << NewDtorName << "\n"; - Dtor.Func->setName(NewDtorName); - Dtor.Func->setLinkage(GlobalValue::ExternalLinkage); - Dtor.Func->setVisibility(GlobalValue::HiddenVisibility); - DtorNames.push_back(mangle(NewDtorName)); - } - } - - auto K = ES.allocateVModule(); - - UnexecutedConstructors[K] = std::move(CtorNames); - UnexecutedDestructors[K] = std::move(DtorNames); - - cantFail(LazyEmitLayer.addModule(K, std::move(M))); - } - - void addObjectFile(std::unique_ptr<object::ObjectFile> O) override { - cantFail(ObjectLayer.addObject( - ES.allocateVModule(), MemoryBuffer::getMemBufferCopy(O->getData()))); - } - - void addObjectFile(object::OwningBinary<object::ObjectFile> O) override { - std::unique_ptr<object::ObjectFile> Obj; - std::unique_ptr<MemoryBuffer> ObjBuffer; - std::tie(Obj, ObjBuffer) = O.takeBinary(); - cantFail(ObjectLayer.addObject(ES.allocateVModule(), std::move(ObjBuffer))); - } - - void addArchive(object::OwningBinary<object::Archive> A) override { - Archives.push_back(std::move(A)); - } - - bool removeModule(Module *M) override { - auto I = Modules.begin(); - for (auto E = Modules.end(); I != E; ++I) - if (I->get() == M) - break; - if (I == Modules.end()) - return false; - Modules.erase(I); - return true; - } - - uint64_t getSymbolAddress(StringRef Name) { - return cantFail(findSymbol(Name).getAddress()); - } - - JITSymbol findSymbol(StringRef Name) { - return findMangledSymbol(mangle(Name)); - } - - void finalizeObject() override { - // This is deprecated - Aim to remove in ExecutionEngine. - // REMOVE IF POSSIBLE - Doesn't make sense for New JIT. - } - - void mapSectionAddress(const void *LocalAddress, - uint64_t TargetAddress) override { - for (auto &P : UnfinalizedSections) - if (P.second.count(LocalAddress)) - ObjectLayer.mapSectionAddress(P.first, LocalAddress, TargetAddress); - } - - uint64_t getGlobalValueAddress(const std::string &Name) override { - return getSymbolAddress(Name); - } - - uint64_t getFunctionAddress(const std::string &Name) override { - return getSymbolAddress(Name); - } - - void *getPointerToFunction(Function *F) override { - uint64_t FAddr = getSymbolAddress(F->getName()); - return reinterpret_cast<void *>(static_cast<uintptr_t>(FAddr)); - } - - void *getPointerToNamedFunction(StringRef Name, - bool AbortOnFailure = true) override { - uint64_t Addr = getSymbolAddress(Name); - if (!Addr && AbortOnFailure) - llvm_unreachable("Missing symbol!"); - return reinterpret_cast<void *>(static_cast<uintptr_t>(Addr)); - } - - GenericValue runFunction(Function *F, - ArrayRef<GenericValue> ArgValues) override; - - void setObjectCache(ObjectCache *NewCache) override { - CompileLayer.getCompiler().setObjectCache(NewCache); - } - - void setProcessAllSections(bool ProcessAllSections) override { - ObjectLayer.setProcessAllSections(ProcessAllSections); - } - - void runStaticConstructorsDestructors(bool isDtors) override; - -private: - JITSymbol findMangledSymbol(StringRef Name) { - if (auto Sym = LazyEmitLayer.findSymbol(std::string(Name), false)) - return Sym; - if (auto Sym = ClientResolver->findSymbol(std::string(Name))) - return Sym; - if (auto Sym = scanArchives(Name)) - return Sym; - - return nullptr; - } - - JITSymbol scanArchives(StringRef Name) { - for (object::OwningBinary<object::Archive> &OB : Archives) { - object::Archive *A = OB.getBinary(); - // Look for our symbols in each Archive - auto OptionalChildOrErr = A->findSym(Name); - if (!OptionalChildOrErr) - report_fatal_error(OptionalChildOrErr.takeError()); - auto &OptionalChild = *OptionalChildOrErr; - if (OptionalChild) { - // FIXME: Support nested archives? - Expected<std::unique_ptr<object::Binary>> ChildBinOrErr = - OptionalChild->getAsBinary(); - if (!ChildBinOrErr) { - // TODO: Actually report errors helpfully. - consumeError(ChildBinOrErr.takeError()); - continue; - } - std::unique_ptr<object::Binary> &ChildBin = ChildBinOrErr.get(); - if (ChildBin->isObject()) { - cantFail(ObjectLayer.addObject( - ES.allocateVModule(), - MemoryBuffer::getMemBufferCopy(ChildBin->getData()))); - if (auto Sym = ObjectLayer.findSymbol(Name, true)) - return Sym; - } - } - } - return nullptr; - } - - class NotifyObjectLoadedT { - public: - using LoadedObjInfoListT = - std::vector<std::unique_ptr<RuntimeDyld::LoadedObjectInfo>>; - - NotifyObjectLoadedT(OrcMCJITReplacement &M) : M(M) {} - - void operator()(VModuleKey K, const object::ObjectFile &Obj, - const RuntimeDyld::LoadedObjectInfo &Info) const { - M.UnfinalizedSections[K] = std::move(M.SectionsAllocatedSinceLastLoad); - M.SectionsAllocatedSinceLastLoad = SectionAddrSet(); - M.MemMgr->notifyObjectLoaded(&M, Obj); - } - private: - OrcMCJITReplacement &M; - }; - - class NotifyFinalizedT { - public: - NotifyFinalizedT(OrcMCJITReplacement &M) : M(M) {} - - void operator()(VModuleKey K, const object::ObjectFile &Obj, - const RuntimeDyld::LoadedObjectInfo &Info) { - M.UnfinalizedSections.erase(K); - } - - private: - OrcMCJITReplacement &M; - }; - - std::string mangle(StringRef Name) { - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mang.getNameWithPrefix(MangledNameStream, Name, getDataLayout()); - } - return MangledName; - } - - using ObjectLayerT = LegacyRTDyldObjectLinkingLayer; - using CompileLayerT = LegacyIRCompileLayer<ObjectLayerT, orc::SimpleCompiler>; - using LazyEmitLayerT = LazyEmittingLayer<CompileLayerT>; - - ExecutionSession ES; - - std::unique_ptr<TargetMachine> TM; - std::shared_ptr<MCJITReplacementMemMgr> MemMgr; - std::shared_ptr<LinkingORCResolver> Resolver; - std::shared_ptr<LegacyJITSymbolResolver> ClientResolver; - Mangler Mang; - - // IMPORTANT: ShouldDelete *must* come before LocalModules: The shared_ptr - // delete blocks in LocalModules refer to the ShouldDelete map, so - // LocalModules needs to be destructed before ShouldDelete. - std::map<Module*, bool> ShouldDelete; - - NotifyObjectLoadedT NotifyObjectLoaded; - NotifyFinalizedT NotifyFinalized; - - ObjectLayerT ObjectLayer; - CompileLayerT CompileLayer; - LazyEmitLayerT LazyEmitLayer; - - std::map<VModuleKey, std::vector<std::string>> UnexecutedConstructors; - std::map<VModuleKey, std::vector<std::string>> UnexecutedDestructors; - - // We need to store ObjLayerT::ObjSetHandles for each of the object sets - // that have been emitted but not yet finalized so that we can forward the - // mapSectionAddress calls appropriately. - using SectionAddrSet = std::set<const void *>; - SectionAddrSet SectionsAllocatedSinceLastLoad; - std::map<VModuleKey, SectionAddrSet> UnfinalizedSections; - - std::vector<object::OwningBinary<object::Archive>> Archives; -}; - -} // end namespace orc - -} // end namespace llvm - -#endif // LLVM_LIB_EXECUTIONENGINE_ORC_MCJITREPLACEMENT_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp index 5933c2e666d1..dfdd2c6c669f 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -6,11 +6,15 @@ // //===----------------------------------------------------------------------===// +#include "llvm-c/LLJIT.h" #include "llvm-c/Orc.h" +#include "llvm-c/OrcEE.h" #include "llvm-c/TargetMachine.h" #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/ExecutionEngine/SectionMemoryManager.h" using namespace llvm; using namespace llvm::orc; @@ -18,6 +22,8 @@ using namespace llvm::orc; namespace llvm { namespace orc { +class InProgressLookupState; + class OrcV2CAPIHelper { public: using PoolEntry = SymbolStringPtr::PoolEntry; @@ -29,58 +35,278 @@ public: return Result; } + static SymbolStringPtr retainSymbolStringPtr(PoolEntryPtr P) { + return SymbolStringPtr(P); + } + static PoolEntryPtr getRawPoolEntryPtr(const SymbolStringPtr &S) { return S.S; } + static void retainPoolEntry(PoolEntryPtr P) { + SymbolStringPtr S(P); + S.S = nullptr; + } + static void releasePoolEntry(PoolEntryPtr P) { SymbolStringPtr S; S.S = P; } + + static InProgressLookupState *extractLookupState(LookupState &LS) { + return LS.IPLS.release(); + } + + static void resetLookupState(LookupState &LS, InProgressLookupState *IPLS) { + return LS.reset(IPLS); + } }; -} // end namespace orc -} // end namespace llvm +} // namespace orc +} // namespace llvm DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ExecutionSession, LLVMOrcExecutionSessionRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SymbolStringPool, LLVMOrcSymbolStringPoolRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcV2CAPIHelper::PoolEntry, LLVMOrcSymbolStringPoolEntryRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(MaterializationUnit, + LLVMOrcMaterializationUnitRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITDylib, LLVMOrcJITDylibRef) -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITDylib::DefinitionGenerator, - LLVMOrcJITDylibDefinitionGeneratorRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ResourceTracker, LLVMOrcResourceTrackerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DefinitionGenerator, + LLVMOrcDefinitionGeneratorRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(InProgressLookupState, LLVMOrcLookupStateRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeContext, LLVMOrcThreadSafeContextRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeModule, LLVMOrcThreadSafeModuleRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITTargetMachineBuilder, LLVMOrcJITTargetMachineBuilderRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ObjectLayer, LLVMOrcObjectLayerRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJITBuilder, LLVMOrcLLJITBuilderRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJIT, LLVMOrcLLJITRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) +namespace llvm { +namespace orc { + +class CAPIDefinitionGenerator final : public DefinitionGenerator { +public: + CAPIDefinitionGenerator( + void *Ctx, + LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction TryToGenerate) + : Ctx(Ctx), TryToGenerate(TryToGenerate) {} + + Error tryToGenerate(LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, + const SymbolLookupSet &LookupSet) override { + + // Take the lookup state. + LLVMOrcLookupStateRef LSR = ::wrap(OrcV2CAPIHelper::extractLookupState(LS)); + + // Translate the lookup kind. + LLVMOrcLookupKind CLookupKind; + switch (K) { + case LookupKind::Static: + CLookupKind = LLVMOrcLookupKindStatic; + break; + case LookupKind::DLSym: + CLookupKind = LLVMOrcLookupKindDLSym; + break; + } + + // Translate the JITDylibSearchFlags. + LLVMOrcJITDylibLookupFlags CJDLookupFlags; + switch (JDLookupFlags) { + case JITDylibLookupFlags::MatchExportedSymbolsOnly: + CJDLookupFlags = LLVMOrcJITDylibLookupFlagsMatchExportedSymbolsOnly; + break; + case JITDylibLookupFlags::MatchAllSymbols: + CJDLookupFlags = LLVMOrcJITDylibLookupFlagsMatchAllSymbols; + break; + } + + // Translate the lookup set. + std::vector<LLVMOrcCLookupSetElement> CLookupSet; + CLookupSet.reserve(LookupSet.size()); + for (auto &KV : LookupSet) { + LLVMOrcSymbolLookupFlags SLF; + LLVMOrcSymbolStringPoolEntryRef Name = + ::wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(KV.first)); + switch (KV.second) { + case SymbolLookupFlags::RequiredSymbol: + SLF = LLVMOrcSymbolLookupFlagsRequiredSymbol; + break; + case SymbolLookupFlags::WeaklyReferencedSymbol: + SLF = LLVMOrcSymbolLookupFlagsWeaklyReferencedSymbol; + break; + } + CLookupSet.push_back({Name, SLF}); + } + + // Run the C TryToGenerate function. + auto Err = unwrap(TryToGenerate(::wrap(this), Ctx, &LSR, CLookupKind, + ::wrap(&JD), CJDLookupFlags, + CLookupSet.data(), CLookupSet.size())); + + // Restore the lookup state. + OrcV2CAPIHelper::resetLookupState(LS, ::unwrap(LSR)); + + return Err; + } + +private: + void *Ctx; + LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction TryToGenerate; +}; + +} // end namespace orc +} // end namespace llvm + +void LLVMOrcExecutionSessionSetErrorReporter( + LLVMOrcExecutionSessionRef ES, LLVMOrcErrorReporterFunction ReportError, + void *Ctx) { + unwrap(ES)->setErrorReporter( + [=](Error Err) { ReportError(Ctx, wrap(std::move(Err))); }); +} + +LLVMOrcSymbolStringPoolRef +LLVMOrcExecutionSessionGetSymbolStringPool(LLVMOrcExecutionSessionRef ES) { + return wrap(unwrap(ES)->getSymbolStringPool().get()); +} + +void LLVMOrcSymbolStringPoolClearDeadEntries(LLVMOrcSymbolStringPoolRef SSP) { + unwrap(SSP)->clearDeadEntries(); +} + LLVMOrcSymbolStringPoolEntryRef LLVMOrcExecutionSessionIntern(LLVMOrcExecutionSessionRef ES, const char *Name) { return wrap( OrcV2CAPIHelper::releaseSymbolStringPtr(unwrap(ES)->intern(Name))); } +void LLVMOrcRetainSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { + OrcV2CAPIHelper::retainPoolEntry(unwrap(S)); +} + void LLVMOrcReleaseSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { OrcV2CAPIHelper::releasePoolEntry(unwrap(S)); } -void LLVMOrcDisposeJITDylibDefinitionGenerator( - LLVMOrcJITDylibDefinitionGeneratorRef DG) { - delete unwrap(DG); +const char *LLVMOrcSymbolStringPoolEntryStr(LLVMOrcSymbolStringPoolEntryRef S) { + return unwrap(S)->getKey().data(); +} + +LLVMOrcResourceTrackerRef +LLVMOrcJITDylibCreateResourceTracker(LLVMOrcJITDylibRef JD) { + auto RT = unwrap(JD)->createResourceTracker(); + // Retain the pointer for the C API client. + RT->Retain(); + return wrap(RT.get()); +} + +LLVMOrcResourceTrackerRef +LLVMOrcJITDylibGetDefaultResourceTracker(LLVMOrcJITDylibRef JD) { + auto RT = unwrap(JD)->getDefaultResourceTracker(); + // Retain the pointer for the C API client. + return wrap(RT.get()); +} + +void LLVMOrcReleaseResourceTracker(LLVMOrcResourceTrackerRef RT) { + ResourceTrackerSP TmpRT(unwrap(RT)); + TmpRT->Release(); +} + +void LLVMOrcResourceTrackerTransferTo(LLVMOrcResourceTrackerRef SrcRT, + LLVMOrcResourceTrackerRef DstRT) { + ResourceTrackerSP TmpRT(unwrap(SrcRT)); + TmpRT->transferTo(*unwrap(DstRT)); +} + +LLVMErrorRef LLVMOrcResourceTrackerRemove(LLVMOrcResourceTrackerRef RT) { + ResourceTrackerSP TmpRT(unwrap(RT)); + return wrap(TmpRT->remove()); +} + +void LLVMOrcDisposeDefinitionGenerator(LLVMOrcDefinitionGeneratorRef DG) { + std::unique_ptr<DefinitionGenerator> TmpDG(unwrap(DG)); +} + +void LLVMOrcDisposeMaterializationUnit(LLVMOrcMaterializationUnitRef MU) { + std::unique_ptr<MaterializationUnit> TmpMU(unwrap(MU)); +} + +LLVMOrcMaterializationUnitRef +LLVMOrcAbsoluteSymbols(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs) { + SymbolMap SM; + for (size_t I = 0; I != NumPairs; ++I) { + JITSymbolFlags Flags; + + if (Syms[I].Sym.Flags.GenericFlags & LLVMJITSymbolGenericFlagsExported) + Flags |= JITSymbolFlags::Exported; + if (Syms[I].Sym.Flags.GenericFlags & LLVMJITSymbolGenericFlagsWeak) + Flags |= JITSymbolFlags::Weak; + + Flags.getTargetFlags() = Syms[I].Sym.Flags.TargetFlags; + + SM[OrcV2CAPIHelper::retainSymbolStringPtr(unwrap(Syms[I].Name))] = + JITEvaluatedSymbol(Syms[I].Sym.Address, Flags); + } + + return wrap(absoluteSymbols(std::move(SM)).release()); +} + +LLVMOrcJITDylibRef +LLVMOrcExecutionSessionCreateBareJITDylib(LLVMOrcExecutionSessionRef ES, + const char *Name) { + return wrap(&unwrap(ES)->createBareJITDylib(Name)); +} + +LLVMErrorRef +LLVMOrcExecutionSessionCreateJITDylib(LLVMOrcExecutionSessionRef ES, + LLVMOrcJITDylibRef *Result, + const char *Name) { + auto JD = unwrap(ES)->createJITDylib(Name); + if (!JD) + return wrap(JD.takeError()); + *Result = wrap(&*JD); + return LLVMErrorSuccess; +} + +LLVMOrcJITDylibRef +LLVMOrcExecutionSessionGetJITDylibByName(LLVMOrcExecutionSessionRef ES, + const char *Name) { + return wrap(unwrap(ES)->getJITDylibByName(Name)); +} + +LLVMErrorRef LLVMOrcJITDylibDefine(LLVMOrcJITDylibRef JD, + LLVMOrcMaterializationUnitRef MU) { + std::unique_ptr<MaterializationUnit> TmpMU(unwrap(MU)); + + if (auto Err = unwrap(JD)->define(TmpMU)) { + TmpMU.release(); + return wrap(std::move(Err)); + } + return LLVMErrorSuccess; +} + +LLVMErrorRef LLVMOrcJITDylibClear(LLVMOrcJITDylibRef JD) { + return wrap(unwrap(JD)->clear()); } void LLVMOrcJITDylibAddGenerator(LLVMOrcJITDylibRef JD, - LLVMOrcJITDylibDefinitionGeneratorRef DG) { - unwrap(JD)->addGenerator( - std::unique_ptr<JITDylib::DefinitionGenerator>(unwrap(DG))); + LLVMOrcDefinitionGeneratorRef DG) { + unwrap(JD)->addGenerator(std::unique_ptr<DefinitionGenerator>(unwrap(DG))); +} + +LLVMOrcDefinitionGeneratorRef LLVMOrcCreateCustomCAPIDefinitionGenerator( + LLVMOrcCAPIDefinitionGeneratorTryToGenerateFunction F, void *Ctx) { + auto DG = std::make_unique<CAPIDefinitionGenerator>(Ctx, F); + return wrap(DG.release()); } LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( - LLVMOrcJITDylibDefinitionGeneratorRef *Result, char GlobalPrefix, + LLVMOrcDefinitionGeneratorRef *Result, char GlobalPrefix, LLVMOrcSymbolPredicate Filter, void *FilterCtx) { assert(Result && "Result can not be null"); assert((Filter || !FilterCtx) && @@ -89,7 +315,7 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( DynamicLibrarySearchGenerator::SymbolPredicate Pred; if (Filter) Pred = [=](const SymbolStringPtr &Name) -> bool { - return Filter(wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name)), FilterCtx); + return Filter(FilterCtx, wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); }; auto ProcessSymsGenerator = @@ -143,7 +369,7 @@ LLVMErrorRef LLVMOrcJITTargetMachineBuilderDetectHost( } LLVMOrcJITTargetMachineBuilderRef -LLVMOrcJITTargetMachineBuilderFromTargetMachine(LLVMTargetMachineRef TM) { +LLVMOrcJITTargetMachineBuilderCreateFromTargetMachine(LLVMTargetMachineRef TM) { auto *TemplateTM = unwrap(TM); auto JTMB = @@ -167,6 +393,10 @@ void LLVMOrcDisposeJITTargetMachineBuilder( delete unwrap(JTMB); } +void lLVMOrcDisposeObjectLayer(LLVMOrcObjectLayerRef ObjLayer) { + delete unwrap(ObjLayer); +} + LLVMOrcLLJITBuilderRef LLVMOrcCreateLLJITBuilder(void) { return wrap(new LLJITBuilder()); } @@ -180,6 +410,17 @@ void LLVMOrcLLJITBuilderSetJITTargetMachineBuilder( unwrap(Builder)->setJITTargetMachineBuilder(*unwrap(JTMB)); } +void LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator( + LLVMOrcLLJITBuilderRef Builder, + LLVMOrcLLJITBuilderObjectLinkingLayerCreatorFunction F, void *Ctx) { + unwrap(Builder)->setObjectLinkingLayerCreator( + [=](ExecutionSession &ES, const Triple &TT) { + auto TTStr = TT.str(); + return std::unique_ptr<ObjectLayer>( + unwrap(F(Ctx, wrap(&ES), TTStr.c_str()))); + }); +} + LLVMErrorRef LLVMOrcCreateLLJIT(LLVMOrcLLJITRef *Result, LLVMOrcLLJITBuilderRef Builder) { assert(Result && "Result can not be null"); @@ -232,10 +473,27 @@ LLVMErrorRef LLVMOrcLLJITAddObjectFile(LLVMOrcLLJITRef J, LLVMOrcJITDylibRef JD, *unwrap(JD), std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); } +LLVMErrorRef LLVMOrcLLJITAddObjectFileWithRT(LLVMOrcLLJITRef J, + LLVMOrcResourceTrackerRef RT, + LLVMMemoryBufferRef ObjBuffer) { + return wrap(unwrap(J)->addObjectFile( + ResourceTrackerSP(unwrap(RT)), + std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); +} + LLVMErrorRef LLVMOrcLLJITAddLLVMIRModule(LLVMOrcLLJITRef J, LLVMOrcJITDylibRef JD, LLVMOrcThreadSafeModuleRef TSM) { - return wrap(unwrap(J)->addIRModule(*unwrap(JD), std::move(*unwrap(TSM)))); + std::unique_ptr<ThreadSafeModule> TmpTSM(unwrap(TSM)); + return wrap(unwrap(J)->addIRModule(*unwrap(JD), std::move(*TmpTSM))); +} + +LLVMErrorRef LLVMOrcLLJITAddLLVMIRModuleWithRT(LLVMOrcLLJITRef J, + LLVMOrcResourceTrackerRef RT, + LLVMOrcThreadSafeModuleRef TSM) { + std::unique_ptr<ThreadSafeModule> TmpTSM(unwrap(TSM)); + return wrap(unwrap(J)->addIRModule(ResourceTrackerSP(unwrap(RT)), + std::move(*TmpTSM))); } LLVMErrorRef LLVMOrcLLJITLookup(LLVMOrcLLJITRef J, @@ -252,3 +510,20 @@ LLVMErrorRef LLVMOrcLLJITLookup(LLVMOrcLLJITRef J, *Result = Sym->getAddress(); return LLVMErrorSuccess; } + +LLVMOrcObjectLayerRef +LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager( + LLVMOrcExecutionSessionRef ES) { + assert(ES && "ES must not be null"); + return wrap(new RTDyldObjectLinkingLayer( + *unwrap(ES), [] { return std::make_unique<SectionMemoryManager>(); })); +} + +void LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener( + LLVMOrcObjectLayerRef RTDyldObjLinkingLayer, + LLVMJITEventListenerRef Listener) { + assert(RTDyldObjLinkingLayer && "RTDyldObjLinkingLayer must not be null"); + assert(Listener && "Listener must not be null"); + reinterpret_cast<RTDyldObjectLinkingLayer *>(unwrap(RTDyldObjLinkingLayer)) + ->registerJITEventListener(*unwrap(Listener)); +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 21925726072e..0ad666ebbebd 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -18,7 +18,7 @@ class JITDylibSearchOrderResolver : public JITSymbolResolver { public: JITDylibSearchOrderResolver(MaterializationResponsibility &MR) : MR(MR) {} - void lookup(const LookupSet &Symbols, OnResolvedFunction OnResolved) { + void lookup(const LookupSet &Symbols, OnResolvedFunction OnResolved) override { auto &ES = MR.getTargetJITDylib().getExecutionSession(); SymbolLookupSet InternedSymbols; @@ -55,7 +55,7 @@ public: RegisterDependencies); } - Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) { + Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) override { LookupSet Result; for (auto &KV : MR.getSymbols()) { @@ -77,35 +77,26 @@ namespace orc { RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager) - : ObjectLayer(ES), GetMemoryManager(GetMemoryManager) {} + : ObjectLayer(ES), GetMemoryManager(GetMemoryManager) { + ES.registerResourceManager(*this); +} RTDyldObjectLinkingLayer::~RTDyldObjectLinkingLayer() { - std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); - for (auto &MemMgr : MemMgrs) { - for (auto *L : EventListeners) - L->notifyFreeingObject( - static_cast<uint64_t>(reinterpret_cast<uintptr_t>(MemMgr.get()))); - MemMgr->deregisterEHFrames(); - } + assert(MemMgrs.empty() && "Layer destroyed with resources still attached"); } -void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, - std::unique_ptr<MemoryBuffer> O) { +void RTDyldObjectLinkingLayer::emit( + std::unique_ptr<MaterializationResponsibility> R, + std::unique_ptr<MemoryBuffer> O) { assert(O && "Object must not be null"); - // This method launches an asynchronous link step that will fulfill our - // materialization responsibility. We need to switch R to be heap - // allocated before that happens so it can live as long as the asynchronous - // link needs it to (i.e. it must be able to outlive this method). - auto SharedR = std::make_shared<MaterializationResponsibility>(std::move(R)); - auto &ES = getExecutionSession(); auto Obj = object::ObjectFile::createObjectFile(*O); if (!Obj) { getExecutionSession().reportError(Obj.takeError()); - SharedR->failMaterialization(); + R->failMaterialization(); return; } @@ -121,7 +112,7 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, continue; } else { ES.reportError(SymType.takeError()); - R.failMaterialization(); + R->failMaterialization(); return; } @@ -129,7 +120,7 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, if (!SymFlagsOrErr) { // TODO: Test this error. ES.reportError(SymFlagsOrErr.takeError()); - R.failMaterialization(); + R->failMaterialization(); return; } @@ -139,46 +130,44 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, InternalSymbols->insert(*SymName); else { ES.reportError(SymName.takeError()); - R.failMaterialization(); + R->failMaterialization(); return; } } } } - auto K = R.getVModuleKey(); - RuntimeDyld::MemoryManager *MemMgr = nullptr; + auto MemMgr = GetMemoryManager(); + auto &MemMgrRef = *MemMgr; - // Create a record a memory manager for this object. - { - auto Tmp = GetMemoryManager(); - std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); - MemMgrs.push_back(std::move(Tmp)); - MemMgr = MemMgrs.back().get(); - } + // Switch to shared ownership of MR so that it can be captured by both + // lambdas below. + std::shared_ptr<MaterializationResponsibility> SharedR(std::move(R)); JITDylibSearchOrderResolver Resolver(*SharedR); jitLinkForORC( object::OwningBinary<object::ObjectFile>(std::move(*Obj), std::move(O)), - *MemMgr, Resolver, ProcessAllSections, - [this, K, SharedR, MemMgr, InternalSymbols]( + MemMgrRef, Resolver, ProcessAllSections, + [this, SharedR, &MemMgrRef, InternalSymbols]( const object::ObjectFile &Obj, - std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, + RuntimeDyld::LoadedObjectInfo &LoadedObjInfo, std::map<StringRef, JITEvaluatedSymbol> ResolvedSymbols) { - return onObjLoad(K, *SharedR, Obj, MemMgr, std::move(LoadedObjInfo), + return onObjLoad(*SharedR, Obj, MemMgrRef, LoadedObjInfo, ResolvedSymbols, *InternalSymbols); }, - [this, K, SharedR, MemMgr](object::OwningBinary<object::ObjectFile> Obj, - Error Err) mutable { - onObjEmit(K, *SharedR, std::move(Obj), MemMgr, std::move(Err)); + [this, SharedR, MemMgr = std::move(MemMgr)]( + object::OwningBinary<object::ObjectFile> Obj, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, + Error Err) mutable { + onObjEmit(*SharedR, std::move(Obj), std::move(MemMgr), + std::move(LoadedObjInfo), std::move(Err)); }); } void RTDyldObjectLinkingLayer::registerJITEventListener(JITEventListener &L) { std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); - assert(llvm::none_of(EventListeners, - [&](JITEventListener *O) { return O == &L; }) && + assert(!llvm::is_contained(EventListeners, &L) && "Listener has already been registered"); EventListeners.push_back(&L); } @@ -191,9 +180,9 @@ void RTDyldObjectLinkingLayer::unregisterJITEventListener(JITEventListener &L) { } Error RTDyldObjectLinkingLayer::onObjLoad( - VModuleKey K, MaterializationResponsibility &R, - const object::ObjectFile &Obj, RuntimeDyld::MemoryManager *MemMgr, - std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, + MaterializationResponsibility &R, const object::ObjectFile &Obj, + RuntimeDyld::MemoryManager &MemMgr, + RuntimeDyld::LoadedObjectInfo &LoadedObjInfo, std::map<StringRef, JITEvaluatedSymbol> Resolved, std::set<StringRef> &InternalSymbols) { SymbolFlagsMap ExtraSymbolsToClaim; @@ -274,19 +263,16 @@ Error RTDyldObjectLinkingLayer::onObjLoad( } if (NotifyLoaded) - NotifyLoaded(K, Obj, *LoadedObjInfo); - - std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); - assert(!LoadedObjInfos.count(MemMgr) && "Duplicate loaded info for MemMgr"); - LoadedObjInfos[MemMgr] = std::move(LoadedObjInfo); + NotifyLoaded(R, Obj, LoadedObjInfo); return Error::success(); } void RTDyldObjectLinkingLayer::onObjEmit( - VModuleKey K, MaterializationResponsibility &R, + MaterializationResponsibility &R, object::OwningBinary<object::ObjectFile> O, - RuntimeDyld::MemoryManager *MemMgr, Error Err) { + std::unique_ptr<RuntimeDyld::MemoryManager> MemMgr, + std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, Error Err) { if (Err) { getExecutionSession().reportError(std::move(Err)); R.failMaterialization(); @@ -306,27 +292,60 @@ void RTDyldObjectLinkingLayer::onObjEmit( // Run EventListener notifyLoaded callbacks. { std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); - auto LOIItr = LoadedObjInfos.find(MemMgr); - assert(LOIItr != LoadedObjInfos.end() && "LoadedObjInfo missing"); for (auto *L : EventListeners) - L->notifyObjectLoaded( - static_cast<uint64_t>(reinterpret_cast<uintptr_t>(MemMgr)), *Obj, - *LOIItr->second); - LoadedObjInfos.erase(MemMgr); + L->notifyObjectLoaded(pointerToJITTargetAddress(MemMgr.get()), *Obj, + *LoadedObjInfo); } if (NotifyEmitted) - NotifyEmitted(K, std::move(ObjBuffer)); + NotifyEmitted(R, std::move(ObjBuffer)); + + if (auto Err = R.withResourceKeyDo( + [&](ResourceKey K) { MemMgrs[K].push_back(std::move(MemMgr)); })) { + getExecutionSession().reportError(std::move(Err)); + R.failMaterialization(); + } } -LegacyRTDyldObjectLinkingLayer::LegacyRTDyldObjectLinkingLayer( - ExecutionSession &ES, ResourcesGetter GetResources, - NotifyLoadedFtor NotifyLoaded, NotifyFinalizedFtor NotifyFinalized, - NotifyFreedFtor NotifyFreed) - : ES(ES), GetResources(std::move(GetResources)), - NotifyLoaded(std::move(NotifyLoaded)), - NotifyFinalized(std::move(NotifyFinalized)), - NotifyFreed(std::move(NotifyFreed)), ProcessAllSections(false) {} +Error RTDyldObjectLinkingLayer::handleRemoveResources(ResourceKey K) { + + std::vector<MemoryManagerUP> MemMgrsToRemove; + + getExecutionSession().runSessionLocked([&] { + auto I = MemMgrs.find(K); + if (I != MemMgrs.end()) { + std::swap(MemMgrsToRemove, I->second); + MemMgrs.erase(I); + } + }); + + { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + for (auto &MemMgr : MemMgrsToRemove) { + for (auto *L : EventListeners) + L->notifyFreeingObject(pointerToJITTargetAddress(MemMgr.get())); + MemMgr->deregisterEHFrames(); + } + } + + return Error::success(); +} + +void RTDyldObjectLinkingLayer::handleTransferResources(ResourceKey DstKey, + ResourceKey SrcKey) { + auto I = MemMgrs.find(SrcKey); + if (I != MemMgrs.end()) { + auto &SrcMemMgrs = I->second; + auto &DstMemMgrs = MemMgrs[DstKey]; + DstMemMgrs.reserve(DstMemMgrs.size() + SrcMemMgrs.size()); + for (auto &MemMgr : SrcMemMgrs) + DstMemMgrs.push_back(std::move(MemMgr)); + + // Erase SrcKey entry using value rather than iterator I: I may have been + // invalidated when we looked up DstKey. + MemMgrs.erase(SrcKey); + } +} } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcError.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcError.cpp new file mode 100644 index 000000000000..fdad90cbcfb7 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcError.cpp @@ -0,0 +1,120 @@ +//===---------------- OrcError.cpp - Error codes for ORC ------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Error codes for ORC. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/ManagedStatic.h" + +#include <type_traits> + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class OrcErrorCategory : public std::error_category { +public: + const char *name() const noexcept override { return "orc"; } + + std::string message(int condition) const override { + switch (static_cast<OrcErrorCode>(condition)) { + case OrcErrorCode::UnknownORCError: + return "Unknown ORC error"; + case OrcErrorCode::DuplicateDefinition: + return "Duplicate symbol definition"; + case OrcErrorCode::JITSymbolNotFound: + return "JIT symbol not found"; + case OrcErrorCode::RemoteAllocatorDoesNotExist: + return "Remote allocator does not exist"; + case OrcErrorCode::RemoteAllocatorIdAlreadyInUse: + return "Remote allocator Id already in use"; + case OrcErrorCode::RemoteMProtectAddrUnrecognized: + return "Remote mprotect call references unallocated memory"; + case OrcErrorCode::RemoteIndirectStubsOwnerDoesNotExist: + return "Remote indirect stubs owner does not exist"; + case OrcErrorCode::RemoteIndirectStubsOwnerIdAlreadyInUse: + return "Remote indirect stubs owner Id already in use"; + case OrcErrorCode::RPCConnectionClosed: + return "RPC connection closed"; + case OrcErrorCode::RPCCouldNotNegotiateFunction: + return "Could not negotiate RPC function"; + case OrcErrorCode::RPCResponseAbandoned: + return "RPC response abandoned"; + case OrcErrorCode::UnexpectedRPCCall: + return "Unexpected RPC call"; + case OrcErrorCode::UnexpectedRPCResponse: + return "Unexpected RPC response"; + case OrcErrorCode::UnknownErrorCodeFromRemote: + return "Unknown error returned from remote RPC function " + "(Use StringError to get error message)"; + case OrcErrorCode::UnknownResourceHandle: + return "Unknown resource handle"; + case OrcErrorCode::MissingSymbolDefinitions: + return "MissingSymbolsDefinitions"; + case OrcErrorCode::UnexpectedSymbolDefinitions: + return "UnexpectedSymbolDefinitions"; + } + llvm_unreachable("Unhandled error code"); + } +}; + +static ManagedStatic<OrcErrorCategory> OrcErrCat; +} // namespace + +namespace llvm { +namespace orc { + +char DuplicateDefinition::ID = 0; +char JITSymbolNotFound::ID = 0; + +std::error_code orcError(OrcErrorCode ErrCode) { + typedef std::underlying_type<OrcErrorCode>::type UT; + return std::error_code(static_cast<UT>(ErrCode), *OrcErrCat); +} + +DuplicateDefinition::DuplicateDefinition(std::string SymbolName) + : SymbolName(std::move(SymbolName)) {} + +std::error_code DuplicateDefinition::convertToErrorCode() const { + return orcError(OrcErrorCode::DuplicateDefinition); +} + +void DuplicateDefinition::log(raw_ostream &OS) const { + OS << "Duplicate definition of symbol '" << SymbolName << "'"; +} + +const std::string &DuplicateDefinition::getSymbolName() const { + return SymbolName; +} + +JITSymbolNotFound::JITSymbolNotFound(std::string SymbolName) + : SymbolName(std::move(SymbolName)) {} + +std::error_code JITSymbolNotFound::convertToErrorCode() const { + typedef std::underlying_type<OrcErrorCode>::type UT; + return std::error_code(static_cast<UT>(OrcErrorCode::JITSymbolNotFound), + *OrcErrCat); +} + +void JITSymbolNotFound::log(raw_ostream &OS) const { + OS << "Could not find symbol '" << SymbolName << "'"; +} + +const std::string &JITSymbolNotFound::getSymbolName() const { + return SymbolName; +} + +} // namespace orc +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/RPCError.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/RPCError.cpp new file mode 100644 index 000000000000..a55cb220f218 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/RPCError.cpp @@ -0,0 +1,58 @@ +//===--------------- RPCError.cpp - RPCERror implementation ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// RPC Error type implmentations. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/RPCUtils.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" + +#include <string> +#include <system_error> + +char llvm::orc::shared::RPCFatalError::ID = 0; +char llvm::orc::shared::ConnectionClosed::ID = 0; +char llvm::orc::shared::ResponseAbandoned::ID = 0; +char llvm::orc::shared::CouldNotNegotiate::ID = 0; + +namespace llvm { +namespace orc { +namespace shared { + +std::error_code ConnectionClosed::convertToErrorCode() const { + return orcError(OrcErrorCode::RPCConnectionClosed); +} + +void ConnectionClosed::log(raw_ostream &OS) const { + OS << "RPC connection already closed"; +} + +std::error_code ResponseAbandoned::convertToErrorCode() const { + return orcError(OrcErrorCode::RPCResponseAbandoned); +} + +void ResponseAbandoned::log(raw_ostream &OS) const { + OS << "RPC response abandoned"; +} + +CouldNotNegotiate::CouldNotNegotiate(std::string Signature) + : Signature(std::move(Signature)) {} + +std::error_code CouldNotNegotiate::convertToErrorCode() const { + return orcError(OrcErrorCode::RPCCouldNotNegotiateFunction); +} + +void CouldNotNegotiate::log(raw_ostream &OS) const { + OS << "Could not negotiate RPC function " << Signature; +} + +} // end namespace shared +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.cpp new file mode 100644 index 000000000000..52d11f0741d4 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.cpp @@ -0,0 +1,44 @@ +//===---------- TargetProcessControlTypes.cpp - Shared TPC types ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// TargetProcessControl types. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" + +namespace llvm { +namespace orc { +namespace tpctypes { + +WrapperFunctionResult WrapperFunctionResult::from(StringRef S) { + CWrapperFunctionResult R; + zeroInit(R); + R.Size = S.size(); + if (R.Size > sizeof(uint64_t)) { + R.Data.ValuePtr = new uint8_t[R.Size]; + memcpy(R.Data.ValuePtr, S.data(), R.Size); + R.Destroy = destroyWithDeleteArray; + } else + memcpy(R.Data.Value, S.data(), R.Size); + return R; +} + +void WrapperFunctionResult::destroyWithFree(CWrapperFunctionResultData Data, + uint64_t Size) { + free(Data.ValuePtr); +} + +void WrapperFunctionResult::destroyWithDeleteArray( + CWrapperFunctionResultData Data, uint64_t Size) { + delete[] Data.ValuePtr; +} + +} // end namespace tpctypes +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp index 7240c1ed0ce9..c2fa4466eab6 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp @@ -106,11 +106,10 @@ BlockFreqQuery::ResultTy BlockFreqQuery::operator()(Function &F) { assert(IBBs.size() == BBFreqs.size() && "BB Count Mismatch"); - llvm::sort(BBFreqs.begin(), BBFreqs.end(), - [](decltype(BBFreqs)::const_reference BBF, - decltype(BBFreqs)::const_reference BBS) { - return BBF.second > BBS.second ? true : false; - }); + llvm::sort(BBFreqs, [](decltype(BBFreqs)::const_reference BBF, + decltype(BBFreqs)::const_reference BBS) { + return BBF.second > BBS.second ? true : false; + }); // ignoring number of direct calls in a BB auto Topk = numBBToGet(BBFreqs.size()); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp index 0530b1a97b67..0b4755fe23cf 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp @@ -16,9 +16,6 @@ #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/Verifier.h" -#include "llvm/Support/Debug.h" - -#include <vector> namespace llvm { @@ -58,7 +55,7 @@ Error Speculator::addSpeculationRuntime(JITDylib &JD, // If two modules, share the same LLVMContext, different threads must // not access them concurrently without locking the associated LLVMContext // this implementation follows this contract. -void IRSpeculationLayer::emit(MaterializationResponsibility R, +void IRSpeculationLayer::emit(std::unique_ptr<MaterializationResponsibility> R, ThreadSafeModule TSM) { assert(TSM && "Speculation Layer received Null Module ?"); @@ -130,7 +127,7 @@ void IRSpeculationLayer::emit(MaterializationResponsibility R, assert(Mutator.GetInsertBlock()->getParent() == &Fn && "IR builder association mismatch?"); S.registerSymbols(internToJITSymbols(IRNames.getValue()), - &R.getTargetJITDylib()); + &R->getTargetJITDylib()); } } } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp new file mode 100644 index 000000000000..bbf3ada1d4ba --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp @@ -0,0 +1,70 @@ +//===---------------- TPCDynamicLibrarySearchGenerator.cpp ----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h" + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<TPCDynamicLibrarySearchGenerator>> +TPCDynamicLibrarySearchGenerator::Load(TargetProcessControl &TPC, + const char *LibraryPath, + SymbolPredicate Allow) { + auto Handle = TPC.loadDylib(LibraryPath); + if (!Handle) + return Handle.takeError(); + + return std::make_unique<TPCDynamicLibrarySearchGenerator>(TPC, *Handle, + std::move(Allow)); +} + +Error TPCDynamicLibrarySearchGenerator::tryToGenerate( + LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { + + if (Symbols.empty()) + return Error::success(); + + SymbolLookupSet LookupSymbols; + + for (auto &KV : Symbols) { + // Skip symbols that don't match the filter. + if (Allow && !Allow(KV.first)) + continue; + LookupSymbols.add(KV.first, SymbolLookupFlags::WeaklyReferencedSymbol); + } + + SymbolMap NewSymbols; + + TargetProcessControl::LookupRequest Request(H, LookupSymbols); + auto Result = TPC.lookupSymbols(Request); + if (!Result) + return Result.takeError(); + + assert(Result->size() == 1 && "Results for more than one library returned"); + assert(Result->front().size() == LookupSymbols.size() && + "Result has incorrect number of elements"); + + auto ResultI = Result->front().begin(); + for (auto &KV : LookupSymbols) { + if (*ResultI) + NewSymbols[KV.first] = + JITEvaluatedSymbol(*ResultI, JITSymbolFlags::Exported); + ++ResultI; + } + + // If there were no resolved symbols bail out. + if (NewSymbols.empty()) + return Error::success(); + + // Define resolved symbols. + return JD.define(absoluteSymbols(std::move(NewSymbols))); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCEHFrameRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCEHFrameRegistrar.cpp new file mode 100644 index 000000000000..4f901ce6d445 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCEHFrameRegistrar.cpp @@ -0,0 +1,80 @@ +//===------ TPCEHFrameRegistrar.cpp - TPC-based eh-frame registration -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TPCEHFrameRegistrar.h" +#include "llvm/Support/BinaryStreamWriter.h" + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<TPCEHFrameRegistrar>> +TPCEHFrameRegistrar::Create(TargetProcessControl &TPC) { + // FIXME: Proper mangling here -- we really need to decouple linker mangling + // from DataLayout. + + // Find the addresses of the registration/deregistration functions in the + // target process. + auto ProcessHandle = TPC.loadDylib(nullptr); + if (!ProcessHandle) + return ProcessHandle.takeError(); + + std::string RegisterWrapperName, DeregisterWrapperName; + if (TPC.getTargetTriple().isOSBinFormatMachO()) { + RegisterWrapperName += '_'; + DeregisterWrapperName += '_'; + } + RegisterWrapperName += "llvm_orc_registerEHFrameSectionWrapper"; + DeregisterWrapperName += "llvm_orc_deregisterEHFrameSectionWrapper"; + + SymbolLookupSet RegistrationSymbols; + RegistrationSymbols.add(TPC.intern(RegisterWrapperName)); + RegistrationSymbols.add(TPC.intern(DeregisterWrapperName)); + + auto Result = TPC.lookupSymbols({{*ProcessHandle, RegistrationSymbols}}); + if (!Result) + return Result.takeError(); + + assert(Result->size() == 1 && "Unexpected number of dylibs in result"); + assert((*Result)[0].size() == 2 && + "Unexpected number of addresses in result"); + + auto RegisterEHFrameWrapperFnAddr = (*Result)[0][0]; + auto DeregisterEHFrameWrapperFnAddr = (*Result)[0][1]; + + return std::make_unique<TPCEHFrameRegistrar>( + TPC, RegisterEHFrameWrapperFnAddr, DeregisterEHFrameWrapperFnAddr); +} + +Error TPCEHFrameRegistrar::registerEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) { + constexpr size_t ArgBufferSize = sizeof(uint64_t) + sizeof(uint64_t); + uint8_t ArgBuffer[ArgBufferSize]; + BinaryStreamWriter ArgWriter( + MutableArrayRef<uint8_t>(ArgBuffer, ArgBufferSize), + support::endianness::big); + cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionAddr))); + cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionSize))); + + return TPC.runWrapper(RegisterEHFrameWrapperFnAddr, ArgBuffer).takeError(); +} + +Error TPCEHFrameRegistrar::deregisterEHFrames( + JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { + constexpr size_t ArgBufferSize = sizeof(uint64_t) + sizeof(uint64_t); + uint8_t ArgBuffer[ArgBufferSize]; + BinaryStreamWriter ArgWriter( + MutableArrayRef<uint8_t>(ArgBuffer, ArgBufferSize), + support::endianness::big); + cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionAddr))); + cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionSize))); + + return TPC.runWrapper(DeregisterEHFrameWrapperFnAddr, ArgBuffer).takeError(); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp new file mode 100644 index 000000000000..7989ec41952d --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp @@ -0,0 +1,423 @@ +//===------ TargetProcessControl.cpp -- Target process control APIs -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h" + +#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" +#include "llvm/Support/MathExtras.h" + +#include <future> + +using namespace llvm; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class TPCIndirectionUtilsAccess { +public: + using IndirectStubInfo = TPCIndirectionUtils::IndirectStubInfo; + using IndirectStubInfoVector = TPCIndirectionUtils::IndirectStubInfoVector; + + static Expected<IndirectStubInfoVector> + getIndirectStubs(TPCIndirectionUtils &TPCIU, unsigned NumStubs) { + return TPCIU.getIndirectStubs(NumStubs); + }; +}; + +} // end namespace orc +} // end namespace llvm + +namespace { + +class TPCTrampolinePool : public TrampolinePool { +public: + TPCTrampolinePool(TPCIndirectionUtils &TPCIU); + Error deallocatePool(); + +protected: + Error grow() override; + + using Allocation = jitlink::JITLinkMemoryManager::Allocation; + + TPCIndirectionUtils &TPCIU; + unsigned TrampolineSize = 0; + unsigned TrampolinesPerPage = 0; + std::vector<std::unique_ptr<Allocation>> TrampolineBlocks; +}; + +class TPCIndirectStubsManager : public IndirectStubsManager, + private TPCIndirectionUtilsAccess { +public: + TPCIndirectStubsManager(TPCIndirectionUtils &TPCIU) : TPCIU(TPCIU) {} + + Error deallocateStubs(); + + Error createStub(StringRef StubName, JITTargetAddress StubAddr, + JITSymbolFlags StubFlags) override; + + Error createStubs(const StubInitsMap &StubInits) override; + + JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; + + JITEvaluatedSymbol findPointer(StringRef Name) override; + + Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; + +private: + using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; + + std::mutex ISMMutex; + TPCIndirectionUtils &TPCIU; + StringMap<StubInfo> StubInfos; +}; + +TPCTrampolinePool::TPCTrampolinePool(TPCIndirectionUtils &TPCIU) + : TPCIU(TPCIU) { + auto &TPC = TPCIU.getTargetProcessControl(); + auto &ABI = TPCIU.getABISupport(); + + TrampolineSize = ABI.getTrampolineSize(); + TrampolinesPerPage = + (TPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; +} + +Error TPCTrampolinePool::deallocatePool() { + Error Err = Error::success(); + for (auto &Alloc : TrampolineBlocks) + Err = joinErrors(std::move(Err), Alloc->deallocate()); + return Err; +} + +Error TPCTrampolinePool::grow() { + assert(AvailableTrampolines.empty() && + "Grow called with trampolines still available"); + + auto ResolverAddress = TPCIU.getResolverBlockAddress(); + assert(ResolverAddress && "Resolver address can not be null"); + + auto &TPC = TPCIU.getTargetProcessControl(); + constexpr auto TrampolinePagePermissions = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + auto PageSize = TPC.getPageSize(); + jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; + Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize), + 0}; + auto Alloc = TPC.getMemMgr().allocate(nullptr, Request); + + if (!Alloc) + return Alloc.takeError(); + + unsigned NumTrampolines = TrampolinesPerPage; + + auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions); + auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions); + + TPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress, + ResolverAddress, NumTrampolines); + + auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions); + for (unsigned I = 0; I < NumTrampolines; ++I) + AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize)); + + if (auto Err = (*Alloc)->finalize()) + return Err; + + TrampolineBlocks.push_back(std::move(*Alloc)); + + return Error::success(); +} + +Error TPCIndirectStubsManager::createStub(StringRef StubName, + JITTargetAddress StubAddr, + JITSymbolFlags StubFlags) { + StubInitsMap SIM; + SIM[StubName] = std::make_pair(StubAddr, StubFlags); + return createStubs(SIM); +} + +Error TPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { + auto AvailableStubInfos = getIndirectStubs(TPCIU, StubInits.size()); + if (!AvailableStubInfos) + return AvailableStubInfos.takeError(); + + { + std::lock_guard<std::mutex> Lock(ISMMutex); + unsigned ASIdx = 0; + for (auto &SI : StubInits) { + auto &A = (*AvailableStubInfos)[ASIdx++]; + StubInfos[SI.first()] = std::make_pair(A, SI.second.second); + } + } + + auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess(); + switch (TPCIU.getABISupport().getPointerSize()) { + case 4: { + unsigned ASIdx = 0; + std::vector<tpctypes::UInt32Write> PtrUpdates; + for (auto &SI : StubInits) + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast<uint32_t>(SI.second.first)}); + return MemAccess.writeUInt32s(PtrUpdates); + } + case 8: { + unsigned ASIdx = 0; + std::vector<tpctypes::UInt64Write> PtrUpdates; + for (auto &SI : StubInits) + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast<uint64_t>(SI.second.first)}); + return MemAccess.writeUInt64s(PtrUpdates); + } + default: + return make_error<StringError>("Unsupported pointer size", + inconvertibleErrorCode()); + } +} + +JITEvaluatedSymbol TPCIndirectStubsManager::findStub(StringRef Name, + bool ExportedStubsOnly) { + std::lock_guard<std::mutex> Lock(ISMMutex); + auto I = StubInfos.find(Name); + if (I == StubInfos.end()) + return nullptr; + return {I->second.first.StubAddress, I->second.second}; +} + +JITEvaluatedSymbol TPCIndirectStubsManager::findPointer(StringRef Name) { + std::lock_guard<std::mutex> Lock(ISMMutex); + auto I = StubInfos.find(Name); + if (I == StubInfos.end()) + return nullptr; + return {I->second.first.PointerAddress, I->second.second}; +} + +Error TPCIndirectStubsManager::updatePointer(StringRef Name, + JITTargetAddress NewAddr) { + + JITTargetAddress PtrAddr = 0; + { + std::lock_guard<std::mutex> Lock(ISMMutex); + auto I = StubInfos.find(Name); + if (I == StubInfos.end()) + return make_error<StringError>("Unknown stub name", + inconvertibleErrorCode()); + PtrAddr = I->second.first.PointerAddress; + } + + auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess(); + switch (TPCIU.getABISupport().getPointerSize()) { + case 4: { + tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr); + return MemAccess.writeUInt32s(PUpdate); + } + case 8: { + tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr); + return MemAccess.writeUInt64s(PUpdate); + } + default: + return make_error<StringError>("Unsupported pointer size", + inconvertibleErrorCode()); + } +} + +} // end anonymous namespace. + +namespace llvm { +namespace orc { + +TPCIndirectionUtils::ABISupport::~ABISupport() {} + +Expected<std::unique_ptr<TPCIndirectionUtils>> +TPCIndirectionUtils::Create(TargetProcessControl &TPC) { + const auto &TT = TPC.getTargetTriple(); + switch (TT.getArch()) { + default: + return make_error<StringError>( + std::string("No TPCIndirectionUtils available for ") + TT.str(), + inconvertibleErrorCode()); + case Triple::aarch64: + case Triple::aarch64_32: + return CreateWithABI<OrcAArch64>(TPC); + + case Triple::x86: + return CreateWithABI<OrcI386>(TPC); + + case Triple::mips: + return CreateWithABI<OrcMips32Be>(TPC); + + case Triple::mipsel: + return CreateWithABI<OrcMips32Le>(TPC); + + case Triple::mips64: + case Triple::mips64el: + return CreateWithABI<OrcMips64>(TPC); + + case Triple::x86_64: + if (TT.getOS() == Triple::OSType::Win32) + return CreateWithABI<OrcX86_64_Win32>(TPC); + else + return CreateWithABI<OrcX86_64_SysV>(TPC); + } +} + +Error TPCIndirectionUtils::cleanup() { + Error Err = Error::success(); + + for (auto &A : IndirectStubAllocs) + Err = joinErrors(std::move(Err), A->deallocate()); + + if (TP) + Err = joinErrors(std::move(Err), + static_cast<TPCTrampolinePool &>(*TP).deallocatePool()); + + if (ResolverBlock) + Err = joinErrors(std::move(Err), ResolverBlock->deallocate()); + + return Err; +} + +Expected<JITTargetAddress> +TPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + assert(ABI && "ABI can not be null"); + constexpr auto ResolverBlockPermissions = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + auto ResolverSize = ABI->getResolverCodeSize(); + + jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; + Request[ResolverBlockPermissions] = {TPC.getPageSize(), + static_cast<size_t>(ResolverSize), 0}; + auto Alloc = TPC.getMemMgr().allocate(nullptr, Request); + if (!Alloc) + return Alloc.takeError(); + + auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions); + ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions); + ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr, + ReentryCtxAddr); + + if (auto Err = (*Alloc)->finalize()) + return std::move(Err); + + ResolverBlock = std::move(*Alloc); + return ResolverBlockAddr; +} + +std::unique_ptr<IndirectStubsManager> +TPCIndirectionUtils::createIndirectStubsManager() { + return std::make_unique<TPCIndirectStubsManager>(*this); +} + +TrampolinePool &TPCIndirectionUtils::getTrampolinePool() { + if (!TP) + TP = std::make_unique<TPCTrampolinePool>(*this); + return *TP; +} + +LazyCallThroughManager &TPCIndirectionUtils::createLazyCallThroughManager( + ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { + assert(!LCTM && + "createLazyCallThroughManager can not have been called before"); + LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, + &getTrampolinePool()); + return *LCTM; +} + +TPCIndirectionUtils::TPCIndirectionUtils(TargetProcessControl &TPC, + std::unique_ptr<ABISupport> ABI) + : TPC(TPC), ABI(std::move(ABI)) { + assert(this->ABI && "ABI can not be null"); + + assert(TPC.getPageSize() > getABISupport().getStubSize() && + "Stubs larger than one page are not supported"); +} + +Expected<TPCIndirectionUtils::IndirectStubInfoVector> +TPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { + + std::lock_guard<std::mutex> Lock(TPCUIMutex); + + // If there aren't enough stubs available then allocate some more. + if (NumStubs > AvailableIndirectStubs.size()) { + auto NumStubsToAllocate = NumStubs; + auto PageSize = TPC.getPageSize(); + auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); + NumStubsToAllocate = StubBytes / ABI->getStubSize(); + auto PointerBytes = + alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); + + constexpr auto StubPagePermissions = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + constexpr auto PointerPagePermissions = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; + Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes), + 0}; + Request[PointerPagePermissions] = {PageSize, 0, PointerBytes}; + auto Alloc = TPC.getMemMgr().allocate(nullptr, Request); + if (!Alloc) + return Alloc.takeError(); + + auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions); + auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions); + + ABI->writeIndirectStubsBlock( + (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr, + PointerTargetAddr, NumStubsToAllocate); + + if (auto Err = (*Alloc)->finalize()) + return std::move(Err); + + for (unsigned I = 0; I != NumStubsToAllocate; ++I) { + AvailableIndirectStubs.push_back( + IndirectStubInfo(StubTargetAddr, PointerTargetAddr)); + StubTargetAddr += ABI->getStubSize(); + PointerTargetAddr += ABI->getPointerSize(); + } + + IndirectStubAllocs.push_back(std::move(*Alloc)); + } + + assert(NumStubs <= AvailableIndirectStubs.size() && + "Sufficient stubs should have been allocated above"); + + IndirectStubInfoVector Result; + while (NumStubs--) { + Result.push_back(AvailableIndirectStubs.back()); + AvailableIndirectStubs.pop_back(); + } + + return std::move(Result); +} + +static JITTargetAddress reentry(JITTargetAddress LCTMAddr, + JITTargetAddress TrampolineAddr) { + auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); + std::promise<JITTargetAddress> LandingAddrP; + auto LandingAddrF = LandingAddrP.get_future(); + LCTM.resolveTrampolineLandingAddress( + TrampolineAddr, + [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); + return LandingAddrF.get(); +} + +Error setUpInProcessLCTMReentryViaTPCIU(TPCIndirectionUtils &TPCIU) { + auto &LCTM = TPCIU.getLazyCallThroughManager(); + return TPCIU + .writeResolverBlock(pointerToJITTargetAddress(&reentry), + pointerToJITTargetAddress(&LCTM)) + .takeError(); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp new file mode 100644 index 000000000000..aff7296cb6e3 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp @@ -0,0 +1,208 @@ +//===--------- RegisterEHFrames.cpp - Register EH frame sections ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" + +#include "llvm/Config/config.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/Compiler.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/DynamicLibrary.h" +#include "llvm/Support/raw_ostream.h" + +#include "llvm/Support/FormatVariadic.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::tpctypes; + +namespace llvm { +namespace orc { + +#if defined(HAVE_REGISTER_FRAME) && defined(HAVE_DEREGISTER_FRAME) && \ + !defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__) + +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +Error registerFrameWrapper(const void *P) { + __register_frame(P); + return Error::success(); +} + +Error deregisterFrameWrapper(const void *P) { + __deregister_frame(P); + return Error::success(); +} + +#else + +// The building compiler does not have __(de)register_frame but +// it may be found at runtime in a dynamically-loaded library. +// For example, this happens when building LLVM with Visual C++ +// but using the MingW runtime. +static Error registerFrameWrapper(const void *P) { + static void((*RegisterFrame)(const void *)) = 0; + + if (!RegisterFrame) + *(void **)&RegisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("__register_frame"); + + if (RegisterFrame) { + RegisterFrame(P); + return Error::success(); + } + + return make_error<StringError>("could not register eh-frame: " + "__register_frame function not found", + inconvertibleErrorCode()); +} + +static Error deregisterFrameWrapper(const void *P) { + static void((*DeregisterFrame)(const void *)) = 0; + + if (!DeregisterFrame) + *(void **)&DeregisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol( + "__deregister_frame"); + + if (DeregisterFrame) { + DeregisterFrame(P); + return Error::success(); + } + + return make_error<StringError>("could not deregister eh-frame: " + "__deregister_frame function not found", + inconvertibleErrorCode()); +} +#endif + +#ifdef __APPLE__ + +template <typename HandleFDEFn> +Error walkAppleEHFrameSection(const char *const SectionStart, + size_t SectionSize, HandleFDEFn HandleFDE) { + const char *CurCFIRecord = SectionStart; + const char *End = SectionStart + SectionSize; + uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + + while (CurCFIRecord != End && Size != 0) { + const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); + if (Size == 0xffffffff) + Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12; + else + Size += 4; + uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); + + LLVM_DEBUG({ + dbgs() << "Registering eh-frame section:\n"; + dbgs() << "Processing " << (Offset ? "FDE" : "CIE") << " @" + << (void *)CurCFIRecord << ": ["; + for (unsigned I = 0; I < Size; ++I) + dbgs() << format(" 0x%02" PRIx8, *(CurCFIRecord + I)); + dbgs() << " ]\n"; + }); + + if (Offset != 0) + if (auto Err = HandleFDE(CurCFIRecord)) + return Err; + + CurCFIRecord += Size; + + Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + } + + return Error::success(); +} + +#endif // __APPLE__ + +Error registerEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize) { +#ifdef __APPLE__ + // On Darwin __register_frame has to be called for each FDE entry. + return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + EHFrameSectionSize, registerFrameWrapper); +#else + // On Linux __register_frame takes a single argument: + // a pointer to the start of the .eh_frame section. + + // How can it find the end? Because crtendS.o is linked + // in and it has an .eh_frame section with four zero chars. + return registerFrameWrapper(EHFrameSectionAddr); +#endif +} + +Error deregisterEHFrameSection(const void *EHFrameSectionAddr, + size_t EHFrameSectionSize) { +#ifdef __APPLE__ + return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + EHFrameSectionSize, deregisterFrameWrapper); +#else + return deregisterFrameWrapper(EHFrameSectionAddr); +#endif +} + +} // end namespace orc +} // end namespace llvm + +extern "C" CWrapperFunctionResult +llvm_orc_registerEHFrameSectionWrapper(uint8_t *Data, uint64_t Size) { + if (Size != sizeof(uint64_t) + sizeof(uint64_t)) + return WrapperFunctionResult::from( + "Invalid arguments to llvm_orc_registerEHFrameSectionWrapper") + .release(); + + uint64_t EHFrameSectionAddr; + uint64_t EHFrameSectionSize; + + { + BinaryStreamReader ArgReader(ArrayRef<uint8_t>(Data, Size), + support::endianness::big); + cantFail(ArgReader.readInteger(EHFrameSectionAddr)); + cantFail(ArgReader.readInteger(EHFrameSectionSize)); + } + + if (auto Err = registerEHFrameSection( + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize)) { + auto ErrMsg = toString(std::move(Err)); + return WrapperFunctionResult::from(ErrMsg).release(); + } + return WrapperFunctionResult().release(); +} + +extern "C" CWrapperFunctionResult +llvm_orc_deregisterEHFrameSectionWrapper(uint8_t *Data, uint64_t Size) { + if (Size != sizeof(uint64_t) + sizeof(uint64_t)) + return WrapperFunctionResult::from( + "Invalid arguments to llvm_orc_registerEHFrameSectionWrapper") + .release(); + + uint64_t EHFrameSectionAddr; + uint64_t EHFrameSectionSize; + + { + BinaryStreamReader ArgReader(ArrayRef<uint8_t>(Data, Size), + support::endianness::big); + cantFail(ArgReader.readInteger(EHFrameSectionAddr)); + cantFail(ArgReader.readInteger(EHFrameSectionSize)); + } + + if (auto Err = deregisterEHFrameSection( + jitTargetAddressToPointer<void *>(EHFrameSectionAddr), + EHFrameSectionSize)) { + auto ErrMsg = toString(std::move(Err)); + return WrapperFunctionResult::from(ErrMsg).release(); + } + return WrapperFunctionResult().release(); +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp new file mode 100644 index 000000000000..a8e6c049cf4b --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp @@ -0,0 +1,43 @@ +//===--- TargetExecutionUtils.cpp - Execution utils for target processes --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" + +#include <vector> + +namespace llvm { +namespace orc { + +int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args, + Optional<StringRef> ProgramName) { + std::vector<std::unique_ptr<char[]>> ArgVStorage; + std::vector<char *> ArgV; + + ArgVStorage.reserve(Args.size() + (ProgramName ? 1 : 0)); + ArgV.reserve(Args.size() + 1 + (ProgramName ? 1 : 0)); + + if (ProgramName) { + ArgVStorage.push_back(std::make_unique<char[]>(ProgramName->size() + 1)); + llvm::copy(*ProgramName, &ArgVStorage.back()[0]); + ArgVStorage.back()[ProgramName->size()] = '\0'; + ArgV.push_back(ArgVStorage.back().get()); + } + + for (const auto &Arg : Args) { + ArgVStorage.push_back(std::make_unique<char[]>(Arg.size() + 1)); + llvm::copy(Arg, &ArgVStorage.back()[0]); + ArgVStorage.back()[Arg.size()] = '\0'; + ArgV.push_back(ArgVStorage.back().get()); + } + ArgV.push_back(nullptr); + + return Main(Args.size() + !!ProgramName, ArgV.data()); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp new file mode 100644 index 000000000000..7bf874e88c26 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp @@ -0,0 +1,153 @@ +//===------ TargetProcessControl.cpp -- Target process control APIs -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/Process.h" + +#include <mutex> + +namespace llvm { +namespace orc { + +TargetProcessControl::MemoryAccess::~MemoryAccess() {} + +TargetProcessControl::~TargetProcessControl() {} + +SelfTargetProcessControl::SelfTargetProcessControl( + std::shared_ptr<SymbolStringPool> SSP, Triple TargetTriple, + unsigned PageSize, std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) + : TargetProcessControl(std::move(SSP)) { + + OwnedMemMgr = std::move(MemMgr); + if (!OwnedMemMgr) + OwnedMemMgr = std::make_unique<jitlink::InProcessMemoryManager>(); + + this->TargetTriple = std::move(TargetTriple); + this->PageSize = PageSize; + this->MemMgr = OwnedMemMgr.get(); + this->MemAccess = this; + if (this->TargetTriple.isOSBinFormatMachO()) + GlobalManglingPrefix = '_'; +} + +Expected<std::unique_ptr<SelfTargetProcessControl>> +SelfTargetProcessControl::Create( + std::shared_ptr<SymbolStringPool> SSP, + std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) { + auto PageSize = sys::Process::getPageSize(); + if (!PageSize) + return PageSize.takeError(); + + Triple TT(sys::getProcessTriple()); + + return std::make_unique<SelfTargetProcessControl>( + std::move(SSP), std::move(TT), *PageSize, std::move(MemMgr)); +} + +Expected<tpctypes::DylibHandle> +SelfTargetProcessControl::loadDylib(const char *DylibPath) { + std::string ErrMsg; + auto Dylib = std::make_unique<sys::DynamicLibrary>( + sys::DynamicLibrary::getPermanentLibrary(DylibPath, &ErrMsg)); + if (!Dylib->isValid()) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + DynamicLibraries.push_back(std::move(Dylib)); + return pointerToJITTargetAddress(DynamicLibraries.back().get()); +} + +Expected<std::vector<tpctypes::LookupResult>> +SelfTargetProcessControl::lookupSymbols(ArrayRef<LookupRequest> Request) { + std::vector<tpctypes::LookupResult> R; + + for (auto &Elem : Request) { + auto *Dylib = jitTargetAddressToPointer<sys::DynamicLibrary *>(Elem.Handle); + assert(llvm::any_of(DynamicLibraries, + [=](const std::unique_ptr<sys::DynamicLibrary> &DL) { + return DL.get() == Dylib; + }) && + "Invalid handle"); + + R.push_back(std::vector<JITTargetAddress>()); + for (auto &KV : Elem.Symbols) { + auto &Sym = KV.first; + std::string Tmp((*Sym).data() + !!GlobalManglingPrefix, + (*Sym).size() - !!GlobalManglingPrefix); + void *Addr = Dylib->getAddressOfSymbol(Tmp.c_str()); + if (!Addr && KV.second == SymbolLookupFlags::RequiredSymbol) { + // FIXME: Collect all failing symbols before erroring out. + SymbolNameVector MissingSymbols; + MissingSymbols.push_back(Sym); + return make_error<SymbolsNotFound>(std::move(MissingSymbols)); + } + R.back().push_back(pointerToJITTargetAddress(Addr)); + } + } + + return R; +} + +Expected<int32_t> +SelfTargetProcessControl::runAsMain(JITTargetAddress MainFnAddr, + ArrayRef<std::string> Args) { + using MainTy = int (*)(int, char *[]); + return orc::runAsMain(jitTargetAddressToFunction<MainTy>(MainFnAddr), Args); +} + +Expected<tpctypes::WrapperFunctionResult> +SelfTargetProcessControl::runWrapper(JITTargetAddress WrapperFnAddr, + ArrayRef<uint8_t> ArgBuffer) { + using WrapperFnTy = + tpctypes::CWrapperFunctionResult (*)(const uint8_t *Data, uint64_t Size); + auto *WrapperFn = jitTargetAddressToFunction<WrapperFnTy>(WrapperFnAddr); + return WrapperFn(ArgBuffer.data(), ArgBuffer.size()); +} + +Error SelfTargetProcessControl::disconnect() { return Error::success(); } + +void SelfTargetProcessControl::writeUInt8s(ArrayRef<tpctypes::UInt8Write> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer<uint8_t *>(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeUInt16s(ArrayRef<tpctypes::UInt16Write> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer<uint16_t *>(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeUInt32s(ArrayRef<tpctypes::UInt32Write> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer<uint32_t *>(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeUInt64s(ArrayRef<tpctypes::UInt64Write> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + *jitTargetAddressToPointer<uint64_t *>(W.Address) = W.Value; + OnWriteComplete(Error::success()); +} + +void SelfTargetProcessControl::writeBuffers(ArrayRef<tpctypes::BufferWrite> Ws, + WriteResultFn OnWriteComplete) { + for (auto &W : Ws) + memcpy(jitTargetAddressToPointer<char *>(W.Address), W.Buffer.data(), + W.Buffer.size()); + OnWriteComplete(Error::success()); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp index 1f4e6f132115..2e128dd23744 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp @@ -15,7 +15,7 @@ namespace llvm { namespace orc { -ThreadSafeModule cloneToNewContext(ThreadSafeModule &TSM, +ThreadSafeModule cloneToNewContext(const ThreadSafeModule &TSM, GVPredicate ShouldCloneDef, GVModifier UpdateClonedDefSource) { assert(TSM && "Can not clone null module"); |