diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine/Orc/Core.cpp')
| -rw-r--r-- | llvm/lib/ExecutionEngine/Orc/Core.cpp | 2494 |
1 files changed, 1580 insertions, 914 deletions
diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index bad13cfebbc6..dfb558808c32 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -11,18 +11,19 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/Config/llvm-config.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" -#include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/MSVCErrorWorkarounds.h" #include <condition_variable> -#if LLVM_ENABLE_THREADS #include <future> -#endif #define DEBUG_TYPE "orc" namespace llvm { namespace orc { +char ResourceTrackerDefunct::ID = 0; char FailedToMaterialize::ID = 0; char SymbolsNotFound::ID = 0; char SymbolsCouldNotBeRemoved::ID = 0; @@ -34,6 +35,45 @@ RegisterDependenciesFunction NoDependenciesToRegister = void MaterializationUnit::anchor() {} +ResourceTracker::ResourceTracker(JITDylibSP JD) { + assert((reinterpret_cast<uintptr_t>(JD.get()) & 0x1) == 0 && + "JITDylib must be two byte aligned"); + JD->Retain(); + JDAndFlag.store(reinterpret_cast<uintptr_t>(JD.get())); +} + +ResourceTracker::~ResourceTracker() { + getJITDylib().getExecutionSession().destroyResourceTracker(*this); + getJITDylib().Release(); +} + +Error ResourceTracker::remove() { + return getJITDylib().getExecutionSession().removeResourceTracker(*this); +} + +void ResourceTracker::transferTo(ResourceTracker &DstRT) { + getJITDylib().getExecutionSession().transferResourceTracker(DstRT, *this); +} + +void ResourceTracker::makeDefunct() { + uintptr_t Val = JDAndFlag.load(); + Val |= 0x1U; + JDAndFlag.store(Val); +} + +ResourceManager::~ResourceManager() {} + +ResourceTrackerDefunct::ResourceTrackerDefunct(ResourceTrackerSP RT) + : RT(std::move(RT)) {} + +std::error_code ResourceTrackerDefunct::convertToErrorCode() const { + return orcError(OrcErrorCode::UnknownORCError); +} + +void ResourceTrackerDefunct::log(raw_ostream &OS) const { + OS << "Resource tracker " << (void *)RT.get() << " became defunct"; +} + FailedToMaterialize::FailedToMaterialize( std::shared_ptr<SymbolDependenceMap> Symbols) : Symbols(std::move(Symbols)) { @@ -137,8 +177,6 @@ void AsynchronousSymbolQuery::handleComplete() { TmpNotifyComplete(std::move(ResolvedSymbols)); } -bool AsynchronousSymbolQuery::canStillFail() { return !!NotifyComplete; } - void AsynchronousSymbolQuery::handleFailed(Error Err) { assert(QueryRegistrations.empty() && ResolvedSymbols.empty() && OutstandingSymbolsCount == 0 && @@ -181,156 +219,9 @@ void AsynchronousSymbolQuery::detach() { QueryRegistrations.clear(); } -MaterializationResponsibility::~MaterializationResponsibility() { - assert(SymbolFlags.empty() && - "All symbols should have been explicitly materialized or failed"); -} - -SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { - return JD->getRequestedSymbols(SymbolFlags); -} - -Error MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { - LLVM_DEBUG({ - dbgs() << "In " << JD->getName() << " resolving " << Symbols << "\n"; - }); -#ifndef NDEBUG - for (auto &KV : Symbols) { - auto WeakFlags = JITSymbolFlags::Weak | JITSymbolFlags::Common; - auto I = SymbolFlags.find(KV.first); - assert(I != SymbolFlags.end() && - "Resolving symbol outside this responsibility set"); - assert(!I->second.hasMaterializationSideEffectsOnly() && - "Can't resolve materialization-side-effects-only symbol"); - assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) && - "Resolving symbol with incorrect flags"); - } -#endif - - return JD->resolve(Symbols); -} - -Error MaterializationResponsibility::notifyEmitted() { - - LLVM_DEBUG({ - dbgs() << "In " << JD->getName() << " emitting " << SymbolFlags << "\n"; - }); - - if (auto Err = JD->emit(SymbolFlags)) - return Err; - - SymbolFlags.clear(); - return Error::success(); -} - -Error MaterializationResponsibility::defineMaterializing( - SymbolFlagsMap NewSymbolFlags) { - - LLVM_DEBUG({ - dbgs() << "In " << JD->getName() << " defining materializing symbols " - << NewSymbolFlags << "\n"; - }); - if (auto AcceptedDefs = JD->defineMaterializing(std::move(NewSymbolFlags))) { - // Add all newly accepted symbols to this responsibility object. - for (auto &KV : *AcceptedDefs) - SymbolFlags.insert(KV); - return Error::success(); - } else - return AcceptedDefs.takeError(); -} - -void MaterializationResponsibility::failMaterialization() { - - LLVM_DEBUG({ - dbgs() << "In " << JD->getName() << " failing materialization for " - << SymbolFlags << "\n"; - }); - - JITDylib::FailedSymbolsWorklist Worklist; - - for (auto &KV : SymbolFlags) - Worklist.push_back(std::make_pair(JD.get(), KV.first)); - SymbolFlags.clear(); - - JD->notifyFailed(std::move(Worklist)); -} - -void MaterializationResponsibility::replace( - std::unique_ptr<MaterializationUnit> MU) { - - // If the replacement MU is empty then return. - if (MU->getSymbols().empty()) - return; - - for (auto &KV : MU->getSymbols()) { - assert(SymbolFlags.count(KV.first) && - "Replacing definition outside this responsibility set"); - SymbolFlags.erase(KV.first); - } - - if (MU->getInitializerSymbol() == InitSymbol) - InitSymbol = nullptr; - - LLVM_DEBUG(JD->getExecutionSession().runSessionLocked([&]() { - dbgs() << "In " << JD->getName() << " replacing symbols with " << *MU - << "\n"; - });); - - JD->replace(std::move(MU)); -} - -MaterializationResponsibility -MaterializationResponsibility::delegate(const SymbolNameSet &Symbols, - VModuleKey NewKey) { - - if (NewKey == VModuleKey()) - NewKey = K; - - SymbolStringPtr DelegatedInitSymbol; - SymbolFlagsMap DelegatedFlags; - - for (auto &Name : Symbols) { - auto I = SymbolFlags.find(Name); - assert(I != SymbolFlags.end() && - "Symbol is not tracked by this MaterializationResponsibility " - "instance"); - - DelegatedFlags[Name] = std::move(I->second); - if (Name == InitSymbol) - std::swap(InitSymbol, DelegatedInitSymbol); - - SymbolFlags.erase(I); - } - - return MaterializationResponsibility(JD, std::move(DelegatedFlags), - std::move(DelegatedInitSymbol), - std::move(NewKey)); -} - -void MaterializationResponsibility::addDependencies( - const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { - LLVM_DEBUG({ - dbgs() << "Adding dependencies for " << Name << ": " << Dependencies - << "\n"; - }); - assert(SymbolFlags.count(Name) && - "Symbol not covered by this MaterializationResponsibility instance"); - JD->addDependencies(Name, Dependencies); -} - -void MaterializationResponsibility::addDependenciesForAll( - const SymbolDependenceMap &Dependencies) { - LLVM_DEBUG({ - dbgs() << "Adding dependencies for all symbols in " << SymbolFlags << ": " - << Dependencies << "\n"; - }); - for (auto &KV : SymbolFlags) - JD->addDependencies(KV.first, Dependencies); -} - AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( - SymbolMap Symbols, VModuleKey K) - : MaterializationUnit(extractFlags(Symbols), nullptr, std::move(K)), + SymbolMap Symbols) + : MaterializationUnit(extractFlags(Symbols), nullptr), Symbols(std::move(Symbols)) {} StringRef AbsoluteSymbolsMaterializationUnit::getName() const { @@ -338,10 +229,10 @@ StringRef AbsoluteSymbolsMaterializationUnit::getName() const { } void AbsoluteSymbolsMaterializationUnit::materialize( - MaterializationResponsibility R) { + std::unique_ptr<MaterializationResponsibility> R) { // No dependencies, so these calls can't fail. - cantFail(R.notifyResolved(Symbols)); - cantFail(R.notifyEmitted()); + cantFail(R->notifyResolved(Symbols)); + cantFail(R->notifyEmitted()); } void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD, @@ -360,26 +251,25 @@ AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { ReExportsMaterializationUnit::ReExportsMaterializationUnit( JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, - SymbolAliasMap Aliases, VModuleKey K) - : MaterializationUnit(extractFlags(Aliases), nullptr, std::move(K)), - SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), - Aliases(std::move(Aliases)) {} + SymbolAliasMap Aliases) + : MaterializationUnit(extractFlags(Aliases), nullptr), SourceJD(SourceJD), + SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {} StringRef ReExportsMaterializationUnit::getName() const { return "<Reexports>"; } void ReExportsMaterializationUnit::materialize( - MaterializationResponsibility R) { + std::unique_ptr<MaterializationResponsibility> R) { - auto &ES = R.getTargetJITDylib().getExecutionSession(); - JITDylib &TgtJD = R.getTargetJITDylib(); + auto &ES = R->getTargetJITDylib().getExecutionSession(); + JITDylib &TgtJD = R->getTargetJITDylib(); JITDylib &SrcJD = SourceJD ? *SourceJD : TgtJD; // Find the set of requested aliases and aliasees. Return any unrequested // aliases back to the JITDylib so as to not prematurely materialize any // aliasees. - auto RequestedSymbols = R.getRequestedSymbols(); + auto RequestedSymbols = R->getRequestedSymbols(); SymbolAliasMap RequestedAliases; for (auto &Name : RequestedSymbols) { @@ -398,19 +288,27 @@ void ReExportsMaterializationUnit::materialize( }); if (!Aliases.empty()) { - if (SourceJD) - R.replace(reexports(*SourceJD, std::move(Aliases), SourceJDLookupFlags)); - else - R.replace(symbolAliases(std::move(Aliases))); + auto Err = SourceJD ? R->replace(reexports(*SourceJD, std::move(Aliases), + SourceJDLookupFlags)) + : R->replace(symbolAliases(std::move(Aliases))); + + if (Err) { + // FIXME: Should this be reported / treated as failure to materialize? + // Or should this be treated as a sanctioned bailing-out? + ES.reportError(std::move(Err)); + R->failMaterialization(); + return; + } } // The OnResolveInfo struct will hold the aliases and responsibilty for each // query in the list. struct OnResolveInfo { - OnResolveInfo(MaterializationResponsibility R, SymbolAliasMap Aliases) + OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R, + SymbolAliasMap Aliases) : R(std::move(R)), Aliases(std::move(Aliases)) {} - MaterializationResponsibility R; + std::unique_ptr<MaterializationResponsibility> R; SymbolAliasMap Aliases; }; @@ -450,8 +348,15 @@ void ReExportsMaterializationUnit::materialize( assert(!QuerySymbols.empty() && "Alias cycle detected!"); - auto QueryInfo = std::make_shared<OnResolveInfo>( - R.delegate(ResponsibilitySymbols), std::move(QueryAliases)); + auto NewR = R->delegate(ResponsibilitySymbols); + if (!NewR) { + ES.reportError(NewR.takeError()); + R->failMaterialization(); + return; + } + + auto QueryInfo = std::make_shared<OnResolveInfo>(std::move(*NewR), + std::move(QueryAliases)); QueryInfos.push_back( make_pair(std::move(QuerySymbols), std::move(QueryInfo))); } @@ -480,12 +385,12 @@ void ReExportsMaterializationUnit::materialize( for (auto &KV : QueryInfo->Aliases) if (SrcJDDeps.count(KV.second.Aliasee)) { PerAliasDeps = {KV.second.Aliasee}; - QueryInfo->R.addDependencies(KV.first, PerAliasDepsMap); + QueryInfo->R->addDependencies(KV.first, PerAliasDepsMap); } }; auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) { - auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession(); + auto &ES = QueryInfo->R->getTargetJITDylib().getExecutionSession(); if (Result) { SymbolMap ResolutionMap; for (auto &KV : QueryInfo->Aliases) { @@ -499,19 +404,19 @@ void ReExportsMaterializationUnit::materialize( ResolutionMap[KV.first] = JITEvaluatedSymbol( (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); } - if (auto Err = QueryInfo->R.notifyResolved(ResolutionMap)) { + if (auto Err = QueryInfo->R->notifyResolved(ResolutionMap)) { ES.reportError(std::move(Err)); - QueryInfo->R.failMaterialization(); + QueryInfo->R->failMaterialization(); return; } - if (auto Err = QueryInfo->R.notifyEmitted()) { + if (auto Err = QueryInfo->R->notifyEmitted()) { ES.reportError(std::move(Err)); - QueryInfo->R.failMaterialization(); + QueryInfo->R->failMaterialization(); return; } } else { ES.reportError(Result.takeError()); - QueryInfo->R.failMaterialization(); + QueryInfo->R->failMaterialization(); } }; @@ -538,20 +443,16 @@ ReExportsMaterializationUnit::extractFlags(const SymbolAliasMap &Aliases) { return SymbolFlags; } -Expected<SymbolAliasMap> -buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { +Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD, + SymbolNameSet Symbols) { SymbolLookupSet LookupSet(Symbols); - auto Flags = SourceJD.lookupFlags( - LookupKind::Static, JITDylibLookupFlags::MatchAllSymbols, LookupSet); + auto Flags = SourceJD.getExecutionSession().lookupFlags( + LookupKind::Static, {{&SourceJD, JITDylibLookupFlags::MatchAllSymbols}}, + SymbolLookupSet(std::move(Symbols))); if (!Flags) return Flags.takeError(); - if (!LookupSet.empty()) { - LookupSet.sortByName(); - return make_error<SymbolsNotFound>(LookupSet.getSymbolNames()); - } - SymbolAliasMap Result; for (auto &Name : Symbols) { assert(Flags->count(Name) && "Missing entry in flags map"); @@ -561,19 +462,100 @@ buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { return Result; } +class InProgressLookupState { +public: + InProgressLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet LookupSet, SymbolState RequiredState) + : K(K), SearchOrder(std::move(SearchOrder)), + LookupSet(std::move(LookupSet)), RequiredState(RequiredState) { + DefGeneratorCandidates = this->LookupSet; + } + virtual ~InProgressLookupState() {} + virtual void complete(std::unique_ptr<InProgressLookupState> IPLS) = 0; + virtual void fail(Error Err) = 0; + + LookupKind K; + JITDylibSearchOrder SearchOrder; + SymbolLookupSet LookupSet; + SymbolState RequiredState; + + std::unique_lock<std::mutex> GeneratorLock; + size_t CurSearchOrderIndex = 0; + bool NewJITDylib = true; + SymbolLookupSet DefGeneratorCandidates; + SymbolLookupSet DefGeneratorNonCandidates; + std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack; +}; + +class InProgressLookupFlagsState : public InProgressLookupState { +public: + InProgressLookupFlagsState( + LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) + : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet), + SymbolState::NeverSearched), + OnComplete(std::move(OnComplete)) {} + + void complete(std::unique_ptr<InProgressLookupState> IPLS) override { + GeneratorLock = {}; // Unlock and release. + auto &ES = SearchOrder.front().first->getExecutionSession(); + ES.OL_completeLookupFlags(std::move(IPLS), std::move(OnComplete)); + } + + void fail(Error Err) override { + GeneratorLock = {}; // Unlock and release. + OnComplete(std::move(Err)); + } + +private: + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete; +}; + +class InProgressFullLookupState : public InProgressLookupState { +public: + InProgressFullLookupState(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet LookupSet, + SymbolState RequiredState, + std::shared_ptr<AsynchronousSymbolQuery> Q, + RegisterDependenciesFunction RegisterDependencies) + : InProgressLookupState(K, std::move(SearchOrder), std::move(LookupSet), + RequiredState), + Q(std::move(Q)), RegisterDependencies(std::move(RegisterDependencies)) { + } + + void complete(std::unique_ptr<InProgressLookupState> IPLS) override { + GeneratorLock = {}; // Unlock and release. + auto &ES = SearchOrder.front().first->getExecutionSession(); + ES.OL_completeLookup(std::move(IPLS), std::move(Q), + std::move(RegisterDependencies)); + } + + void fail(Error Err) override { + GeneratorLock = {}; + Q->detach(); + Q->handleFailed(std::move(Err)); + } + +private: + std::shared_ptr<AsynchronousSymbolQuery> Q; + RegisterDependenciesFunction RegisterDependencies; +}; + ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolPredicate Allow) : SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), Allow(std::move(Allow)) {} -Error ReexportsGenerator::tryToGenerate(LookupKind K, JITDylib &JD, +Error ReexportsGenerator::tryToGenerate(LookupState &LS, LookupKind K, + JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &LookupSet) { assert(&JD != &SourceJD && "Cannot re-export from the same dylib"); // Use lookupFlags to find the subset of symbols that match our lookup. - auto Flags = SourceJD.lookupFlags(K, JDLookupFlags, LookupSet); + auto Flags = JD.getExecutionSession().lookupFlags( + K, {{&SourceJD, JDLookupFlags}}, LookupSet); if (!Flags) return Flags.takeError(); @@ -590,19 +572,63 @@ Error ReexportsGenerator::tryToGenerate(LookupKind K, JITDylib &JD, return JD.define(reexports(SourceJD, AliasMap, SourceJDLookupFlags)); } -JITDylib::DefinitionGenerator::~DefinitionGenerator() {} +LookupState::LookupState(std::unique_ptr<InProgressLookupState> IPLS) + : IPLS(std::move(IPLS)) {} -void JITDylib::removeGenerator(DefinitionGenerator &G) { +void LookupState::reset(InProgressLookupState *IPLS) { this->IPLS.reset(IPLS); } + +LookupState::LookupState() = default; +LookupState::LookupState(LookupState &&) = default; +LookupState &LookupState::operator=(LookupState &&) = default; +LookupState::~LookupState() = default; + +void LookupState::continueLookup(Error Err) { + assert(IPLS && "Cannot call continueLookup on empty LookupState"); + auto &ES = IPLS->SearchOrder.begin()->first->getExecutionSession(); + ES.OL_applyQueryPhase1(std::move(IPLS), std::move(Err)); +} + +DefinitionGenerator::~DefinitionGenerator() {} + +Error JITDylib::clear() { + std::vector<ResourceTrackerSP> TrackersToRemove; ES.runSessionLocked([&]() { - auto I = std::find_if(DefGenerators.begin(), DefGenerators.end(), - [&](const std::unique_ptr<DefinitionGenerator> &H) { - return H.get() == &G; - }); - assert(I != DefGenerators.end() && "Generator not found"); - DefGenerators.erase(I); + for (auto &KV : TrackerSymbols) + TrackersToRemove.push_back(KV.first); + TrackersToRemove.push_back(getDefaultResourceTracker()); + }); + + Error Err = Error::success(); + for (auto &RT : TrackersToRemove) + Err = joinErrors(std::move(Err), RT->remove()); + return Err; +} + +ResourceTrackerSP JITDylib::getDefaultResourceTracker() { + return ES.runSessionLocked([this] { + if (!DefaultTracker) + DefaultTracker = new ResourceTracker(this); + return DefaultTracker; + }); +} + +ResourceTrackerSP JITDylib::createResourceTracker() { + return ES.runSessionLocked([this] { + ResourceTrackerSP RT = new ResourceTracker(this); + return RT; }); } +void JITDylib::removeGenerator(DefinitionGenerator &G) { + std::lock_guard<std::mutex> Lock(GeneratorsMutex); + auto I = llvm::find_if(DefGenerators, + [&](const std::shared_ptr<DefinitionGenerator> &H) { + return H.get() == &G; + }); + assert(I != DefGenerators.end() && "Generator not found"); + DefGenerators.erase(I); +} + Expected<SymbolFlagsMap> JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { @@ -652,11 +678,18 @@ JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { }); } -void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { +Error JITDylib::replace(MaterializationResponsibility &FromMR, + std::unique_ptr<MaterializationUnit> MU) { assert(MU != nullptr && "Can not replace with a null MaterializationUnit"); + std::unique_ptr<MaterializationUnit> MustRunMU; + std::unique_ptr<MaterializationResponsibility> MustRunMR; - auto MustRunMU = - ES.runSessionLocked([&, this]() -> std::unique_ptr<MaterializationUnit> { + auto Err = + ES.runSessionLocked([&, this]() -> Error { + auto RT = getTracker(FromMR); + + if (RT->isDefunct()) + return make_error<ResourceTrackerDefunct>(std::move(RT)); #ifndef NDEBUG for (auto &KV : MU->getSymbols()) { @@ -671,18 +704,27 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { } #endif // NDEBUG + // If the tracker is defunct we need to bail out immediately. + // If any symbol has pending queries against it then we need to // materialize MU immediately. for (auto &KV : MU->getSymbols()) { auto MII = MaterializingInfos.find(KV.first); if (MII != MaterializingInfos.end()) { - if (MII->second.hasQueriesPending()) - return std::move(MU); + if (MII->second.hasQueriesPending()) { + MustRunMR = ES.createMaterializationResponsibility( + *RT, std::move(MU->SymbolFlags), std::move(MU->InitSymbol)); + MustRunMU = std::move(MU); + return Error::success(); + } } } // Otherwise, make MU responsible for all the symbols. - auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); + auto RTI = MRTrackers.find(&FromMR); + assert(RTI != MRTrackers.end() && "No tracker for FromMR"); + auto UMI = + std::make_shared<UnmaterializedInfo>(std::move(MU), RTI->second); for (auto &KV : UMI->MU->getSymbols()) { auto SymI = Symbols.find(KV.first); assert(SymI->second.getState() == SymbolState::Materializing && @@ -700,14 +742,36 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { UMIEntry = UMI; } - return nullptr; + return Error::success(); }); + if (Err) + return Err; + if (MustRunMU) { - auto MR = - MustRunMU->createMaterializationResponsibility(shared_from_this()); - ES.dispatchMaterialization(std::move(MustRunMU), std::move(MR)); + assert(MustRunMR && "MustRunMU set implies MustRunMR set"); + ES.dispatchMaterialization(std::move(MustRunMU), std::move(MustRunMR)); + } else { + assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset"); } + + return Error::success(); +} + +Expected<std::unique_ptr<MaterializationResponsibility>> +JITDylib::delegate(MaterializationResponsibility &FromMR, + SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) { + + return ES.runSessionLocked( + [&]() -> Expected<std::unique_ptr<MaterializationResponsibility>> { + auto RT = getTracker(FromMR); + + if (RT->isDefunct()) + return make_error<ResourceTrackerDefunct>(std::move(RT)); + + return ES.createMaterializationResponsibility( + *RT, std::move(SymbolFlags), std::move(InitSymbol)); + }); } SymbolNameSet @@ -808,89 +872,93 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, Symbols[Name].setFlags(Symbols[Name].getFlags() | JITSymbolFlags::HasError); } -Error JITDylib::resolve(const SymbolMap &Resolved) { - SymbolNameSet SymbolsInErrorState; +Error JITDylib::resolve(MaterializationResponsibility &MR, + const SymbolMap &Resolved) { AsynchronousSymbolQuerySet CompletedQueries; - ES.runSessionLocked([&, this]() { - struct WorklistEntry { - SymbolTable::iterator SymI; - JITEvaluatedSymbol ResolvedSym; - }; + if (auto Err = ES.runSessionLocked([&, this]() -> Error { + auto RTI = MRTrackers.find(&MR); + assert(RTI != MRTrackers.end() && "No resource tracker for MR?"); + if (RTI->second->isDefunct()) + return make_error<ResourceTrackerDefunct>(RTI->second); - std::vector<WorklistEntry> Worklist; - Worklist.reserve(Resolved.size()); + struct WorklistEntry { + SymbolTable::iterator SymI; + JITEvaluatedSymbol ResolvedSym; + }; - // Build worklist and check for any symbols in the error state. - for (const auto &KV : Resolved) { + SymbolNameSet SymbolsInErrorState; + std::vector<WorklistEntry> Worklist; + Worklist.reserve(Resolved.size()); - assert(!KV.second.getFlags().hasError() && - "Resolution result can not have error flag set"); + // Build worklist and check for any symbols in the error state. + for (const auto &KV : Resolved) { - auto SymI = Symbols.find(KV.first); + assert(!KV.second.getFlags().hasError() && + "Resolution result can not have error flag set"); - assert(SymI != Symbols.end() && "Symbol not found"); - assert(!SymI->second.hasMaterializerAttached() && - "Resolving symbol with materializer attached?"); - assert(SymI->second.getState() == SymbolState::Materializing && - "Symbol should be materializing"); - assert(SymI->second.getAddress() == 0 && - "Symbol has already been resolved"); + auto SymI = Symbols.find(KV.first); - if (SymI->second.getFlags().hasError()) - SymbolsInErrorState.insert(KV.first); - else { - auto Flags = KV.second.getFlags(); - Flags &= ~(JITSymbolFlags::Weak | JITSymbolFlags::Common); - assert(Flags == (SymI->second.getFlags() & - ~(JITSymbolFlags::Weak | JITSymbolFlags::Common)) && - "Resolved flags should match the declared flags"); + assert(SymI != Symbols.end() && "Symbol not found"); + assert(!SymI->second.hasMaterializerAttached() && + "Resolving symbol with materializer attached?"); + assert(SymI->second.getState() == SymbolState::Materializing && + "Symbol should be materializing"); + assert(SymI->second.getAddress() == 0 && + "Symbol has already been resolved"); - Worklist.push_back( - {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)}); - } - } + if (SymI->second.getFlags().hasError()) + SymbolsInErrorState.insert(KV.first); + else { + auto Flags = KV.second.getFlags(); + Flags &= ~(JITSymbolFlags::Weak | JITSymbolFlags::Common); + assert(Flags == + (SymI->second.getFlags() & + ~(JITSymbolFlags::Weak | JITSymbolFlags::Common)) && + "Resolved flags should match the declared flags"); - // If any symbols were in the error state then bail out. - if (!SymbolsInErrorState.empty()) - return; + Worklist.push_back( + {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)}); + } + } - while (!Worklist.empty()) { - auto SymI = Worklist.back().SymI; - auto ResolvedSym = Worklist.back().ResolvedSym; - Worklist.pop_back(); + // If any symbols were in the error state then bail out. + if (!SymbolsInErrorState.empty()) { + auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); + return make_error<FailedToMaterialize>( + std::move(FailedSymbolsDepMap)); + } - auto &Name = SymI->first; + while (!Worklist.empty()) { + auto SymI = Worklist.back().SymI; + auto ResolvedSym = Worklist.back().ResolvedSym; + Worklist.pop_back(); - // Resolved symbols can not be weak: discard the weak flag. - JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags(); - SymI->second.setAddress(ResolvedSym.getAddress()); - SymI->second.setFlags(ResolvedFlags); - SymI->second.setState(SymbolState::Resolved); + auto &Name = SymI->first; - auto MII = MaterializingInfos.find(Name); - if (MII == MaterializingInfos.end()) - continue; + // Resolved symbols can not be weak: discard the weak flag. + JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags(); + SymI->second.setAddress(ResolvedSym.getAddress()); + SymI->second.setFlags(ResolvedFlags); + SymI->second.setState(SymbolState::Resolved); - auto &MI = MII->second; - for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { - Q->notifySymbolMetRequiredState(Name, ResolvedSym); - Q->removeQueryDependence(*this, Name); - if (Q->isComplete()) - CompletedQueries.insert(std::move(Q)); - } - } - }); + auto MII = MaterializingInfos.find(Name); + if (MII == MaterializingInfos.end()) + continue; - assert((SymbolsInErrorState.empty() || CompletedQueries.empty()) && - "Can't fail symbols and completed queries at the same time"); + auto &MI = MII->second; + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { + Q->notifySymbolMetRequiredState(Name, ResolvedSym); + Q->removeQueryDependence(*this, Name); + if (Q->isComplete()) + CompletedQueries.insert(std::move(Q)); + } + } - // If we failed any symbols then return an error. - if (!SymbolsInErrorState.empty()) { - auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); - (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); - return make_error<FailedToMaterialize>(std::move(FailedSymbolsDepMap)); - } + return Error::success(); + })) + return Err; // Otherwise notify all the completed queries. for (auto &Q : CompletedQueries) { @@ -901,139 +969,145 @@ Error JITDylib::resolve(const SymbolMap &Resolved) { return Error::success(); } -Error JITDylib::emit(const SymbolFlagsMap &Emitted) { +Error JITDylib::emit(MaterializationResponsibility &MR, + const SymbolFlagsMap &Emitted) { AsynchronousSymbolQuerySet CompletedQueries; - SymbolNameSet SymbolsInErrorState; DenseMap<JITDylib *, SymbolNameVector> ReadySymbols; - ES.runSessionLocked([&, this]() { - std::vector<SymbolTable::iterator> Worklist; + if (auto Err = ES.runSessionLocked([&, this]() -> Error { + auto RTI = MRTrackers.find(&MR); + assert(RTI != MRTrackers.end() && "No resource tracker for MR?"); + if (RTI->second->isDefunct()) + return make_error<ResourceTrackerDefunct>(RTI->second); - // Scan to build worklist, record any symbols in the erorr state. - for (const auto &KV : Emitted) { - auto &Name = KV.first; + SymbolNameSet SymbolsInErrorState; + std::vector<SymbolTable::iterator> Worklist; - auto SymI = Symbols.find(Name); - assert(SymI != Symbols.end() && "No symbol table entry for Name"); + // Scan to build worklist, record any symbols in the erorr state. + for (const auto &KV : Emitted) { + auto &Name = KV.first; - if (SymI->second.getFlags().hasError()) - SymbolsInErrorState.insert(Name); - else - Worklist.push_back(SymI); - } + auto SymI = Symbols.find(Name); + assert(SymI != Symbols.end() && "No symbol table entry for Name"); - // If any symbols were in the error state then bail out. - if (!SymbolsInErrorState.empty()) - return; + if (SymI->second.getFlags().hasError()) + SymbolsInErrorState.insert(Name); + else + Worklist.push_back(SymI); + } - // Otherwise update dependencies and move to the emitted state. - while (!Worklist.empty()) { - auto SymI = Worklist.back(); - Worklist.pop_back(); + // If any symbols were in the error state then bail out. + if (!SymbolsInErrorState.empty()) { + auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); + return make_error<FailedToMaterialize>( + std::move(FailedSymbolsDepMap)); + } - auto &Name = SymI->first; - auto &SymEntry = SymI->second; + // Otherwise update dependencies and move to the emitted state. + while (!Worklist.empty()) { + auto SymI = Worklist.back(); + Worklist.pop_back(); - // Move symbol to the emitted state. - assert(((SymEntry.getFlags().hasMaterializationSideEffectsOnly() && - SymEntry.getState() == SymbolState::Materializing) || - SymEntry.getState() == SymbolState::Resolved) && - "Emitting from state other than Resolved"); - SymEntry.setState(SymbolState::Emitted); + auto &Name = SymI->first; + auto &SymEntry = SymI->second; - auto MII = MaterializingInfos.find(Name); + // Move symbol to the emitted state. + assert(((SymEntry.getFlags().hasMaterializationSideEffectsOnly() && + SymEntry.getState() == SymbolState::Materializing) || + SymEntry.getState() == SymbolState::Resolved) && + "Emitting from state other than Resolved"); + SymEntry.setState(SymbolState::Emitted); - // If this symbol has no MaterializingInfo then it's trivially ready. - // Update its state and continue. - if (MII == MaterializingInfos.end()) { - SymEntry.setState(SymbolState::Ready); - continue; - } + auto MII = MaterializingInfos.find(Name); - auto &MI = MII->second; + // If this symbol has no MaterializingInfo then it's trivially ready. + // Update its state and continue. + if (MII == MaterializingInfos.end()) { + SymEntry.setState(SymbolState::Ready); + continue; + } - // For each dependant, transfer this node's emitted dependencies to - // it. If the dependant node is ready (i.e. has no unemitted - // dependencies) then notify any pending queries. - for (auto &KV : MI.Dependants) { - auto &DependantJD = *KV.first; - auto &DependantJDReadySymbols = ReadySymbols[&DependantJD]; - for (auto &DependantName : KV.second) { - auto DependantMII = - DependantJD.MaterializingInfos.find(DependantName); - assert(DependantMII != DependantJD.MaterializingInfos.end() && - "Dependant should have MaterializingInfo"); + auto &MI = MII->second; - auto &DependantMI = DependantMII->second; + // For each dependant, transfer this node's emitted dependencies to + // it. If the dependant node is ready (i.e. has no unemitted + // dependencies) then notify any pending queries. + for (auto &KV : MI.Dependants) { + auto &DependantJD = *KV.first; + auto &DependantJDReadySymbols = ReadySymbols[&DependantJD]; + for (auto &DependantName : KV.second) { + auto DependantMII = + DependantJD.MaterializingInfos.find(DependantName); + assert(DependantMII != DependantJD.MaterializingInfos.end() && + "Dependant should have MaterializingInfo"); - // Remove the dependant's dependency on this node. - assert(DependantMI.UnemittedDependencies.count(this) && - "Dependant does not have an unemitted dependencies record for " - "this JITDylib"); - assert(DependantMI.UnemittedDependencies[this].count(Name) && - "Dependant does not count this symbol as a dependency?"); + auto &DependantMI = DependantMII->second; - DependantMI.UnemittedDependencies[this].erase(Name); - if (DependantMI.UnemittedDependencies[this].empty()) - DependantMI.UnemittedDependencies.erase(this); + // Remove the dependant's dependency on this node. + assert(DependantMI.UnemittedDependencies.count(this) && + "Dependant does not have an unemitted dependencies record " + "for " + "this JITDylib"); + assert(DependantMI.UnemittedDependencies[this].count(Name) && + "Dependant does not count this symbol as a dependency?"); - // Transfer unemitted dependencies from this node to the dependant. - DependantJD.transferEmittedNodeDependencies(DependantMI, - DependantName, MI); + DependantMI.UnemittedDependencies[this].erase(Name); + if (DependantMI.UnemittedDependencies[this].empty()) + DependantMI.UnemittedDependencies.erase(this); - auto DependantSymI = DependantJD.Symbols.find(DependantName); - assert(DependantSymI != DependantJD.Symbols.end() && - "Dependant has no entry in the Symbols table"); - auto &DependantSymEntry = DependantSymI->second; + // Transfer unemitted dependencies from this node to the + // dependant. + DependantJD.transferEmittedNodeDependencies(DependantMI, + DependantName, MI); - // If the dependant is emitted and this node was the last of its - // unemitted dependencies then the dependant node is now ready, so - // notify any pending queries on the dependant node. - if (DependantSymEntry.getState() == SymbolState::Emitted && - DependantMI.UnemittedDependencies.empty()) { - assert(DependantMI.Dependants.empty() && - "Dependants should be empty by now"); + auto DependantSymI = DependantJD.Symbols.find(DependantName); + assert(DependantSymI != DependantJD.Symbols.end() && + "Dependant has no entry in the Symbols table"); + auto &DependantSymEntry = DependantSymI->second; - // Since this dependant is now ready, we erase its MaterializingInfo - // and update its materializing state. - DependantSymEntry.setState(SymbolState::Ready); - DependantJDReadySymbols.push_back(DependantName); + // If the dependant is emitted and this node was the last of its + // unemitted dependencies then the dependant node is now ready, so + // notify any pending queries on the dependant node. + if (DependantSymEntry.getState() == SymbolState::Emitted && + DependantMI.UnemittedDependencies.empty()) { + assert(DependantMI.Dependants.empty() && + "Dependants should be empty by now"); - for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { - Q->notifySymbolMetRequiredState( - DependantName, DependantSymI->second.getSymbol()); + // Since this dependant is now ready, we erase its + // MaterializingInfo and update its materializing state. + DependantSymEntry.setState(SymbolState::Ready); + DependantJDReadySymbols.push_back(DependantName); + + for (auto &Q : + DependantMI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState( + DependantName, DependantSymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); + Q->removeQueryDependence(DependantJD, DependantName); + } + } + } + } + + auto &ThisJDReadySymbols = ReadySymbols[this]; + MI.Dependants.clear(); + if (MI.UnemittedDependencies.empty()) { + SymI->second.setState(SymbolState::Ready); + ThisJDReadySymbols.push_back(Name); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); if (Q->isComplete()) CompletedQueries.insert(Q); - Q->removeQueryDependence(DependantJD, DependantName); + Q->removeQueryDependence(*this, Name); } } } - } - auto &ThisJDReadySymbols = ReadySymbols[this]; - MI.Dependants.clear(); - if (MI.UnemittedDependencies.empty()) { - SymI->second.setState(SymbolState::Ready); - ThisJDReadySymbols.push_back(Name); - for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { - Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); - if (Q->isComplete()) - CompletedQueries.insert(Q); - Q->removeQueryDependence(*this, Name); - } - } - } - }); - - assert((SymbolsInErrorState.empty() || CompletedQueries.empty()) && - "Can't fail symbols and completed queries at the same time"); - - // If we failed any symbols then return an error. - if (!SymbolsInErrorState.empty()) { - auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>(); - (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState); - return make_error<FailedToMaterialize>(std::move(FailedSymbolsDepMap)); - } + return Error::success(); + })) + return Err; // Otherwise notify all the completed queries. for (auto &Q : CompletedQueries) { @@ -1044,120 +1118,122 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { return Error::success(); } -void JITDylib::notifyFailed(FailedSymbolsWorklist Worklist) { +void JITDylib::unlinkMaterializationResponsibility( + MaterializationResponsibility &MR) { + ES.runSessionLocked([&]() { + auto I = MRTrackers.find(&MR); + assert(I != MRTrackers.end() && "MaterializationResponsibility not linked"); + MRTrackers.erase(I); + }); +} + +std::pair<JITDylib::AsynchronousSymbolQuerySet, + std::shared_ptr<SymbolDependenceMap>> +JITDylib::failSymbols(FailedSymbolsWorklist Worklist) { AsynchronousSymbolQuerySet FailedQueries; auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); - // Failing no symbols is a no-op. - if (Worklist.empty()) - return; - - auto &ES = Worklist.front().first->getExecutionSession(); - - ES.runSessionLocked([&]() { - while (!Worklist.empty()) { - assert(Worklist.back().first && "Failed JITDylib can not be null"); - auto &JD = *Worklist.back().first; - auto Name = std::move(Worklist.back().second); - Worklist.pop_back(); + while (!Worklist.empty()) { + assert(Worklist.back().first && "Failed JITDylib can not be null"); + auto &JD = *Worklist.back().first; + auto Name = std::move(Worklist.back().second); + Worklist.pop_back(); - (*FailedSymbolsMap)[&JD].insert(Name); + (*FailedSymbolsMap)[&JD].insert(Name); - assert(JD.Symbols.count(Name) && "No symbol table entry for Name"); - auto &Sym = JD.Symbols[Name]; + assert(JD.Symbols.count(Name) && "No symbol table entry for Name"); + auto &Sym = JD.Symbols[Name]; - // Move the symbol into the error state. - // Note that this may be redundant: The symbol might already have been - // moved to this state in response to the failure of a dependence. - Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError); + // Move the symbol into the error state. + // Note that this may be redundant: The symbol might already have been + // moved to this state in response to the failure of a dependence. + Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError); - // FIXME: Come up with a sane mapping of state to - // presence-of-MaterializingInfo so that we can assert presence / absence - // here, rather than testing it. - auto MII = JD.MaterializingInfos.find(Name); + // FIXME: Come up with a sane mapping of state to + // presence-of-MaterializingInfo so that we can assert presence / absence + // here, rather than testing it. + auto MII = JD.MaterializingInfos.find(Name); - if (MII == JD.MaterializingInfos.end()) - continue; + if (MII == JD.MaterializingInfos.end()) + continue; - auto &MI = MII->second; + auto &MI = MII->second; - // Move all dependants to the error state and disconnect from them. - for (auto &KV : MI.Dependants) { - auto &DependantJD = *KV.first; - for (auto &DependantName : KV.second) { - assert(DependantJD.Symbols.count(DependantName) && - "No symbol table entry for DependantName"); - auto &DependantSym = DependantJD.Symbols[DependantName]; - DependantSym.setFlags(DependantSym.getFlags() | - JITSymbolFlags::HasError); + // Move all dependants to the error state and disconnect from them. + for (auto &KV : MI.Dependants) { + auto &DependantJD = *KV.first; + for (auto &DependantName : KV.second) { + assert(DependantJD.Symbols.count(DependantName) && + "No symbol table entry for DependantName"); + auto &DependantSym = DependantJD.Symbols[DependantName]; + DependantSym.setFlags(DependantSym.getFlags() | + JITSymbolFlags::HasError); - assert(DependantJD.MaterializingInfos.count(DependantName) && - "No MaterializingInfo for dependant"); - auto &DependantMI = DependantJD.MaterializingInfos[DependantName]; + assert(DependantJD.MaterializingInfos.count(DependantName) && + "No MaterializingInfo for dependant"); + auto &DependantMI = DependantJD.MaterializingInfos[DependantName]; - auto UnemittedDepI = DependantMI.UnemittedDependencies.find(&JD); - assert(UnemittedDepI != DependantMI.UnemittedDependencies.end() && - "No UnemittedDependencies entry for this JITDylib"); - assert(UnemittedDepI->second.count(Name) && - "No UnemittedDependencies entry for this symbol"); - UnemittedDepI->second.erase(Name); - if (UnemittedDepI->second.empty()) - DependantMI.UnemittedDependencies.erase(UnemittedDepI); + auto UnemittedDepI = DependantMI.UnemittedDependencies.find(&JD); + assert(UnemittedDepI != DependantMI.UnemittedDependencies.end() && + "No UnemittedDependencies entry for this JITDylib"); + assert(UnemittedDepI->second.count(Name) && + "No UnemittedDependencies entry for this symbol"); + UnemittedDepI->second.erase(Name); + if (UnemittedDepI->second.empty()) + DependantMI.UnemittedDependencies.erase(UnemittedDepI); - // If this symbol is already in the emitted state then we need to - // take responsibility for failing its queries, so add it to the - // worklist. - if (DependantSym.getState() == SymbolState::Emitted) { - assert(DependantMI.Dependants.empty() && - "Emitted symbol should not have dependants"); - Worklist.push_back(std::make_pair(&DependantJD, DependantName)); - } + // If this symbol is already in the emitted state then we need to + // take responsibility for failing its queries, so add it to the + // worklist. + if (DependantSym.getState() == SymbolState::Emitted) { + assert(DependantMI.Dependants.empty() && + "Emitted symbol should not have dependants"); + Worklist.push_back(std::make_pair(&DependantJD, DependantName)); } } - MI.Dependants.clear(); - - // Disconnect from all unemitted depenencies. - for (auto &KV : MI.UnemittedDependencies) { - auto &UnemittedDepJD = *KV.first; - for (auto &UnemittedDepName : KV.second) { - auto UnemittedDepMII = - UnemittedDepJD.MaterializingInfos.find(UnemittedDepName); - assert(UnemittedDepMII != UnemittedDepJD.MaterializingInfos.end() && - "Missing MII for unemitted dependency"); - assert(UnemittedDepMII->second.Dependants.count(&JD) && - "JD not listed as a dependant of unemitted dependency"); - assert(UnemittedDepMII->second.Dependants[&JD].count(Name) && - "Name is not listed as a dependant of unemitted dependency"); - UnemittedDepMII->second.Dependants[&JD].erase(Name); - if (UnemittedDepMII->second.Dependants[&JD].empty()) - UnemittedDepMII->second.Dependants.erase(&JD); - } - } - MI.UnemittedDependencies.clear(); + } + MI.Dependants.clear(); - // Collect queries to be failed for this MII. - AsynchronousSymbolQueryList ToDetach; - for (auto &Q : MII->second.pendingQueries()) { - // Add the query to the list to be failed and detach it. - FailedQueries.insert(Q); - ToDetach.push_back(Q); + // Disconnect from all unemitted depenencies. + for (auto &KV : MI.UnemittedDependencies) { + auto &UnemittedDepJD = *KV.first; + for (auto &UnemittedDepName : KV.second) { + auto UnemittedDepMII = + UnemittedDepJD.MaterializingInfos.find(UnemittedDepName); + assert(UnemittedDepMII != UnemittedDepJD.MaterializingInfos.end() && + "Missing MII for unemitted dependency"); + assert(UnemittedDepMII->second.Dependants.count(&JD) && + "JD not listed as a dependant of unemitted dependency"); + assert(UnemittedDepMII->second.Dependants[&JD].count(Name) && + "Name is not listed as a dependant of unemitted dependency"); + UnemittedDepMII->second.Dependants[&JD].erase(Name); + if (UnemittedDepMII->second.Dependants[&JD].empty()) + UnemittedDepMII->second.Dependants.erase(&JD); } - for (auto &Q : ToDetach) - Q->detach(); + } + MI.UnemittedDependencies.clear(); - assert(MI.Dependants.empty() && - "Can not delete MaterializingInfo with dependants still attached"); - assert(MI.UnemittedDependencies.empty() && - "Can not delete MaterializingInfo with unemitted dependencies " - "still attached"); - assert(!MI.hasQueriesPending() && - "Can not delete MaterializingInfo with queries pending"); - JD.MaterializingInfos.erase(MII); + // Collect queries to be failed for this MII. + AsynchronousSymbolQueryList ToDetach; + for (auto &Q : MII->second.pendingQueries()) { + // Add the query to the list to be failed and detach it. + FailedQueries.insert(Q); + ToDetach.push_back(Q); } - }); + for (auto &Q : ToDetach) + Q->detach(); - for (auto &Q : FailedQueries) - Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbolsMap)); + assert(MI.Dependants.empty() && + "Can not delete MaterializingInfo with dependants still attached"); + assert(MI.UnemittedDependencies.empty() && + "Can not delete MaterializingInfo with unemitted dependencies " + "still attached"); + assert(!MI.hasQueriesPending() && + "Can not delete MaterializingInfo with queries pending"); + JD.MaterializingInfos.erase(MII); + } + + return std::make_pair(std::move(FailedQueries), std::move(FailedSymbolsMap)); } void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder, @@ -1168,8 +1244,7 @@ void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder, if (NewLinkOrder.empty() || NewLinkOrder.front().first != this) LinkOrder.push_back( std::make_pair(this, JITDylibLookupFlags::MatchAllSymbols)); - LinkOrder.insert(LinkOrder.end(), NewLinkOrder.begin(), - NewLinkOrder.end()); + llvm::append_range(LinkOrder, NewLinkOrder); } else LinkOrder = std::move(NewLinkOrder); }); @@ -1192,10 +1267,10 @@ void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD, void JITDylib::removeFromLinkOrder(JITDylib &JD) { ES.runSessionLocked([&]() { - auto I = std::find_if(LinkOrder.begin(), LinkOrder.end(), - [&](const JITDylibSearchOrder::value_type &KV) { - return KV.first == &JD; - }); + auto I = llvm::find_if(LinkOrder, + [&](const JITDylibSearchOrder::value_type &KV) { + return KV.first == &JD; + }); if (I != LinkOrder.end()) LinkOrder.erase(I); }); @@ -1257,279 +1332,6 @@ Error JITDylib::remove(const SymbolNameSet &Names) { }); } -Expected<SymbolFlagsMap> -JITDylib::lookupFlags(LookupKind K, JITDylibLookupFlags JDLookupFlags, - SymbolLookupSet LookupSet) { - return ES.runSessionLocked([&, this]() -> Expected<SymbolFlagsMap> { - SymbolFlagsMap Result; - lookupFlagsImpl(Result, K, JDLookupFlags, LookupSet); - - // Run any definition generators. - for (auto &DG : DefGenerators) { - - // Bail out early if we found everything. - if (LookupSet.empty()) - break; - - // Run this generator. - if (auto Err = DG->tryToGenerate(K, *this, JDLookupFlags, LookupSet)) - return std::move(Err); - - // Re-try the search. - lookupFlagsImpl(Result, K, JDLookupFlags, LookupSet); - } - - return Result; - }); -} - -void JITDylib::lookupFlagsImpl(SymbolFlagsMap &Result, LookupKind K, - JITDylibLookupFlags JDLookupFlags, - SymbolLookupSet &LookupSet) { - - LookupSet.forEachWithRemoval( - [&](const SymbolStringPtr &Name, SymbolLookupFlags Flags) -> bool { - auto I = Symbols.find(Name); - if (I == Symbols.end()) - return false; - assert(!Result.count(Name) && "Symbol already present in Flags map"); - Result[Name] = I->second.getFlags(); - return true; - }); -} - -Error JITDylib::lodgeQuery(MaterializationUnitList &MUs, - std::shared_ptr<AsynchronousSymbolQuery> &Q, - LookupKind K, JITDylibLookupFlags JDLookupFlags, - SymbolLookupSet &Unresolved) { - assert(Q && "Query can not be null"); - - if (auto Err = lodgeQueryImpl(MUs, Q, K, JDLookupFlags, Unresolved)) - return Err; - - // Run any definition generators. - for (auto &DG : DefGenerators) { - - // Bail out early if we have resolved everything. - if (Unresolved.empty()) - break; - - // Run the generator. - if (auto Err = DG->tryToGenerate(K, *this, JDLookupFlags, Unresolved)) - return Err; - - // Lodge query. This can not fail as any new definitions were added - // by the generator under the session locked. Since they can't have - // started materializing yet they can not have failed. - cantFail(lodgeQueryImpl(MUs, Q, K, JDLookupFlags, Unresolved)); - } - - return Error::success(); -} - -Error JITDylib::lodgeQueryImpl(MaterializationUnitList &MUs, - std::shared_ptr<AsynchronousSymbolQuery> &Q, - LookupKind K, JITDylibLookupFlags JDLookupFlags, - SymbolLookupSet &Unresolved) { - - return Unresolved.forEachWithRemoval( - [&](const SymbolStringPtr &Name, - SymbolLookupFlags SymLookupFlags) -> Expected<bool> { - // Search for name in symbols. If not found then continue without - // removal. - auto SymI = Symbols.find(Name); - if (SymI == Symbols.end()) - return false; - - // If we match against a materialization-side-effects only symbol then - // make sure it is weakly-referenced. Otherwise bail out with an error. - if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && - SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) - return make_error<SymbolsNotFound>(SymbolNameVector({Name})); - - // If this is a non exported symbol and we're matching exported symbols - // only then skip this symbol without removal. - if (!SymI->second.getFlags().isExported() && - JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) - return false; - - // If we matched against this symbol but it is in the error state then - // bail out and treat it as a failure to materialize. - if (SymI->second.getFlags().hasError()) { - auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); - (*FailedSymbolsMap)[this] = {Name}; - return make_error<FailedToMaterialize>(std::move(FailedSymbolsMap)); - } - - // If this symbol already meets the required state for then notify the - // query, then remove the symbol and continue. - if (SymI->second.getState() >= Q->getRequiredState()) { - Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); - return true; - } - - // Otherwise this symbol does not yet meet the required state. Check - // whether it has a materializer attached, and if so prepare to run it. - if (SymI->second.hasMaterializerAttached()) { - assert(SymI->second.getAddress() == 0 && - "Symbol not resolved but already has address?"); - auto UMII = UnmaterializedInfos.find(Name); - assert(UMII != UnmaterializedInfos.end() && - "Lazy symbol should have UnmaterializedInfo"); - auto MU = std::move(UMII->second->MU); - assert(MU != nullptr && "Materializer should not be null"); - - // Move all symbols associated with this MaterializationUnit into - // materializing state. - for (auto &KV : MU->getSymbols()) { - auto SymK = Symbols.find(KV.first); - SymK->second.setMaterializerAttached(false); - SymK->second.setState(SymbolState::Materializing); - UnmaterializedInfos.erase(KV.first); - } - - // Add MU to the list of MaterializationUnits to be materialized. - MUs.push_back(std::move(MU)); - } - - // Add the query to the PendingQueries list and continue, deleting the - // element. - assert(SymI->second.getState() != SymbolState::NeverSearched && - SymI->second.getState() != SymbolState::Ready && - "By this line the symbol should be materializing"); - auto &MI = MaterializingInfos[Name]; - MI.addQuery(Q); - Q->addQueryDependence(*this, Name); - return true; - }); -} - -Expected<SymbolNameSet> -JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, - SymbolNameSet Names) { - assert(Q && "Query can not be null"); - - ES.runOutstandingMUs(); - - bool QueryComplete = false; - std::vector<std::unique_ptr<MaterializationUnit>> MUs; - - SymbolLookupSet Unresolved(Names); - auto Err = ES.runSessionLocked([&, this]() -> Error { - QueryComplete = lookupImpl(Q, MUs, Unresolved); - - // Run any definition generators. - for (auto &DG : DefGenerators) { - - // Bail out early if we have resolved everything. - if (Unresolved.empty()) - break; - - assert(!QueryComplete && "query complete but unresolved symbols remain?"); - if (auto Err = DG->tryToGenerate(LookupKind::Static, *this, - JITDylibLookupFlags::MatchAllSymbols, - Unresolved)) - return Err; - - if (!Unresolved.empty()) - QueryComplete = lookupImpl(Q, MUs, Unresolved); - } - return Error::success(); - }); - - if (Err) - return std::move(Err); - - assert((MUs.empty() || !QueryComplete) && - "If action flags are set, there should be no work to do (so no MUs)"); - - if (QueryComplete) - Q->handleComplete(); - - // FIXME: Swap back to the old code below once RuntimeDyld works with - // callbacks from asynchronous queries. - // Add MUs to the OutstandingMUs list. - { - std::lock_guard<std::recursive_mutex> Lock(ES.OutstandingMUsMutex); - auto ThisJD = shared_from_this(); - for (auto &MU : MUs) { - auto MR = MU->createMaterializationResponsibility(ThisJD); - ES.OutstandingMUs.push_back(make_pair(std::move(MU), std::move(MR))); - } - } - ES.runOutstandingMUs(); - - // Dispatch any required MaterializationUnits for materialization. - // for (auto &MU : MUs) - // ES.dispatchMaterialization(*this, std::move(MU)); - - SymbolNameSet RemainingSymbols; - for (auto &KV : Unresolved) - RemainingSymbols.insert(KV.first); - - return RemainingSymbols; -} - -bool JITDylib::lookupImpl( - std::shared_ptr<AsynchronousSymbolQuery> &Q, - std::vector<std::unique_ptr<MaterializationUnit>> &MUs, - SymbolLookupSet &Unresolved) { - bool QueryComplete = false; - - std::vector<SymbolStringPtr> ToRemove; - Unresolved.forEachWithRemoval( - [&](const SymbolStringPtr &Name, SymbolLookupFlags Flags) -> bool { - // Search for the name in Symbols. Skip without removing if not found. - auto SymI = Symbols.find(Name); - if (SymI == Symbols.end()) - return false; - - // If the symbol is already in the required state then notify the query - // and remove. - if (SymI->second.getState() >= Q->getRequiredState()) { - Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); - if (Q->isComplete()) - QueryComplete = true; - return true; - } - - // If the symbol is lazy, get the MaterialiaztionUnit for it. - if (SymI->second.hasMaterializerAttached()) { - assert(SymI->second.getAddress() == 0 && - "Lazy symbol should not have a resolved address"); - auto UMII = UnmaterializedInfos.find(Name); - assert(UMII != UnmaterializedInfos.end() && - "Lazy symbol should have UnmaterializedInfo"); - auto MU = std::move(UMII->second->MU); - assert(MU != nullptr && "Materializer should not be null"); - - // Kick all symbols associated with this MaterializationUnit into - // materializing state. - for (auto &KV : MU->getSymbols()) { - auto SymK = Symbols.find(KV.first); - assert(SymK != Symbols.end() && "Missing symbol table entry"); - SymK->second.setState(SymbolState::Materializing); - SymK->second.setMaterializerAttached(false); - UnmaterializedInfos.erase(KV.first); - } - - // Add MU to the list of MaterializationUnits to be materialized. - MUs.push_back(std::move(MU)); - } - - // Add the query to the PendingQueries list. - assert(SymI->second.getState() != SymbolState::NeverSearched && - SymI->second.getState() != SymbolState::Ready && - "By this line the symbol should be materializing"); - auto &MI = MaterializingInfos[Name]; - MI.addQuery(Q); - Q->addQueryDependence(*this, Name); - return true; - }); - - return QueryComplete; -} - void JITDylib::dump(raw_ostream &OS) { ES.runSessionLocked([&, this]() { OS << "JITDylib \"" << JITDylibName << "\" (ES: " @@ -1589,11 +1391,10 @@ void JITDylib::MaterializingInfo::addQuery( void JITDylib::MaterializingInfo::removeQuery( const AsynchronousSymbolQuery &Q) { // FIXME: Implement 'find_as' for shared_ptr<T>/T*. - auto I = - std::find_if(PendingQueries.begin(), PendingQueries.end(), - [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) { - return V.get() == &Q; - }); + auto I = llvm::find_if( + PendingQueries, [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) { + return V.get() == &Q; + }); assert(I != PendingQueries.end() && "Query is not attached to this MaterializingInfo"); PendingQueries.erase(I); @@ -1618,6 +1419,137 @@ JITDylib::JITDylib(ExecutionSession &ES, std::string Name) LinkOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols}); } +ResourceTrackerSP JITDylib::getTracker(MaterializationResponsibility &MR) { + auto I = MRTrackers.find(&MR); + assert(I != MRTrackers.end() && "MR is not linked"); + assert(I->second && "Linked tracker is null"); + return I->second; +} + +std::pair<JITDylib::AsynchronousSymbolQuerySet, + std::shared_ptr<SymbolDependenceMap>> +JITDylib::removeTracker(ResourceTracker &RT) { + // Note: Should be called under the session lock. + + SymbolNameVector SymbolsToRemove; + std::vector<std::pair<JITDylib *, SymbolStringPtr>> SymbolsToFail; + + if (&RT == DefaultTracker.get()) { + SymbolNameSet TrackedSymbols; + for (auto &KV : TrackerSymbols) + for (auto &Sym : KV.second) + TrackedSymbols.insert(Sym); + + for (auto &KV : Symbols) { + auto &Sym = KV.first; + if (!TrackedSymbols.count(Sym)) + SymbolsToRemove.push_back(Sym); + } + + DefaultTracker.reset(); + } else { + /// Check for a non-default tracker. + auto I = TrackerSymbols.find(&RT); + if (I != TrackerSymbols.end()) { + SymbolsToRemove = std::move(I->second); + TrackerSymbols.erase(I); + } + // ... if not found this tracker was already defunct. Nothing to do. + } + + for (auto &Sym : SymbolsToRemove) { + assert(Symbols.count(Sym) && "Symbol not in symbol table"); + + // If there is a MaterializingInfo then collect any queries to fail. + auto MII = MaterializingInfos.find(Sym); + if (MII != MaterializingInfos.end()) + SymbolsToFail.push_back({this, Sym}); + } + + AsynchronousSymbolQuerySet QueriesToFail; + auto Result = failSymbols(std::move(SymbolsToFail)); + + // Removed symbols should be taken out of the table altogether. + for (auto &Sym : SymbolsToRemove) { + auto I = Symbols.find(Sym); + assert(I != Symbols.end() && "Symbol not present in table"); + + // Remove Materializer if present. + if (I->second.hasMaterializerAttached()) { + // FIXME: Should this discard the symbols? + UnmaterializedInfos.erase(Sym); + } else { + assert(!UnmaterializedInfos.count(Sym) && + "Symbol has materializer attached"); + } + + Symbols.erase(I); + } + + return Result; +} + +void JITDylib::transferTracker(ResourceTracker &DstRT, ResourceTracker &SrcRT) { + assert(&DstRT != &SrcRT && "No-op transfers shouldn't call transferTracker"); + assert(&DstRT.getJITDylib() == this && "DstRT is not for this JITDylib"); + assert(&SrcRT.getJITDylib() == this && "SrcRT is not for this JITDylib"); + + // Update trackers for any not-yet materialized units. + for (auto &KV : UnmaterializedInfos) { + if (KV.second->RT == &SrcRT) + KV.second->RT = &DstRT; + } + + // Update trackers for any active materialization responsibilities. + for (auto &KV : MRTrackers) { + if (KV.second == &SrcRT) + KV.second = &DstRT; + } + + // If we're transfering to the default tracker we just need to delete the + // tracked symbols for the source tracker. + if (&DstRT == DefaultTracker.get()) { + TrackerSymbols.erase(&SrcRT); + return; + } + + // If we're transferring from the default tracker we need to find all + // currently untracked symbols. + if (&SrcRT == DefaultTracker.get()) { + assert(!TrackerSymbols.count(&SrcRT) && + "Default tracker should not appear in TrackerSymbols"); + + SymbolNameVector SymbolsToTrack; + + SymbolNameSet CurrentlyTrackedSymbols; + for (auto &KV : TrackerSymbols) + for (auto &Sym : KV.second) + CurrentlyTrackedSymbols.insert(Sym); + + for (auto &KV : Symbols) { + auto &Sym = KV.first; + if (!CurrentlyTrackedSymbols.count(Sym)) + SymbolsToTrack.push_back(Sym); + } + + TrackerSymbols[&DstRT] = std::move(SymbolsToTrack); + return; + } + + auto &DstTrackedSymbols = TrackerSymbols[&DstRT]; + + // Finally if neither SrtRT or DstRT are the default tracker then + // just append DstRT's tracked symbols to SrtRT's. + auto SI = TrackerSymbols.find(&SrcRT); + if (SI == TrackerSymbols.end()) + return; + + DstTrackedSymbols.reserve(DstTrackedSymbols.size() + SI->second.size()); + for (auto &Sym : SI->second) + DstTrackedSymbols.push_back(std::move(Sym)); + TrackerSymbols.erase(SI); +} + Error JITDylib::defineImpl(MaterializationUnit &MU) { LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; }); @@ -1685,6 +1617,22 @@ Error JITDylib::defineImpl(MaterializationUnit &MU) { return Error::success(); } +void JITDylib::installMaterializationUnit( + std::unique_ptr<MaterializationUnit> MU, ResourceTracker &RT) { + + /// defineImpl succeeded. + if (&RT != DefaultTracker.get()) { + auto &TS = TrackerSymbols[&RT]; + TS.reserve(TS.size() + MU->getSymbols().size()); + for (auto &KV : MU->getSymbols()) + TS.push_back(KV.first); + } + + auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU), &RT); + for (auto &KV : UMI->MU->getSymbols()) + UnmaterializedInfos[KV.first] = UMI; +} + void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q, const SymbolNameSet &QuerySymbols) { for (auto &QuerySymbol : QuerySymbols) { @@ -1773,7 +1721,39 @@ Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols( } ExecutionSession::ExecutionSession(std::shared_ptr<SymbolStringPool> SSP) - : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) { + : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) {} + +Error ExecutionSession::endSession() { + LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n"); + + std::vector<JITDylibSP> JITDylibsToClose = runSessionLocked([&] { + SessionOpen = false; + return std::move(JDs); + }); + + // TODO: notifiy platform? run static deinits? + + Error Err = Error::success(); + for (auto &JD : JITDylibsToClose) + Err = joinErrors(std::move(Err), JD->clear()); + return Err; +} + +void ExecutionSession::registerResourceManager(ResourceManager &RM) { + runSessionLocked([&] { ResourceManagers.push_back(&RM); }); +} + +void ExecutionSession::deregisterResourceManager(ResourceManager &RM) { + runSessionLocked([&] { + assert(!ResourceManagers.empty() && "No managers registered"); + if (ResourceManagers.back() == &RM) + ResourceManagers.pop_back(); + else { + auto I = llvm::find(ResourceManagers, &RM); + assert(I != ResourceManagers.end() && "RM not registered"); + ResourceManagers.erase(I); + } + }); } JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { @@ -1788,8 +1768,7 @@ JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { JITDylib &ExecutionSession::createBareJITDylib(std::string Name) { assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { - JDs.push_back( - std::shared_ptr<JITDylib>(new JITDylib(*this, std::move(Name)))); + JDs.push_back(new JITDylib(*this, std::move(Name))); return *JDs.back(); }); } @@ -1802,86 +1781,80 @@ Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) { return JD; } -void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { - assert(!!Err && "Error should be in failure state"); +std::vector<JITDylibSP> JITDylib::getDFSLinkOrder(ArrayRef<JITDylibSP> JDs) { + if (JDs.empty()) + return {}; - bool SendErrorToQuery; - runSessionLocked([&]() { - Q.detach(); - SendErrorToQuery = Q.canStillFail(); + auto &ES = JDs.front()->getExecutionSession(); + return ES.runSessionLocked([&]() { + DenseSet<JITDylib *> Visited; + std::vector<JITDylibSP> Result; + + for (auto &JD : JDs) { + + if (Visited.count(JD.get())) + continue; + + SmallVector<JITDylibSP, 64> WorkStack; + WorkStack.push_back(JD); + Visited.insert(JD.get()); + + while (!WorkStack.empty()) { + Result.push_back(std::move(WorkStack.back())); + WorkStack.pop_back(); + + for (auto &KV : llvm::reverse(Result.back()->LinkOrder)) { + auto &JD = *KV.first; + if (Visited.count(&JD)) + continue; + Visited.insert(&JD); + WorkStack.push_back(&JD); + } + } + } + return Result; }); +} - if (SendErrorToQuery) - Q.handleFailed(std::move(Err)); - else - reportError(std::move(Err)); +std::vector<JITDylibSP> +JITDylib::getReverseDFSLinkOrder(ArrayRef<JITDylibSP> JDs) { + auto Tmp = getDFSLinkOrder(JDs); + std::reverse(Tmp.begin(), Tmp.end()); + return Tmp; } -Expected<SymbolMap> ExecutionSession::legacyLookup( - LegacyAsyncLookupFunction AsyncLookup, SymbolNameSet Names, - SymbolState RequiredState, - RegisterDependenciesFunction RegisterDependencies) { -#if LLVM_ENABLE_THREADS - // In the threaded case we use promises to return the results. - std::promise<SymbolMap> PromisedResult; - Error ResolutionError = Error::success(); - auto NotifyComplete = [&](Expected<SymbolMap> R) { - if (R) - PromisedResult.set_value(std::move(*R)); - else { - ErrorAsOutParameter _(&ResolutionError); - ResolutionError = R.takeError(); - PromisedResult.set_value(SymbolMap()); - } - }; -#else - SymbolMap Result; - Error ResolutionError = Error::success(); +std::vector<JITDylibSP> JITDylib::getDFSLinkOrder() { + return getDFSLinkOrder({this}); +} - auto NotifyComplete = [&](Expected<SymbolMap> R) { - ErrorAsOutParameter _(&ResolutionError); - if (R) - Result = std::move(*R); - else - ResolutionError = R.takeError(); - }; -#endif +std::vector<JITDylibSP> JITDylib::getReverseDFSLinkOrder() { + return getReverseDFSLinkOrder({this}); +} - auto Query = std::make_shared<AsynchronousSymbolQuery>( - SymbolLookupSet(Names), RequiredState, std::move(NotifyComplete)); - // FIXME: This should be run session locked along with the registration code - // and error reporting below. - SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); +void ExecutionSession::lookupFlags( + LookupKind K, JITDylibSearchOrder SearchOrder, SymbolLookupSet LookupSet, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) { - // If the query was lodged successfully then register the dependencies, - // otherwise fail it with an error. - if (UnresolvedSymbols.empty()) - RegisterDependencies(Query->QueryRegistrations); - else { - bool DeliverError = runSessionLocked([&]() { - Query->detach(); - return Query->canStillFail(); - }); - auto Err = make_error<SymbolsNotFound>(std::move(UnresolvedSymbols)); - if (DeliverError) - Query->handleFailed(std::move(Err)); - else - reportError(std::move(Err)); - } + OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>( + K, std::move(SearchOrder), std::move(LookupSet), + std::move(OnComplete)), + Error::success()); +} -#if LLVM_ENABLE_THREADS - auto ResultFuture = PromisedResult.get_future(); - auto Result = ResultFuture.get(); - if (ResolutionError) - return std::move(ResolutionError); - return std::move(Result); +Expected<SymbolFlagsMap> +ExecutionSession::lookupFlags(LookupKind K, JITDylibSearchOrder SearchOrder, + SymbolLookupSet LookupSet) { -#else - if (ResolutionError) - return std::move(ResolutionError); + std::promise<MSVCPExpected<SymbolFlagsMap>> ResultP; + OL_applyQueryPhase1(std::make_unique<InProgressLookupFlagsState>( + K, std::move(SearchOrder), std::move(LookupSet), + [&ResultP](Expected<SymbolFlagsMap> Result) { + ResultP.set_value(std::move(Result)); + }), + Error::success()); - return Result; -#endif + auto ResultF = ResultP.get_future(); + return ResultF.get(); } void ExecutionSession::lookup( @@ -1900,94 +1873,17 @@ void ExecutionSession::lookup( // lookup can be re-entered recursively if running on a single thread. Run any // outstanding MUs in case this query depends on them, otherwise this lookup // will starve waiting for a result from an MU that is stuck in the queue. - runOutstandingMUs(); + dispatchOutstandingMUs(); auto Unresolved = std::move(Symbols); - std::map<JITDylib *, MaterializationUnitList> CollectedMUsMap; auto Q = std::make_shared<AsynchronousSymbolQuery>(Unresolved, RequiredState, std::move(NotifyComplete)); - bool QueryComplete = false; - - auto LodgingErr = runSessionLocked([&]() -> Error { - auto LodgeQuery = [&]() -> Error { - for (auto &KV : SearchOrder) { - assert(KV.first && "JITDylibList entries must not be null"); - assert(!CollectedMUsMap.count(KV.first) && - "JITDylibList should not contain duplicate entries"); - - auto &JD = *KV.first; - auto JDLookupFlags = KV.second; - if (auto Err = JD.lodgeQuery(CollectedMUsMap[&JD], Q, K, JDLookupFlags, - Unresolved)) - return Err; - } - // Strip any weakly referenced symbols that were not found. - Unresolved.forEachWithRemoval( - [&](const SymbolStringPtr &Name, SymbolLookupFlags Flags) { - if (Flags == SymbolLookupFlags::WeaklyReferencedSymbol) { - Q->dropSymbol(Name); - return true; - } - return false; - }); - - if (!Unresolved.empty()) - return make_error<SymbolsNotFound>(Unresolved.getSymbolNames()); - - return Error::success(); - }; - - if (auto Err = LodgeQuery()) { - // Query failed. - - // Disconnect the query from its dependencies. - Q->detach(); - - // Replace the MUs. - for (auto &KV : CollectedMUsMap) - for (auto &MU : KV.second) - KV.first->replace(std::move(MU)); - - return Err; - } - - // Query lodged successfully. - - // Record whether this query is fully ready / resolved. We will use - // this to call handleFullyResolved/handleFullyReady outside the session - // lock. - QueryComplete = Q->isComplete(); - - // Call the register dependencies function. - if (RegisterDependencies && !Q->QueryRegistrations.empty()) - RegisterDependencies(Q->QueryRegistrations); - - return Error::success(); - }); - - if (LodgingErr) { - Q->handleFailed(std::move(LodgingErr)); - return; - } - - if (QueryComplete) - Q->handleComplete(); + auto IPLS = std::make_unique<InProgressFullLookupState>( + K, SearchOrder, std::move(Unresolved), RequiredState, std::move(Q), + std::move(RegisterDependencies)); - // Move the MUs to the OutstandingMUs list, then materialize. - { - std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); - - for (auto &KV : CollectedMUsMap) { - auto JD = KV.first->shared_from_this(); - for (auto &MU : KV.second) { - auto MR = MU->createMaterializationResponsibility(JD); - OutstandingMUs.push_back(std::make_pair(std::move(MU), std::move(MR))); - } - } - } - - runOutstandingMUs(); + OL_applyQueryPhase1(std::move(IPLS), Error::success()); } Expected<SymbolMap> @@ -2077,10 +1973,11 @@ void ExecutionSession::dump(raw_ostream &OS) { }); } -void ExecutionSession::runOutstandingMUs() { +void ExecutionSession::dispatchOutstandingMUs() { + LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n"); while (1) { Optional<std::pair<std::unique_ptr<MaterializationUnit>, - MaterializationResponsibility>> + std::unique_ptr<MaterializationResponsibility>>> JMU; { @@ -2095,8 +1992,777 @@ void ExecutionSession::runOutstandingMUs() { break; assert(JMU->first && "No MU?"); + LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n"); dispatchMaterialization(std::move(JMU->first), std::move(JMU->second)); } + LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n"); +} + +Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) { + LLVM_DEBUG({ + dbgs() << "In " << RT.getJITDylib().getName() << " removing tracker " + << formatv("{0:x}", RT.getKeyUnsafe()) << "\n"; + }); + std::vector<ResourceManager *> CurrentResourceManagers; + + JITDylib::AsynchronousSymbolQuerySet QueriesToFail; + std::shared_ptr<SymbolDependenceMap> FailedSymbols; + + runSessionLocked([&] { + CurrentResourceManagers = ResourceManagers; + RT.makeDefunct(); + std::tie(QueriesToFail, FailedSymbols) = RT.getJITDylib().removeTracker(RT); + }); + + Error Err = Error::success(); + + for (auto *L : reverse(CurrentResourceManagers)) + Err = + joinErrors(std::move(Err), L->handleRemoveResources(RT.getKeyUnsafe())); + + for (auto &Q : QueriesToFail) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); + + return Err; +} + +void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT, + ResourceTracker &SrcRT) { + LLVM_DEBUG({ + dbgs() << "In " << SrcRT.getJITDylib().getName() + << " transfering resources from tracker " + << formatv("{0:x}", SrcRT.getKeyUnsafe()) << " to tracker " + << formatv("{0:x}", DstRT.getKeyUnsafe()) << "\n"; + }); + + // No-op transfers are allowed and do not invalidate the source. + if (&DstRT == &SrcRT) + return; + + assert(&DstRT.getJITDylib() == &SrcRT.getJITDylib() && + "Can't transfer resources between JITDylibs"); + runSessionLocked([&]() { + SrcRT.makeDefunct(); + auto &JD = DstRT.getJITDylib(); + JD.transferTracker(DstRT, SrcRT); + for (auto *L : reverse(ResourceManagers)) + L->handleTransferResources(DstRT.getKeyUnsafe(), SrcRT.getKeyUnsafe()); + }); +} + +void ExecutionSession::destroyResourceTracker(ResourceTracker &RT) { + runSessionLocked([&]() { + LLVM_DEBUG({ + dbgs() << "In " << RT.getJITDylib().getName() << " destroying tracker " + << formatv("{0:x}", RT.getKeyUnsafe()) << "\n"; + }); + if (!RT.isDefunct()) + transferResourceTracker(*RT.getJITDylib().getDefaultResourceTracker(), + RT); + }); +} + +Error ExecutionSession::IL_updateCandidatesFor( + JITDylib &JD, JITDylibLookupFlags JDLookupFlags, + SymbolLookupSet &Candidates, SymbolLookupSet *NonCandidates) { + return Candidates.forEachWithRemoval( + [&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) -> Expected<bool> { + /// Search for the symbol. If not found then continue without + /// removal. + auto SymI = JD.Symbols.find(Name); + if (SymI == JD.Symbols.end()) + return false; + + // If this is a non-exported symbol and we're matching exported + // symbols only then remove this symbol from the candidates list. + // + // If we're tracking non-candidates then add this to the non-candidate + // list. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) { + if (NonCandidates) + NonCandidates->add(Name, SymLookupFlags); + return true; + } + + // If we match against a materialization-side-effects only symbol + // then make sure it is weakly-referenced. Otherwise bail out with + // an error. + // FIXME: Use a "materialization-side-effects-only symbols must be + // weakly referenced" specific error here to reduce confusion. + if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && + SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) + return make_error<SymbolsNotFound>(SymbolNameVector({Name})); + + // If we matched against this symbol but it is in the error state + // then bail out and treat it as a failure to materialize. + if (SymI->second.getFlags().hasError()) { + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsMap)[&JD] = {Name}; + return make_error<FailedToMaterialize>(std::move(FailedSymbolsMap)); + } + + // Otherwise this is a match. Remove it from the candidate set. + return true; + }); +} + +void ExecutionSession::OL_applyQueryPhase1( + std::unique_ptr<InProgressLookupState> IPLS, Error Err) { + + LLVM_DEBUG({ + dbgs() << "Entering OL_applyQueryPhase1:\n" + << " Lookup kind: " << IPLS->K << "\n" + << " Search order: " << IPLS->SearchOrder + << ", Current index = " << IPLS->CurSearchOrderIndex + << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n" + << " Lookup set: " << IPLS->LookupSet << "\n" + << " Definition generator candidates: " + << IPLS->DefGeneratorCandidates << "\n" + << " Definition generator non-candidates: " + << IPLS->DefGeneratorNonCandidates << "\n"; + }); + + // FIXME: We should attach the query as we go: This provides a result in a + // single pass in the common case where all symbols have already reached the + // required state. The query could be detached again in the 'fail' method on + // IPLS. Phase 2 would be reduced to collecting and dispatching the MUs. + + while (IPLS->CurSearchOrderIndex != IPLS->SearchOrder.size()) { + + // If we've been handed an error or received one back from a generator then + // fail the query. We don't need to unlink: At this stage the query hasn't + // actually been lodged. + if (Err) + return IPLS->fail(std::move(Err)); + + // Get the next JITDylib and lookup flags. + auto &KV = IPLS->SearchOrder[IPLS->CurSearchOrderIndex]; + auto &JD = *KV.first; + auto JDLookupFlags = KV.second; + + LLVM_DEBUG({ + dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags + << ") with lookup set " << IPLS->LookupSet << ":\n"; + }); + + // If we've just reached a new JITDylib then perform some setup. + if (IPLS->NewJITDylib) { + + // Acquire the generator lock for this JITDylib. + IPLS->GeneratorLock = std::unique_lock<std::mutex>(JD.GeneratorsMutex); + + // Add any non-candidates from the last JITDylib (if any) back on to the + // list of definition candidates for this JITDylib, reset definition + // non-candiates to the empty set. + SymbolLookupSet Tmp; + std::swap(IPLS->DefGeneratorNonCandidates, Tmp); + IPLS->DefGeneratorCandidates.append(std::move(Tmp)); + + LLVM_DEBUG({ + dbgs() << " First time visiting " << JD.getName() + << ", resetting candidate sets and building generator stack\n"; + }); + + // Build the definition generator stack for this JITDylib. + for (auto &DG : reverse(JD.DefGenerators)) + IPLS->CurDefGeneratorStack.push_back(DG); + + // Flag that we've done our initialization. + IPLS->NewJITDylib = false; + } + + // Remove any generation candidates that are already defined (and match) in + // this JITDylib. + runSessionLocked([&] { + // Update the list of candidates (and non-candidates) for definition + // generation. + LLVM_DEBUG(dbgs() << " Updating candidate set...\n"); + Err = IL_updateCandidatesFor( + JD, JDLookupFlags, IPLS->DefGeneratorCandidates, + JD.DefGenerators.empty() ? nullptr + : &IPLS->DefGeneratorNonCandidates); + LLVM_DEBUG({ + dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates + << "\n"; + }); + }); + + // If we encountered an error while filtering generation candidates then + // bail out. + if (Err) + return IPLS->fail(std::move(Err)); + + /// Apply any definition generators on the stack. + LLVM_DEBUG({ + if (IPLS->CurDefGeneratorStack.empty()) + LLVM_DEBUG(dbgs() << " No generators to run for this JITDylib.\n"); + else if (IPLS->DefGeneratorCandidates.empty()) + LLVM_DEBUG(dbgs() << " No candidates to generate.\n"); + else + dbgs() << " Running " << IPLS->CurDefGeneratorStack.size() + << " remaining generators for " + << IPLS->DefGeneratorCandidates.size() << " candidates\n"; + }); + while (!IPLS->CurDefGeneratorStack.empty() && + !IPLS->DefGeneratorCandidates.empty()) { + auto DG = IPLS->CurDefGeneratorStack.back().lock(); + IPLS->CurDefGeneratorStack.pop_back(); + + if (!DG) + return IPLS->fail(make_error<StringError>( + "DefinitionGenerator removed while lookup in progress", + inconvertibleErrorCode())); + + auto K = IPLS->K; + auto &LookupSet = IPLS->DefGeneratorCandidates; + + // Run the generator. If the generator takes ownership of QA then this + // will break the loop. + { + LLVM_DEBUG(dbgs() << " Attempting to generate " << LookupSet << "\n"); + LookupState LS(std::move(IPLS)); + Err = DG->tryToGenerate(LS, K, JD, JDLookupFlags, LookupSet); + IPLS = std::move(LS.IPLS); + } + + // If there was an error then fail the query. + if (Err) { + LLVM_DEBUG({ + dbgs() << " Error attempting to generate " << LookupSet << "\n"; + }); + assert(IPLS && "LS cannot be retained if error is returned"); + return IPLS->fail(std::move(Err)); + } + + // Otherwise if QA was captured then break the loop. + if (!IPLS) { + LLVM_DEBUG( + { dbgs() << " LookupState captured. Exiting phase1 for now.\n"; }); + return; + } + + // Otherwise if we're continuing around the loop then update candidates + // for the next round. + runSessionLocked([&] { + LLVM_DEBUG(dbgs() << " Updating candidate set post-generation\n"); + Err = IL_updateCandidatesFor( + JD, JDLookupFlags, IPLS->DefGeneratorCandidates, + JD.DefGenerators.empty() ? nullptr + : &IPLS->DefGeneratorNonCandidates); + }); + + // If updating candidates failed then fail the query. + if (Err) { + LLVM_DEBUG(dbgs() << " Error encountered while updating candidates\n"); + return IPLS->fail(std::move(Err)); + } + } + + // If we get here then we've moved on to the next JITDylib. + LLVM_DEBUG(dbgs() << "Phase 1 moving to next JITDylib.\n"); + ++IPLS->CurSearchOrderIndex; + IPLS->NewJITDylib = true; + } + + // Remove any weakly referenced candidates that could not be found/generated. + IPLS->DefGeneratorCandidates.remove_if( + [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { + return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol; + }); + + // If we get here then we've finished searching all JITDylibs. + // If we matched all symbols then move to phase 2, otherwise fail the query + // with a SymbolsNotFound error. + if (IPLS->DefGeneratorCandidates.empty()) { + LLVM_DEBUG(dbgs() << "Phase 1 succeeded.\n"); + IPLS->complete(std::move(IPLS)); + } else { + LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n"); + IPLS->fail(make_error<SymbolsNotFound>( + IPLS->DefGeneratorCandidates.getSymbolNames())); + } +} + +void ExecutionSession::OL_completeLookup( + std::unique_ptr<InProgressLookupState> IPLS, + std::shared_ptr<AsynchronousSymbolQuery> Q, + RegisterDependenciesFunction RegisterDependencies) { + + LLVM_DEBUG({ + dbgs() << "Entering OL_completeLookup:\n" + << " Lookup kind: " << IPLS->K << "\n" + << " Search order: " << IPLS->SearchOrder + << ", Current index = " << IPLS->CurSearchOrderIndex + << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n" + << " Lookup set: " << IPLS->LookupSet << "\n" + << " Definition generator candidates: " + << IPLS->DefGeneratorCandidates << "\n" + << " Definition generator non-candidates: " + << IPLS->DefGeneratorNonCandidates << "\n"; + }); + + bool QueryComplete = false; + DenseMap<JITDylib *, JITDylib::UnmaterializedInfosList> CollectedUMIs; + + auto LodgingErr = runSessionLocked([&]() -> Error { + for (auto &KV : IPLS->SearchOrder) { + auto &JD = *KV.first; + auto JDLookupFlags = KV.second; + LLVM_DEBUG({ + dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags + << ") with lookup set " << IPLS->LookupSet << ":\n"; + }); + + auto Err = IPLS->LookupSet.forEachWithRemoval( + [&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) -> Expected<bool> { + LLVM_DEBUG({ + dbgs() << " Attempting to match \"" << Name << "\" (" + << SymLookupFlags << ")... "; + }); + + /// Search for the symbol. If not found then continue without + /// removal. + auto SymI = JD.Symbols.find(Name); + if (SymI == JD.Symbols.end()) { + LLVM_DEBUG(dbgs() << "skipping: not present\n"); + return false; + } + + // If this is a non-exported symbol and we're matching exported + // symbols only then skip this symbol without removal. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == + JITDylibLookupFlags::MatchExportedSymbolsOnly) { + LLVM_DEBUG(dbgs() << "skipping: not exported\n"); + return false; + } + + // If we match against a materialization-side-effects only symbol + // then make sure it is weakly-referenced. Otherwise bail out with + // an error. + // FIXME: Use a "materialization-side-effects-only symbols must be + // weakly referenced" specific error here to reduce confusion. + if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && + SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) { + LLVM_DEBUG({ + dbgs() << "error: " + "required, but symbol is has-side-effects-only\n"; + }); + return make_error<SymbolsNotFound>(SymbolNameVector({Name})); + } + + // If we matched against this symbol but it is in the error state + // then bail out and treat it as a failure to materialize. + if (SymI->second.getFlags().hasError()) { + LLVM_DEBUG(dbgs() << "error: symbol is in error state\n"); + auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>(); + (*FailedSymbolsMap)[&JD] = {Name}; + return make_error<FailedToMaterialize>( + std::move(FailedSymbolsMap)); + } + + // Otherwise this is a match. + + // If this symbol is already in the requried state then notify the + // query, remove the symbol and continue. + if (SymI->second.getState() >= Q->getRequiredState()) { + LLVM_DEBUG(dbgs() + << "matched, symbol already in required state\n"); + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + return true; + } + + // Otherwise this symbol does not yet meet the required state. Check + // whether it has a materializer attached, and if so prepare to run + // it. + if (SymI->second.hasMaterializerAttached()) { + assert(SymI->second.getAddress() == 0 && + "Symbol not resolved but already has address?"); + auto UMII = JD.UnmaterializedInfos.find(Name); + assert(UMII != JD.UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + + auto UMI = UMII->second; + assert(UMI->MU && "Materializer should not be null"); + assert(UMI->RT && "Tracker should not be null"); + LLVM_DEBUG({ + dbgs() << "matched, preparing to dispatch MU@" << UMI->MU.get() + << " (" << UMI->MU->getName() << ")\n"; + }); + + // Move all symbols associated with this MaterializationUnit into + // materializing state. + for (auto &KV : UMI->MU->getSymbols()) { + auto SymK = JD.Symbols.find(KV.first); + assert(SymK != JD.Symbols.end() && + "No entry for symbol covered by MaterializationUnit"); + SymK->second.setMaterializerAttached(false); + SymK->second.setState(SymbolState::Materializing); + JD.UnmaterializedInfos.erase(KV.first); + } + + // Add MU to the list of MaterializationUnits to be materialized. + CollectedUMIs[&JD].push_back(std::move(UMI)); + } else + LLVM_DEBUG(dbgs() << "matched, registering query"); + + // Add the query to the PendingQueries list and continue, deleting + // the element from the lookup set. + assert(SymI->second.getState() != SymbolState::NeverSearched && + SymI->second.getState() != SymbolState::Ready && + "By this line the symbol should be materializing"); + auto &MI = JD.MaterializingInfos[Name]; + MI.addQuery(Q); + Q->addQueryDependence(JD, Name); + + return true; + }); + + // Handle failure. + if (Err) { + + LLVM_DEBUG({ + dbgs() << "Lookup failed. Detaching query and replacing MUs.\n"; + }); + + // Detach the query. + Q->detach(); + + // Replace the MUs. + for (auto &KV : CollectedUMIs) { + auto &JD = *KV.first; + for (auto &UMI : KV.second) + for (auto &KV2 : UMI->MU->getSymbols()) { + assert(!JD.UnmaterializedInfos.count(KV2.first) && + "Unexpected materializer in map"); + auto SymI = JD.Symbols.find(KV2.first); + assert(SymI != JD.Symbols.end() && "Missing symbol entry"); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "MaterializerAttached flag should not be set"); + SymI->second.setMaterializerAttached(true); + JD.UnmaterializedInfos[KV2.first] = UMI; + } + } + + return Err; + } + } + + LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-refererced symbols\n"); + IPLS->LookupSet.forEachWithRemoval( + [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { + if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) { + Q->dropSymbol(Name); + return true; + } else + return false; + }); + + if (!IPLS->LookupSet.empty()) { + LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n"); + return make_error<SymbolsNotFound>(IPLS->LookupSet.getSymbolNames()); + } + + // Record whether the query completed. + QueryComplete = Q->isComplete(); + + LLVM_DEBUG({ + dbgs() << "Query successfully " + << (QueryComplete ? "completed" : "lodged") << "\n"; + }); + + // Move the collected MUs to the OutstandingMUs list. + if (!CollectedUMIs.empty()) { + std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); + + LLVM_DEBUG(dbgs() << "Adding MUs to dispatch:\n"); + for (auto &KV : CollectedUMIs) { + auto &JD = *KV.first; + LLVM_DEBUG({ + dbgs() << " For " << JD.getName() << ": Adding " << KV.second.size() + << " MUs.\n"; + }); + for (auto &UMI : KV.second) { + std::unique_ptr<MaterializationResponsibility> MR( + new MaterializationResponsibility( + &JD, std::move(UMI->MU->SymbolFlags), + std::move(UMI->MU->InitSymbol))); + JD.MRTrackers[MR.get()] = UMI->RT; + OutstandingMUs.push_back( + std::make_pair(std::move(UMI->MU), std::move(MR))); + } + } + } else + LLVM_DEBUG(dbgs() << "No MUs to dispatch.\n"); + + if (RegisterDependencies && !Q->QueryRegistrations.empty()) { + LLVM_DEBUG(dbgs() << "Registering dependencies\n"); + RegisterDependencies(Q->QueryRegistrations); + } else + LLVM_DEBUG(dbgs() << "No dependencies to register\n"); + + return Error::success(); + }); + + if (LodgingErr) { + LLVM_DEBUG(dbgs() << "Failing query\n"); + Q->detach(); + Q->handleFailed(std::move(LodgingErr)); + return; + } + + if (QueryComplete) { + LLVM_DEBUG(dbgs() << "Completing query\n"); + Q->handleComplete(); + } + + dispatchOutstandingMUs(); +} + +void ExecutionSession::OL_completeLookupFlags( + std::unique_ptr<InProgressLookupState> IPLS, + unique_function<void(Expected<SymbolFlagsMap>)> OnComplete) { + + auto Result = runSessionLocked([&]() -> Expected<SymbolFlagsMap> { + LLVM_DEBUG({ + dbgs() << "Entering OL_completeLookupFlags:\n" + << " Lookup kind: " << IPLS->K << "\n" + << " Search order: " << IPLS->SearchOrder + << ", Current index = " << IPLS->CurSearchOrderIndex + << (IPLS->NewJITDylib ? " (entering new JITDylib)" : "") << "\n" + << " Lookup set: " << IPLS->LookupSet << "\n" + << " Definition generator candidates: " + << IPLS->DefGeneratorCandidates << "\n" + << " Definition generator non-candidates: " + << IPLS->DefGeneratorNonCandidates << "\n"; + }); + + SymbolFlagsMap Result; + + // Attempt to find flags for each symbol. + for (auto &KV : IPLS->SearchOrder) { + auto &JD = *KV.first; + auto JDLookupFlags = KV.second; + LLVM_DEBUG({ + dbgs() << "Visiting \"" << JD.getName() << "\" (" << JDLookupFlags + << ") with lookup set " << IPLS->LookupSet << ":\n"; + }); + + IPLS->LookupSet.forEachWithRemoval([&](const SymbolStringPtr &Name, + SymbolLookupFlags SymLookupFlags) { + LLVM_DEBUG({ + dbgs() << " Attempting to match \"" << Name << "\" (" + << SymLookupFlags << ")... "; + }); + + // Search for the symbol. If not found then continue without removing + // from the lookup set. + auto SymI = JD.Symbols.find(Name); + if (SymI == JD.Symbols.end()) { + LLVM_DEBUG(dbgs() << "skipping: not present\n"); + return false; + } + + // If this is a non-exported symbol then it doesn't match. Skip it. + if (!SymI->second.getFlags().isExported() && + JDLookupFlags == JITDylibLookupFlags::MatchExportedSymbolsOnly) { + LLVM_DEBUG(dbgs() << "skipping: not exported\n"); + return false; + } + + LLVM_DEBUG({ + dbgs() << "matched, \"" << Name << "\" -> " << SymI->second.getFlags() + << "\n"; + }); + Result[Name] = SymI->second.getFlags(); + return true; + }); + } + + // Remove any weakly referenced symbols that haven't been resolved. + IPLS->LookupSet.remove_if( + [](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { + return SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol; + }); + + if (!IPLS->LookupSet.empty()) { + LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n"); + return make_error<SymbolsNotFound>(IPLS->LookupSet.getSymbolNames()); + } + + LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n"); + return Result; + }); + + // Run the callback on the result. + LLVM_DEBUG(dbgs() << "Sending result to handler.\n"); + OnComplete(std::move(Result)); +} + +void ExecutionSession::OL_destroyMaterializationResponsibility( + MaterializationResponsibility &MR) { + + assert(MR.SymbolFlags.empty() && + "All symbols should have been explicitly materialized or failed"); + MR.JD->unlinkMaterializationResponsibility(MR); +} + +SymbolNameSet ExecutionSession::OL_getRequestedSymbols( + const MaterializationResponsibility &MR) { + return MR.JD->getRequestedSymbols(MR.SymbolFlags); +} + +Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR, + const SymbolMap &Symbols) { + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " resolving " << Symbols << "\n"; + }); +#ifndef NDEBUG + for (auto &KV : Symbols) { + auto WeakFlags = JITSymbolFlags::Weak | JITSymbolFlags::Common; + auto I = MR.SymbolFlags.find(KV.first); + assert(I != MR.SymbolFlags.end() && + "Resolving symbol outside this responsibility set"); + assert(!I->second.hasMaterializationSideEffectsOnly() && + "Can't resolve materialization-side-effects-only symbol"); + assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) && + "Resolving symbol with incorrect flags"); + } +#endif + + return MR.JD->resolve(MR, Symbols); +} + +Error ExecutionSession::OL_notifyEmitted(MaterializationResponsibility &MR) { + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " emitting " << MR.SymbolFlags << "\n"; + }); + + if (auto Err = MR.JD->emit(MR, MR.SymbolFlags)) + return Err; + + MR.SymbolFlags.clear(); + return Error::success(); +} + +Error ExecutionSession::OL_defineMaterializing( + MaterializationResponsibility &MR, SymbolFlagsMap NewSymbolFlags) { + + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " defining materializing symbols " + << NewSymbolFlags << "\n"; + }); + if (auto AcceptedDefs = MR.JD->defineMaterializing(std::move(NewSymbolFlags))) { + // Add all newly accepted symbols to this responsibility object. + for (auto &KV : *AcceptedDefs) + MR.SymbolFlags.insert(KV); + return Error::success(); + } else + return AcceptedDefs.takeError(); +} + +void ExecutionSession::OL_notifyFailed(MaterializationResponsibility &MR) { + + LLVM_DEBUG({ + dbgs() << "In " << MR.JD->getName() << " failing materialization for " + << MR.SymbolFlags << "\n"; + }); + + JITDylib::FailedSymbolsWorklist Worklist; + + for (auto &KV : MR.SymbolFlags) + Worklist.push_back(std::make_pair(MR.JD.get(), KV.first)); + MR.SymbolFlags.clear(); + + if (Worklist.empty()) + return; + + JITDylib::AsynchronousSymbolQuerySet FailedQueries; + std::shared_ptr<SymbolDependenceMap> FailedSymbols; + + runSessionLocked([&]() { + auto RTI = MR.JD->MRTrackers.find(&MR); + assert(RTI != MR.JD->MRTrackers.end() && "No tracker for this"); + if (RTI->second->isDefunct()) + return; + + std::tie(FailedQueries, FailedSymbols) = + JITDylib::failSymbols(std::move(Worklist)); + }); + + for (auto &Q : FailedQueries) + Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols)); +} + +Error ExecutionSession::OL_replace(MaterializationResponsibility &MR, + std::unique_ptr<MaterializationUnit> MU) { + for (auto &KV : MU->getSymbols()) { + assert(MR.SymbolFlags.count(KV.first) && + "Replacing definition outside this responsibility set"); + MR.SymbolFlags.erase(KV.first); + } + + if (MU->getInitializerSymbol() == MR.InitSymbol) + MR.InitSymbol = nullptr; + + LLVM_DEBUG(MR.JD->getExecutionSession().runSessionLocked([&]() { + dbgs() << "In " << MR.JD->getName() << " replacing symbols with " << *MU + << "\n"; + });); + + return MR.JD->replace(MR, std::move(MU)); +} + +Expected<std::unique_ptr<MaterializationResponsibility>> +ExecutionSession::OL_delegate(MaterializationResponsibility &MR, + const SymbolNameSet &Symbols) { + + SymbolStringPtr DelegatedInitSymbol; + SymbolFlagsMap DelegatedFlags; + + for (auto &Name : Symbols) { + auto I = MR.SymbolFlags.find(Name); + assert(I != MR.SymbolFlags.end() && + "Symbol is not tracked by this MaterializationResponsibility " + "instance"); + + DelegatedFlags[Name] = std::move(I->second); + if (Name == MR.InitSymbol) + std::swap(MR.InitSymbol, DelegatedInitSymbol); + + MR.SymbolFlags.erase(I); + } + + return MR.JD->delegate(MR, std::move(DelegatedFlags), + std::move(DelegatedInitSymbol)); +} + +void ExecutionSession::OL_addDependencies( + MaterializationResponsibility &MR, const SymbolStringPtr &Name, + const SymbolDependenceMap &Dependencies) { + LLVM_DEBUG({ + dbgs() << "Adding dependencies for " << Name << ": " << Dependencies + << "\n"; + }); + assert(MR.SymbolFlags.count(Name) && + "Symbol not covered by this MaterializationResponsibility instance"); + MR.JD->addDependencies(Name, Dependencies); +} + +void ExecutionSession::OL_addDependenciesForAll( + MaterializationResponsibility &MR, + const SymbolDependenceMap &Dependencies) { + LLVM_DEBUG({ + dbgs() << "Adding dependencies for all symbols in " << MR.SymbolFlags << ": " + << Dependencies << "\n"; + }); + for (auto &KV : MR.SymbolFlags) + MR.JD->addDependencies(KV.first, Dependencies); } #ifndef NDEBUG |