diff options
Diffstat (limited to 'lib/ExecutionEngine/Orc')
24 files changed, 1342 insertions, 864 deletions
diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index 241eb3600da7..99bf53bc3afa 100644 --- a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -1,9 +1,8 @@ //===----- CompileOnDemandLayer.cpp - Lazily emit IR on first call --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/CompileUtils.cpp b/lib/ExecutionEngine/Orc/CompileUtils.cpp new file mode 100644 index 000000000000..d46b6fcf9a5f --- /dev/null +++ b/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -0,0 +1,86 @@ +//===------ CompileUtils.cpp - Utilities for compiling IR in the JIT ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Target/TargetMachine.h" + +#include <algorithm> + +namespace llvm { +namespace orc { + +/// Compile a Module to an ObjectFile. +SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) { + CompileResult CachedObject = tryToLoadFromObjectCache(M); + if (CachedObject) + return CachedObject; + + SmallVector<char, 0> ObjBufferSV; + + { + raw_svector_ostream ObjStream(ObjBufferSV); + + legacy::PassManager PM; + MCContext *Ctx; + if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) + llvm_unreachable("Target does not support MC emission."); + PM.run(M); + } + + auto ObjBuffer = llvm::make_unique<SmallVectorMemoryBuffer>( + std::move(ObjBufferSV), + "<in memory object compiled from " + M.getModuleIdentifier() + ">"); + + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); + + if (Obj) { + notifyObjectCompiled(M, *ObjBuffer); + return std::move(ObjBuffer); + } + + // TODO: Actually report errors helpfully. + consumeError(Obj.takeError()); + return nullptr; +} + +SimpleCompiler::CompileResult +SimpleCompiler::tryToLoadFromObjectCache(const Module &M) { + if (!ObjCache) + return CompileResult(); + + return ObjCache->getObject(&M); +} + +void SimpleCompiler::notifyObjectCompiled(const Module &M, + const MemoryBuffer &ObjBuffer) { + if (ObjCache) + ObjCache->notifyObjectCompiled(&M, ObjBuffer.getMemBufferRef()); +} + +ConcurrentIRCompiler::ConcurrentIRCompiler(JITTargetMachineBuilder JTMB, + ObjectCache *ObjCache) + : JTMB(std::move(JTMB)), ObjCache(ObjCache) {} + +std::unique_ptr<MemoryBuffer> ConcurrentIRCompiler::operator()(Module &M) { + auto TM = cantFail(JTMB.createTargetMachine()); + SimpleCompiler C(*TM, ObjCache); + return C(M); +} + +} // end namespace orc +} // end namespace llvm diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp index 73c0bcdf7d28..dac37e030e0c 100644 --- a/lib/ExecutionEngine/Orc/Core.cpp +++ b/lib/ExecutionEngine/Orc/Core.cpp @@ -1,9 +1,8 @@ //===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,17 +26,17 @@ namespace { #ifndef NDEBUG -cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(false), +cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(true), cl::desc("debug print hidden symbols defined by " "materialization units"), cl::Hidden); -cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(false), +cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(true), cl::desc("debug print callable symbols defined by " "materialization units"), cl::Hidden); -cl::opt<bool> PrintData("debug-orc-print-data", cl::init(false), +cl::opt<bool> PrintData("debug-orc-print-data", cl::init(true), cl::desc("debug print data symbols defined by " "materialization units"), cl::Hidden); @@ -134,8 +133,6 @@ struct PrintSymbolMapElemsMatchingCLOpts { namespace llvm { namespace orc { - SymbolStringPool::PoolMapEntry SymbolStringPtr::Tombstone(0); - char FailedToMaterialize::ID = 0; char SymbolsNotFound::ID = 0; char SymbolsCouldNotBeRemoved::ID = 0; @@ -222,6 +219,31 @@ raw_ostream &operator<<(raw_ostream &OS, const JITDylibSearchList &JDs) { return OS; } +raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) { + OS << "{"; + for (auto &KV : Aliases) + OS << " " << *KV.first << ": " << KV.second.Aliasee << " " + << KV.second.AliasFlags; + OS << " }\n"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { + switch (S) { + case SymbolState::Invalid: + return OS << "Invalid"; + case SymbolState::NeverSearched: + return OS << "Never-Searched"; + case SymbolState::Materializing: + return OS << "Materializing"; + case SymbolState::Resolved: + return OS << "Resolved"; + case SymbolState::Ready: + return OS << "Ready"; + } + llvm_unreachable("Invalid state"); +} + FailedToMaterialize::FailedToMaterialize(SymbolNameSet Symbols) : Symbols(std::move(Symbols)) { assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); @@ -262,85 +284,46 @@ void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const { } AsynchronousSymbolQuery::AsynchronousSymbolQuery( - const SymbolNameSet &Symbols, SymbolsResolvedCallback NotifySymbolsResolved, - SymbolsReadyCallback NotifySymbolsReady) - : NotifySymbolsResolved(std::move(NotifySymbolsResolved)), - NotifySymbolsReady(std::move(NotifySymbolsReady)) { - NotYetResolvedCount = NotYetReadyCount = Symbols.size(); + const SymbolNameSet &Symbols, SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete) + : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) { + assert(RequiredState >= SymbolState::Resolved && + "Cannot query for a symbols that have not reached the resolve state " + "yet"); + + OutstandingSymbolsCount = Symbols.size(); for (auto &S : Symbols) ResolvedSymbols[S] = nullptr; } -void AsynchronousSymbolQuery::resolve(const SymbolStringPtr &Name, - JITEvaluatedSymbol Sym) { +void AsynchronousSymbolQuery::notifySymbolMetRequiredState( + const SymbolStringPtr &Name, JITEvaluatedSymbol Sym) { auto I = ResolvedSymbols.find(Name); assert(I != ResolvedSymbols.end() && "Resolving symbol outside the requested set"); assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); I->second = std::move(Sym); - --NotYetResolvedCount; -} - -void AsynchronousSymbolQuery::handleFullyResolved() { - assert(NotYetResolvedCount == 0 && "Not fully resolved?"); - - if (!NotifySymbolsResolved) { - // handleFullyResolved may be called by handleFullyReady (see comments in - // that method), in which case this is a no-op, so bail out. - assert(!NotifySymbolsReady && - "NotifySymbolsResolved already called or an error occurred"); - return; - } - - auto TmpNotifySymbolsResolved = std::move(NotifySymbolsResolved); - NotifySymbolsResolved = SymbolsResolvedCallback(); - TmpNotifySymbolsResolved(std::move(ResolvedSymbols)); -} - -void AsynchronousSymbolQuery::notifySymbolReady() { - assert(NotYetReadyCount != 0 && "All symbols already emitted"); - --NotYetReadyCount; + --OutstandingSymbolsCount; } -void AsynchronousSymbolQuery::handleFullyReady() { - assert(NotifySymbolsReady && - "NotifySymbolsReady already called or an error occurred"); +void AsynchronousSymbolQuery::handleComplete() { + assert(OutstandingSymbolsCount == 0 && + "Symbols remain, handleComplete called prematurely"); - auto TmpNotifySymbolsReady = std::move(NotifySymbolsReady); - NotifySymbolsReady = SymbolsReadyCallback(); - - if (NotYetResolvedCount == 0 && NotifySymbolsResolved) { - // The NotifyResolved callback of one query must have caused this query to - // become ready (i.e. there is still a handleFullyResolved callback waiting - // to be made back up the stack). Fold the handleFullyResolved call into - // this one before proceeding. This will cause the call further up the - // stack to become a no-op. - handleFullyResolved(); - } - - assert(QueryRegistrations.empty() && - "Query is still registered with some symbols"); - assert(!NotifySymbolsResolved && "Resolution not applied yet"); - TmpNotifySymbolsReady(Error::success()); + auto TmpNotifyComplete = std::move(NotifyComplete); + NotifyComplete = SymbolsResolvedCallback(); + TmpNotifyComplete(std::move(ResolvedSymbols)); } -bool AsynchronousSymbolQuery::canStillFail() { - return (NotifySymbolsResolved || NotifySymbolsReady); -} +bool AsynchronousSymbolQuery::canStillFail() { return !!NotifyComplete; } void AsynchronousSymbolQuery::handleFailed(Error Err) { assert(QueryRegistrations.empty() && ResolvedSymbols.empty() && - NotYetResolvedCount == 0 && NotYetReadyCount == 0 && + OutstandingSymbolsCount == 0 && "Query should already have been abandoned"); - if (NotifySymbolsResolved) { - NotifySymbolsResolved(std::move(Err)); - NotifySymbolsResolved = SymbolsResolvedCallback(); - } else { - assert(NotifySymbolsReady && "Failed after both callbacks issued?"); - NotifySymbolsReady(std::move(Err)); - } - NotifySymbolsReady = SymbolsReadyCallback(); + NotifyComplete(std::move(Err)); + NotifyComplete = SymbolsResolvedCallback(); } void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD, @@ -363,8 +346,7 @@ void AsynchronousSymbolQuery::removeQueryDependence( void AsynchronousSymbolQuery::detach() { ResolvedSymbols.clear(); - NotYetResolvedCount = 0; - NotYetReadyCount = 0; + OutstandingSymbolsCount = 0; for (auto &KV : QueryRegistrations) KV.first->detachQueryHelper(*this, KV.second); QueryRegistrations.clear(); @@ -374,11 +356,6 @@ MaterializationResponsibility::MaterializationResponsibility( JITDylib &JD, SymbolFlagsMap SymbolFlags, VModuleKey K) : JD(JD), SymbolFlags(std::move(SymbolFlags)), K(std::move(K)) { assert(!this->SymbolFlags.empty() && "Materializing nothing?"); - -#ifndef NDEBUG - for (auto &KV : this->SymbolFlags) - KV.second |= JITSymbolFlags::Materializing; -#endif } MaterializationResponsibility::~MaterializationResponsibility() { @@ -390,16 +367,15 @@ SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { return JD.getRequestedSymbols(SymbolFlags); } -void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { - LLVM_DEBUG(dbgs() << "In " << JD.getName() << " resolving " << Symbols - << "\n"); +void MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " resolving " << Symbols << "\n"; + }); #ifndef NDEBUG for (auto &KV : Symbols) { auto I = SymbolFlags.find(KV.first); assert(I != SymbolFlags.end() && "Resolving symbol outside this responsibility set"); - assert(I->second.isMaterializing() && "Duplicate resolution"); - I->second &= ~JITSymbolFlags::Materializing; if (I->second.isWeak()) assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) && "Resolving symbol with incorrect flags"); @@ -412,12 +388,11 @@ void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { JD.resolve(Symbols); } -void MaterializationResponsibility::emit() { -#ifndef NDEBUG - for (auto &KV : SymbolFlags) - assert(!KV.second.isMaterializing() && - "Failed to resolve symbol before emission"); -#endif // NDEBUG +void MaterializationResponsibility::notifyEmitted() { + + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " emitting " << SymbolFlags << "\n"; + }); JD.emit(SymbolFlags); SymbolFlags.clear(); @@ -429,19 +404,19 @@ Error MaterializationResponsibility::defineMaterializing( // It's ok if we hit a duplicate here: In that case the new version will be // discarded, and the JITDylib::defineMaterializing method will return a // duplicate symbol error. - for (auto &KV : NewSymbolFlags) { - auto I = SymbolFlags.insert(KV).first; - (void)I; -#ifndef NDEBUG - I->second |= JITSymbolFlags::Materializing; -#endif - } + for (auto &KV : NewSymbolFlags) + SymbolFlags.insert(KV); return JD.defineMaterializing(NewSymbolFlags); } void MaterializationResponsibility::failMaterialization() { + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " failing materialization for " + << SymbolFlags << "\n"; + }); + SymbolNameSet FailedSymbols; for (auto &KV : SymbolFlags) FailedSymbols.insert(KV.first); @@ -510,8 +485,8 @@ StringRef AbsoluteSymbolsMaterializationUnit::getName() const { void AbsoluteSymbolsMaterializationUnit::materialize( MaterializationResponsibility R) { - R.resolve(Symbols); - R.emit(); + R.notifyResolved(Symbols); + R.notifyEmitted(); } void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD, @@ -559,6 +534,14 @@ void ReExportsMaterializationUnit::materialize( Aliases.erase(I); } + LLVM_DEBUG({ + ES.runSessionLocked([&]() { + dbgs() << "materializing reexports: target = " << TgtJD.getName() + << ", source = " << SrcJD.getName() << " " << RequestedAliases + << "\n"; + }); + }); + if (!Aliases.empty()) { if (SourceJD) R.replace(reexports(*SourceJD, std::move(Aliases), MatchNonExported)); @@ -641,7 +624,7 @@ void ReExportsMaterializationUnit::materialize( } }; - auto OnResolve = [QueryInfo](Expected<SymbolMap> Result) { + auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) { if (Result) { SymbolMap ResolutionMap; for (auto &KV : QueryInfo->Aliases) { @@ -650,8 +633,8 @@ void ReExportsMaterializationUnit::materialize( ResolutionMap[KV.first] = JITEvaluatedSymbol( (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); } - QueryInfo->R.resolve(ResolutionMap); - QueryInfo->R.emit(); + QueryInfo->R.notifyResolved(ResolutionMap); + QueryInfo->R.notifyEmitted(); } else { auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession(); ES.reportError(Result.takeError()); @@ -659,10 +642,8 @@ void ReExportsMaterializationUnit::materialize( } }; - auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; - ES.lookup(JITDylibSearchList({{&SrcJD, MatchNonExported}}), QuerySymbols, - std::move(OnResolve), std::move(OnReady), + SymbolState::Resolved, std::move(OnComplete), std::move(RegisterDependencies)); } } @@ -687,17 +668,20 @@ Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { auto Flags = SourceJD.lookupFlags(Symbols); - if (Flags.size() != Symbols.size()) { + if (!Flags) + return Flags.takeError(); + + if (Flags->size() != Symbols.size()) { SymbolNameSet Unresolved = Symbols; - for (auto &KV : Flags) + for (auto &KV : *Flags) Unresolved.erase(KV.first); return make_error<SymbolsNotFound>(std::move(Unresolved)); } SymbolAliasMap Result; for (auto &Name : Symbols) { - assert(Flags.count(Name) && "Missing entry in flags map"); - Result[Name] = SymbolAliasMapEntry(Name, Flags[Name]); + assert(Flags->count(Name) && "Missing entry in flags map"); + Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]); } return Result; @@ -709,14 +693,17 @@ ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, : SourceJD(SourceJD), MatchNonExported(MatchNonExported), Allow(std::move(Allow)) {} -SymbolNameSet ReexportsGenerator::operator()(JITDylib &JD, - const SymbolNameSet &Names) { +Expected<SymbolNameSet> +ReexportsGenerator::operator()(JITDylib &JD, const SymbolNameSet &Names) { orc::SymbolNameSet Added; orc::SymbolAliasMap AliasMap; auto Flags = SourceJD.lookupFlags(Names); - for (auto &KV : Flags) { + if (!Flags) + return Flags.takeError(); + + for (auto &KV : *Flags) { if (Allow && !Allow(KV.first)) continue; AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second); @@ -731,21 +718,19 @@ SymbolNameSet ReexportsGenerator::operator()(JITDylib &JD, Error JITDylib::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { return ES.runSessionLocked([&]() -> Error { - std::vector<SymbolMap::iterator> AddedSyms; + std::vector<SymbolTable::iterator> AddedSyms; for (auto &KV : SymbolFlags) { - SymbolMap::iterator EntryItr; + SymbolTable::iterator EntryItr; bool Added; - auto NewFlags = KV.second; - NewFlags |= JITSymbolFlags::Materializing; - - std::tie(EntryItr, Added) = Symbols.insert( - std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); + std::tie(EntryItr, Added) = + Symbols.insert(std::make_pair(KV.first, SymbolTableEntry(KV.second))); - if (Added) + if (Added) { AddedSyms.push_back(EntryItr); - else { + EntryItr->second.setState(SymbolState::Materializing); + } else { // Remove any symbols already added. for (auto &SI : AddedSyms) Symbols.erase(SI); @@ -769,9 +754,10 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { for (auto &KV : MU->getSymbols()) { auto SymI = Symbols.find(KV.first); assert(SymI != Symbols.end() && "Replacing unknown symbol"); - assert(!SymI->second.getFlags().isLazy() && - SymI->second.getFlags().isMaterializing() && - "Can not replace symbol that is not materializing"); + assert(SymI->second.isInMaterializationPhase() && + "Can not call replace on a symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Symbol should not have materializer attached already"); assert(UnmaterializedInfos.count(KV.first) == 0 && "Symbol being replaced should have no UnmaterializedInfo"); } @@ -782,7 +768,7 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { for (auto &KV : MU->getSymbols()) { auto MII = MaterializingInfos.find(KV.first); if (MII != MaterializingInfos.end()) { - if (!MII->second.PendingQueries.empty()) + if (MII->second.hasQueriesPending()) return std::move(MU); } } @@ -790,16 +776,15 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { // Otherwise, make MU responsible for all the symbols. auto UMI = std::make_shared<UnmaterializedInfo>(std::move(MU)); for (auto &KV : UMI->MU->getSymbols()) { - assert(!KV.second.isLazy() && - "Lazy flag should be managed internally."); - assert(!KV.second.isMaterializing() && - "Materializing flags should be managed internally."); - auto SymI = Symbols.find(KV.first); - JITSymbolFlags ReplaceFlags = KV.second; - ReplaceFlags |= JITSymbolFlags::Lazy; - SymI->second = JITEvaluatedSymbol(SymI->second.getAddress(), - std::move(ReplaceFlags)); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace a symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Can not replace a symbol that has a materializer attached"); + assert(UnmaterializedInfos.count(KV.first) == 0 && + "Unexpected materializer entry in map"); + SymI->second.setAddress(SymI->second.getAddress()); + SymI->second.setMaterializerAttached(true); UnmaterializedInfos[KV.first] = UMI; } @@ -817,14 +802,14 @@ JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { for (auto &KV : SymbolFlags) { assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?"); - assert(Symbols.find(KV.first)->second.getFlags().isMaterializing() && - "getRequestedSymbols can only be called for materializing " - "symbols"); + assert(Symbols.find(KV.first)->second.isInMaterializationPhase() && + "getRequestedSymbols can only be called for symbols that have " + "started materializing"); auto I = MaterializingInfos.find(KV.first); if (I == MaterializingInfos.end()) continue; - if (!I->second.PendingQueries.empty()) + if (I->second.hasQueriesPending()) RequestedSymbols.insert(KV.first); } @@ -835,9 +820,8 @@ JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { void JITDylib::addDependencies(const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { assert(Symbols.count(Name) && "Name not in symbol table"); - assert((Symbols[Name].getFlags().isLazy() || - Symbols[Name].getFlags().isMaterializing()) && - "Symbol is not lazy or materializing"); + assert(Symbols[Name].isInMaterializationPhase() && + "Can not add dependencies for a symbol that is not materializing"); auto &MI = MaterializingInfos[Name]; assert(!MI.IsEmitted && "Can not add dependencies to an emitted symbol"); @@ -852,9 +836,8 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, // Assert that this symbol exists and has not been emitted already. auto SymI = OtherJITDylib.Symbols.find(OtherSymbol); assert(SymI != OtherJITDylib.Symbols.end() && - (SymI->second.getFlags().isLazy() || - SymI->second.getFlags().isMaterializing()) && - "Dependency on emitted symbol"); + (SymI->second.getState() != SymbolState::Ready && + "Dependency on emitted symbol")); #endif auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; @@ -873,54 +856,52 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, } void JITDylib::resolve(const SymbolMap &Resolved) { - auto FullyResolvedQueries = ES.runSessionLocked([&, this]() { - AsynchronousSymbolQuerySet FullyResolvedQueries; + auto CompletedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet CompletedQueries; for (const auto &KV : Resolved) { auto &Name = KV.first; auto Sym = KV.second; - assert(!Sym.getFlags().isLazy() && !Sym.getFlags().isMaterializing() && - "Materializing flags should be managed internally"); - auto I = Symbols.find(Name); assert(I != Symbols.end() && "Symbol not found"); - assert(!I->second.getFlags().isLazy() && - I->second.getFlags().isMaterializing() && + assert(!I->second.hasMaterializerAttached() && + "Resolving symbol with materializer attached?"); + assert(I->second.getState() == SymbolState::Materializing && "Symbol should be materializing"); assert(I->second.getAddress() == 0 && "Symbol has already been resolved"); assert((Sym.getFlags() & ~JITSymbolFlags::Weak) == - (JITSymbolFlags::stripTransientFlags(I->second.getFlags()) & - ~JITSymbolFlags::Weak) && + (I->second.getFlags() & ~JITSymbolFlags::Weak) && "Resolved flags should match the declared flags"); // Once resolved, symbols can never be weak. JITSymbolFlags ResolvedFlags = Sym.getFlags(); ResolvedFlags &= ~JITSymbolFlags::Weak; - ResolvedFlags |= JITSymbolFlags::Materializing; - I->second = JITEvaluatedSymbol(Sym.getAddress(), ResolvedFlags); + I->second.setAddress(Sym.getAddress()); + I->second.setFlags(ResolvedFlags); + I->second.setState(SymbolState::Resolved); auto &MI = MaterializingInfos[Name]; - for (auto &Q : MI.PendingQueries) { - Q->resolve(Name, Sym); - if (Q->isFullyResolved()) - FullyResolvedQueries.insert(Q); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { + Q->notifySymbolMetRequiredState(Name, Sym); + if (Q->isComplete()) + CompletedQueries.insert(std::move(Q)); } } - return FullyResolvedQueries; + return CompletedQueries; }); - for (auto &Q : FullyResolvedQueries) { - assert(Q->isFullyResolved() && "Q not fully resolved"); - Q->handleFullyResolved(); + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q not completed"); + Q->handleComplete(); } } void JITDylib::emit(const SymbolFlagsMap &Emitted) { - auto FullyReadyQueries = ES.runSessionLocked([&, this]() { - AsynchronousSymbolQuerySet ReadyQueries; + auto CompletedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet CompletedQueries; for (const auto &KV : Emitted) { const auto &Name = KV.first; @@ -962,20 +943,22 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { DependantMI.UnemittedDependencies.empty()) { assert(DependantMI.Dependants.empty() && "Dependants should be empty by now"); - for (auto &Q : DependantMI.PendingQueries) { - Q->notifySymbolReady(); - if (Q->isFullyReady()) - ReadyQueries.insert(Q); - Q->removeQueryDependence(DependantJD, DependantName); - } // Since this dependant is now ready, we erase its MaterializingInfo // and update its materializing state. - assert(DependantJD.Symbols.count(DependantName) && + auto DependantSymI = DependantJD.Symbols.find(DependantName); + assert(DependantSymI != DependantJD.Symbols.end() && "Dependant has no entry in the Symbols table"); - auto &DependantSym = DependantJD.Symbols[DependantName]; - DependantSym.setFlags(DependantSym.getFlags() & - ~JITSymbolFlags::Materializing); + DependantSymI->second.setState(SymbolState::Ready); + + for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState( + DependantName, DependantSymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); + Q->removeQueryDependence(DependantJD, DependantName); + } + DependantJD.MaterializingInfos.erase(DependantMII); } } @@ -984,26 +967,25 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { MI.IsEmitted = true; if (MI.UnemittedDependencies.empty()) { - for (auto &Q : MI.PendingQueries) { - Q->notifySymbolReady(); - if (Q->isFullyReady()) - ReadyQueries.insert(Q); + auto SymI = Symbols.find(Name); + assert(SymI != Symbols.end() && "Symbol has no entry in Symbols table"); + SymI->second.setState(SymbolState::Ready); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); Q->removeQueryDependence(*this, Name); } - assert(Symbols.count(Name) && - "Symbol has no entry in the Symbols table"); - auto &Sym = Symbols[Name]; - Sym.setFlags(Sym.getFlags() & ~JITSymbolFlags::Materializing); MaterializingInfos.erase(MII); } } - return ReadyQueries; + return CompletedQueries; }); - for (auto &Q : FullyReadyQueries) { - assert(Q->isFullyReady() && "Q is not fully ready"); - Q->handleFullyReady(); + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q is not complete"); + Q->handleComplete(); } } @@ -1013,6 +995,7 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { auto FailedQueriesToNotify = ES.runSessionLocked([&, this]() { AsynchronousSymbolQuerySet FailedQueries; + std::vector<MaterializingInfosMap::iterator> MIIsToRemove; for (auto &Name : FailedSymbols) { auto I = Symbols.find(Name); @@ -1026,17 +1009,40 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { if (MII == MaterializingInfos.end()) continue; + // Remove this symbol from the dependants list of any dependencies. + for (auto &KV : MII->second.UnemittedDependencies) { + auto *DependencyJD = KV.first; + auto &Dependencies = KV.second; + for (auto &DependencyName : Dependencies) { + auto DependencyMII = + DependencyJD->MaterializingInfos.find(DependencyName); + assert(DependencyMII != DependencyJD->MaterializingInfos.end() && + "Unemitted dependency must have a MaterializingInfo entry"); + assert(DependencyMII->second.Dependants.count(this) && + "Dependency's dependants list does not contain this JITDylib"); + assert(DependencyMII->second.Dependants[this].count(Name) && + "Dependency's dependants list does not contain dependant"); + DependencyMII->second.Dependants[this].erase(Name); + } + } + // Copy all the queries to the FailedQueries list, then abandon them. // This has to be a copy, and the copy has to come before the abandon // operation: Each Q.detach() call will reach back into this // PendingQueries list to remove Q. - for (auto &Q : MII->second.PendingQueries) + for (auto &Q : MII->second.pendingQueries()) FailedQueries.insert(Q); - for (auto &Q : FailedQueries) - Q->detach(); + MIIsToRemove.push_back(std::move(MII)); + } + + // Detach failed queries. + for (auto &Q : FailedQueries) + Q->detach(); - assert(MII->second.PendingQueries.empty() && + // Remove the MaterializingInfos. + for (auto &MII : MIIsToRemove) { + assert(!MII->second.hasQueriesPending() && "Queries remain after symbol was failed"); MaterializingInfos.erase(MII); @@ -1052,9 +1058,11 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder, bool SearchThisJITDylibFirst, bool MatchNonExportedInThisDylib) { - if (SearchThisJITDylibFirst && NewSearchOrder.front().first != this) - NewSearchOrder.insert(NewSearchOrder.begin(), - {this, MatchNonExportedInThisDylib}); + if (SearchThisJITDylibFirst) { + if (NewSearchOrder.empty() || NewSearchOrder.front().first != this) + NewSearchOrder.insert(NewSearchOrder.begin(), + {this, MatchNonExportedInThisDylib}); + } ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); } @@ -1092,7 +1100,7 @@ void JITDylib::removeFromSearchOrder(JITDylib &JD) { Error JITDylib::remove(const SymbolNameSet &Names) { return ES.runSessionLocked([&]() -> Error { using SymbolMaterializerItrPair = - std::pair<SymbolMap::iterator, UnmaterializedInfosMap::iterator>; + std::pair<SymbolTable::iterator, UnmaterializedInfosMap::iterator>; std::vector<SymbolMaterializerItrPair> SymbolsToRemove; SymbolNameSet Missing; SymbolNameSet Materializing; @@ -1107,13 +1115,14 @@ Error JITDylib::remove(const SymbolNameSet &Names) { } // Note symbol materializing. - if (I->second.getFlags().isMaterializing()) { + if (I->second.isInMaterializationPhase()) { Materializing.insert(Name); continue; } - auto UMII = I->second.getFlags().isLazy() ? UnmaterializedInfos.find(Name) - : UnmaterializedInfos.end(); + auto UMII = I->second.hasMaterializerAttached() + ? UnmaterializedInfos.find(Name) + : UnmaterializedInfos.end(); SymbolsToRemove.push_back(std::make_pair(I, UMII)); } @@ -1143,16 +1152,23 @@ Error JITDylib::remove(const SymbolNameSet &Names) { }); } -SymbolFlagsMap JITDylib::lookupFlags(const SymbolNameSet &Names) { - return ES.runSessionLocked([&, this]() { +Expected<SymbolFlagsMap> JITDylib::lookupFlags(const SymbolNameSet &Names) { + return ES.runSessionLocked([&, this]() -> Expected<SymbolFlagsMap> { SymbolFlagsMap Result; auto Unresolved = lookupFlagsImpl(Result, Names); - if (DefGenerator && !Unresolved.empty()) { - auto NewDefs = DefGenerator(*this, Unresolved); - if (!NewDefs.empty()) { - auto Unresolved2 = lookupFlagsImpl(Result, NewDefs); + if (!Unresolved) + return Unresolved.takeError(); + + if (DefGenerator && !Unresolved->empty()) { + auto NewDefs = DefGenerator(*this, *Unresolved); + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + auto Unresolved2 = lookupFlagsImpl(Result, *NewDefs); + if (!Unresolved2) + return Unresolved2.takeError(); (void)Unresolved2; - assert(Unresolved2.empty() && + assert(Unresolved2->empty() && "All fallback defs should have been found by lookupFlagsImpl"); } }; @@ -1160,41 +1176,42 @@ SymbolFlagsMap JITDylib::lookupFlags(const SymbolNameSet &Names) { }); } -SymbolNameSet JITDylib::lookupFlagsImpl(SymbolFlagsMap &Flags, - const SymbolNameSet &Names) { +Expected<SymbolNameSet> JITDylib::lookupFlagsImpl(SymbolFlagsMap &Flags, + const SymbolNameSet &Names) { SymbolNameSet Unresolved; for (auto &Name : Names) { auto I = Symbols.find(Name); - - if (I == Symbols.end()) { + if (I != Symbols.end()) { + assert(!Flags.count(Name) && "Symbol already present in Flags map"); + Flags[Name] = I->second.getFlags(); + } else Unresolved.insert(Name); - continue; - } - - assert(!Flags.count(Name) && "Symbol already present in Flags map"); - Flags[Name] = JITSymbolFlags::stripTransientFlags(I->second.getFlags()); } return Unresolved; } -void JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs) { +Error JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, bool MatchNonExported, + MaterializationUnitList &MUs) { assert(Q && "Query can not be null"); lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs); if (DefGenerator && !Unresolved.empty()) { auto NewDefs = DefGenerator(*this, Unresolved); - if (!NewDefs.empty()) { - for (auto &D : NewDefs) + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + for (auto &D : *NewDefs) Unresolved.erase(D); - lodgeQueryImpl(Q, NewDefs, MatchNonExported, MUs); - assert(NewDefs.empty() && + lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs); + assert(NewDefs->empty() && "All fallback defs should have been found by lookupImpl"); } } + + return Error::success(); } void JITDylib::lodgeQueryImpl( @@ -1204,6 +1221,7 @@ void JITDylib::lodgeQueryImpl( std::vector<SymbolStringPtr> ToRemove; for (auto Name : Unresolved) { + // Search for the name in Symbols. Skip it if not found. auto SymI = Symbols.find(Name); if (SymI == Symbols.end()) @@ -1213,20 +1231,22 @@ void JITDylib::lodgeQueryImpl( if (!SymI->second.getFlags().isExported() && !MatchNonExported) continue; - // If we matched against Name in JD, mark it to be removed from the Unresolved - // set. + // If we matched against Name in JD, mark it to be removed from the + // Unresolved set. ToRemove.push_back(Name); - // If the symbol has an address then resolve it. - if (SymI->second.getAddress() != 0) - Q->resolve(Name, SymI->second); + // If this symbol already meets the required state for then notify the + // query and continue. + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + continue; + } - // If the symbol is lazy, get the MaterialiaztionUnit for it. - if (SymI->second.getFlags().isLazy()) { + // Otherwise this symbol does not yet meet the required state. Check whether + // it has a materializer attached, and if so prepare to run it. + if (SymI->second.hasMaterializerAttached()) { assert(SymI->second.getAddress() == 0 && - "Lazy symbol should not have a resolved address"); - assert(!SymI->second.getFlags().isMaterializing() && - "Materializing and lazy should not both be set"); + "Symbol not resolved but already has address?"); auto UMII = UnmaterializedInfos.find(Name); assert(UMII != UnmaterializedInfos.end() && "Lazy symbol should have UnmaterializedInfo"); @@ -1237,27 +1257,20 @@ void JITDylib::lodgeQueryImpl( // materializing state. for (auto &KV : MU->getSymbols()) { auto SymK = Symbols.find(KV.first); - auto Flags = SymK->second.getFlags(); - Flags &= ~JITSymbolFlags::Lazy; - Flags |= JITSymbolFlags::Materializing; - SymK->second.setFlags(Flags); + SymK->second.setMaterializerAttached(false); + SymK->second.setState(SymbolState::Materializing); UnmaterializedInfos.erase(KV.first); } // Add MU to the list of MaterializationUnits to be materialized. MUs.push_back(std::move(MU)); - } else if (!SymI->second.getFlags().isMaterializing()) { - // The symbol is neither lazy nor materializing, so it must be - // ready. Notify the query and continue. - Q->notifySymbolReady(); - continue; } // Add the query to the PendingQueries list. - assert(SymI->second.getFlags().isMaterializing() && + assert(SymI->second.isInMaterializationPhase() && "By this line the symbol should be materializing"); auto &MI = MaterializingInfos[Name]; - MI.PendingQueries.push_back(Q); + MI.addQuery(Q); Q->addQueryDependence(*this, Name); } @@ -1266,40 +1279,43 @@ void JITDylib::lodgeQueryImpl( Unresolved.erase(Name); } -SymbolNameSet JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, - SymbolNameSet Names) { +Expected<SymbolNameSet> +JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, + SymbolNameSet Names) { assert(Q && "Query can not be null"); ES.runOutstandingMUs(); - LookupImplActionFlags ActionFlags = None; + bool QueryComplete = false; std::vector<std::unique_ptr<MaterializationUnit>> MUs; SymbolNameSet Unresolved = std::move(Names); - ES.runSessionLocked([&, this]() { - ActionFlags = lookupImpl(Q, MUs, Unresolved); + auto Err = ES.runSessionLocked([&, this]() -> Error { + QueryComplete = lookupImpl(Q, MUs, Unresolved); if (DefGenerator && !Unresolved.empty()) { - assert(ActionFlags == None && - "ActionFlags set but unresolved symbols remain?"); + assert(!QueryComplete && "query complete but unresolved symbols remain?"); auto NewDefs = DefGenerator(*this, Unresolved); - if (!NewDefs.empty()) { - for (auto &D : NewDefs) + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + for (auto &D : *NewDefs) Unresolved.erase(D); - ActionFlags = lookupImpl(Q, MUs, NewDefs); - assert(NewDefs.empty() && + QueryComplete = lookupImpl(Q, MUs, *NewDefs); + assert(NewDefs->empty() && "All fallback defs should have been found by lookupImpl"); } } + return Error::success(); }); - assert((MUs.empty() || ActionFlags == None) && - "If action flags are set, there should be no work to do (so no MUs)"); + if (Err) + return std::move(Err); - if (ActionFlags & NotifyFullyResolved) - Q->handleFullyResolved(); + assert((MUs.empty() || !QueryComplete) && + "If action flags are set, there should be no work to do (so no MUs)"); - if (ActionFlags & NotifyFullyReady) - Q->handleFullyReady(); + if (QueryComplete) + Q->handleComplete(); // FIXME: Swap back to the old code below once RuntimeDyld works with // callbacks from asynchronous queries. @@ -1318,13 +1334,13 @@ SymbolNameSet JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, return Unresolved; } -JITDylib::LookupImplActionFlags -JITDylib::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, - std::vector<std::unique_ptr<MaterializationUnit>> &MUs, - SymbolNameSet &Unresolved) { - LookupImplActionFlags ActionFlags = None; - std::vector<SymbolStringPtr> ToRemove; +bool JITDylib::lookupImpl( + std::shared_ptr<AsynchronousSymbolQuery> &Q, + std::vector<std::unique_ptr<MaterializationUnit>> &MUs, + SymbolNameSet &Unresolved) { + bool QueryComplete = false; + std::vector<SymbolStringPtr> ToRemove; for (auto Name : Unresolved) { // Search for the name in Symbols. Skip it if not found. @@ -1335,19 +1351,17 @@ JITDylib::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, // If we found Name, mark it to be removed from the Unresolved set. ToRemove.push_back(Name); - // If the symbol has an address then resolve it. - if (SymI->second.getAddress() != 0) { - Q->resolve(Name, SymI->second); - if (Q->isFullyResolved()) - ActionFlags |= NotifyFullyResolved; + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + QueryComplete = true; + continue; } // If the symbol is lazy, get the MaterialiaztionUnit for it. - if (SymI->second.getFlags().isLazy()) { + if (SymI->second.hasMaterializerAttached()) { assert(SymI->second.getAddress() == 0 && "Lazy symbol should not have a resolved address"); - assert(!SymI->second.getFlags().isMaterializing() && - "Materializing and lazy should not both be set"); auto UMII = UnmaterializedInfos.find(Name); assert(UMII != UnmaterializedInfos.end() && "Lazy symbol should have UnmaterializedInfo"); @@ -1358,29 +1372,21 @@ JITDylib::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, // materializing state. for (auto &KV : MU->getSymbols()) { auto SymK = Symbols.find(KV.first); - auto Flags = SymK->second.getFlags(); - Flags &= ~JITSymbolFlags::Lazy; - Flags |= JITSymbolFlags::Materializing; - SymK->second.setFlags(Flags); + assert(SymK != Symbols.end() && "Missing symbol table entry"); + SymK->second.setState(SymbolState::Materializing); + SymK->second.setMaterializerAttached(false); UnmaterializedInfos.erase(KV.first); } // Add MU to the list of MaterializationUnits to be materialized. MUs.push_back(std::move(MU)); - } else if (!SymI->second.getFlags().isMaterializing()) { - // The symbol is neither lazy nor materializing, so it must be ready. - // Notify the query and continue. - Q->notifySymbolReady(); - if (Q->isFullyReady()) - ActionFlags |= NotifyFullyReady; - continue; } // Add the query to the PendingQueries list. - assert(SymI->second.getFlags().isMaterializing() && + assert(SymI->second.isInMaterializationPhase() && "By this line the symbol should be materializing"); auto &MI = MaterializingInfos[Name]; - MI.PendingQueries.push_back(Q); + MI.addQuery(Q); Q->addQueryDependence(*this, Name); } @@ -1388,7 +1394,7 @@ JITDylib::lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, for (auto &Name : ToRemove) Unresolved.erase(Name); - return ActionFlags; + return QueryComplete; } void JITDylib::dump(raw_ostream &OS) { @@ -1405,21 +1411,19 @@ void JITDylib::dump(raw_ostream &OS) { for (auto &KV : Symbols) { OS << " \"" << *KV.first << "\": "; if (auto Addr = KV.second.getAddress()) - OS << format("0x%016" PRIx64, Addr) << ", " << KV.second.getFlags(); + OS << format("0x%016" PRIx64, Addr) << ", " << KV.second.getFlags() + << " "; else - OS << "<not resolved>"; - if (KV.second.getFlags().isLazy() || - KV.second.getFlags().isMaterializing()) { - OS << " ("; - if (KV.second.getFlags().isLazy()) { - auto I = UnmaterializedInfos.find(KV.first); - assert(I != UnmaterializedInfos.end() && - "Lazy symbol should have UnmaterializedInfo"); - OS << " Lazy (MU=" << I->second->MU.get() << ")"; - } - if (KV.second.getFlags().isMaterializing()) - OS << " Materializing"; - OS << ", " << KV.second.getFlags() << " )\n"; + OS << "<not resolved> "; + + OS << KV.second.getState(); + + if (KV.second.hasMaterializerAttached()) { + OS << " (Materializer "; + auto I = UnmaterializedInfos.find(KV.first); + assert(I != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + OS << I->second->MU.get() << ")\n"; } else OS << "\n"; } @@ -1430,10 +1434,10 @@ void JITDylib::dump(raw_ostream &OS) { OS << " \"" << *KV.first << "\":\n" << " IsEmitted = " << (KV.second.IsEmitted ? "true" : "false") << "\n" - << " " << KV.second.PendingQueries.size() + << " " << KV.second.pendingQueries().size() << " pending queries: { "; - for (auto &Q : KV.second.PendingQueries) - OS << Q.get() << " "; + for (const auto &Q : KV.second.pendingQueries()) + OS << Q.get() << " (" << Q->getRequiredState() << ") "; OS << "}\n Dependants:\n"; for (auto &KV2 : KV.second.Dependants) OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; @@ -1444,6 +1448,51 @@ void JITDylib::dump(raw_ostream &OS) { }); } +void JITDylib::MaterializingInfo::addQuery( + std::shared_ptr<AsynchronousSymbolQuery> Q) { + + auto I = std::lower_bound( + PendingQueries.rbegin(), PendingQueries.rend(), Q->getRequiredState(), + [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) { + return V->getRequiredState() <= S; + }); + PendingQueries.insert(I.base(), std::move(Q)); +} + +void JITDylib::MaterializingInfo::removeQuery( + const AsynchronousSymbolQuery &Q) { + // FIXME: Implement 'find_as' for shared_ptr<T>/T*. + auto I = + std::find_if(PendingQueries.begin(), PendingQueries.end(), + [&Q](const std::shared_ptr<AsynchronousSymbolQuery> &V) { + return V.get() == &Q; + }); + assert(I != PendingQueries.end() && + "Query is not attached to this MaterializingInfo"); + PendingQueries.erase(I); +} + +JITDylib::AsynchronousSymbolQueryList +JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) { + AsynchronousSymbolQueryList Result; + while (!PendingQueries.empty()) { + if (PendingQueries.back()->getRequiredState() > RequiredState) + break; + + Result.push_back(std::move(PendingQueries.back())); + PendingQueries.pop_back(); + } + + return Result; +} + +JITDylib::AsynchronousSymbolQueryList +JITDylib::MaterializingInfo::takeAllQueries() { + AsynchronousSymbolQueryList Result; + std::swap(Result, PendingQueries); + return Result; +} + JITDylib::JITDylib(ExecutionSession &ES, std::string Name) : ES(ES), JITDylibName(std::move(Name)) { SearchOrder.push_back({this, true}); @@ -1451,77 +1500,52 @@ JITDylib::JITDylib(ExecutionSession &ES, std::string Name) Error JITDylib::defineImpl(MaterializationUnit &MU) { SymbolNameSet Duplicates; - SymbolNameSet MUDefsOverridden; - - struct ExistingDefOverriddenEntry { - SymbolMap::iterator ExistingDefItr; - JITSymbolFlags NewFlags; - }; - std::vector<ExistingDefOverriddenEntry> ExistingDefsOverridden; - - for (auto &KV : MU.getSymbols()) { - assert(!KV.second.isLazy() && "Lazy flag should be managed internally."); - assert(!KV.second.isMaterializing() && - "Materializing flags should be managed internally."); + std::vector<SymbolStringPtr> ExistingDefsOverridden; + std::vector<SymbolStringPtr> MUDefsOverridden; - SymbolMap::iterator EntryItr; - bool Added; + for (const auto &KV : MU.getSymbols()) { + auto I = Symbols.find(KV.first); - auto NewFlags = KV.second; - NewFlags |= JITSymbolFlags::Lazy; - - std::tie(EntryItr, Added) = Symbols.insert( - std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); - - if (!Added) { + if (I != Symbols.end()) { if (KV.second.isStrong()) { - if (EntryItr->second.getFlags().isStrong() || - (EntryItr->second.getFlags() & JITSymbolFlags::Materializing)) + if (I->second.getFlags().isStrong() || + I->second.getState() > SymbolState::NeverSearched) Duplicates.insert(KV.first); - else - ExistingDefsOverridden.push_back({EntryItr, NewFlags}); + else { + assert(I->second.getState() == SymbolState::NeverSearched && + "Overridden existing def should be in the never-searched " + "state"); + ExistingDefsOverridden.push_back(KV.first); + } } else - MUDefsOverridden.insert(KV.first); + MUDefsOverridden.push_back(KV.first); } } - if (!Duplicates.empty()) { - // We need to remove the symbols we added. - for (auto &KV : MU.getSymbols()) { - if (Duplicates.count(KV.first)) - continue; - - bool Found = false; - for (const auto &EDO : ExistingDefsOverridden) - if (EDO.ExistingDefItr->first == KV.first) - Found = true; - - if (!Found) - Symbols.erase(KV.first); - } - - // FIXME: Return all duplicates. + // If there were any duplicate definitions then bail out. + if (!Duplicates.empty()) return make_error<DuplicateDefinition>(**Duplicates.begin()); - } - // Update flags on existing defs and call discard on their materializers. - for (auto &EDO : ExistingDefsOverridden) { - assert(EDO.ExistingDefItr->second.getFlags().isLazy() && - !EDO.ExistingDefItr->second.getFlags().isMaterializing() && - "Overridden existing def should be in the Lazy state"); + // Discard any overridden defs in this MU. + for (auto &S : MUDefsOverridden) + MU.doDiscard(*this, S); - EDO.ExistingDefItr->second.setFlags(EDO.NewFlags); + // Discard existing overridden defs. + for (auto &S : ExistingDefsOverridden) { - auto UMII = UnmaterializedInfos.find(EDO.ExistingDefItr->first); + auto UMII = UnmaterializedInfos.find(S); assert(UMII != UnmaterializedInfos.end() && "Overridden existing def should have an UnmaterializedInfo"); - - UMII->second->MU->doDiscard(*this, EDO.ExistingDefItr->first); + UMII->second->MU->doDiscard(*this, S); } - // Discard overridden symbols povided by MU. - for (auto &Sym : MUDefsOverridden) - MU.doDiscard(*this, Sym); + // Finally, add the defs from this MU. + for (auto &KV : MU.getSymbols()) { + auto &SymEntry = Symbols[KV.first]; + SymEntry.setFlags(KV.second); + SymEntry.setState(SymbolState::NeverSearched); + SymEntry.setMaterializerAttached(true); + } return Error::success(); } @@ -1532,17 +1556,7 @@ void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q, assert(MaterializingInfos.count(QuerySymbol) && "QuerySymbol does not have MaterializingInfo"); auto &MI = MaterializingInfos[QuerySymbol]; - - auto IdenticalQuery = - [&](const std::shared_ptr<AsynchronousSymbolQuery> &R) { - return R.get() == &Q; - }; - - auto I = std::find_if(MI.PendingQueries.begin(), MI.PendingQueries.end(), - IdenticalQuery); - assert(I != MI.PendingQueries.end() && - "Query Q should be in the PendingQueries list for QuerySymbol"); - MI.PendingQueries.erase(I); + MI.removeQuery(Q); } } @@ -1582,8 +1596,18 @@ JITDylib &ExecutionSession::getMainJITDylib() { return runSessionLocked([this]() -> JITDylib & { return *JDs.front(); }); } +JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { + return runSessionLocked([&, this]() -> JITDylib * { + for (auto &JD : JDs) + if (JD->getName() == Name) + return JD.get(); + return nullptr; + }); +} + JITDylib &ExecutionSession::createJITDylib(std::string Name, bool AddToMainDylibSearchOrder) { + assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { JDs.push_back( std::unique_ptr<JITDylib>(new JITDylib(*this, std::move(Name)))); @@ -1610,74 +1634,36 @@ void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { Expected<SymbolMap> ExecutionSession::legacyLookup( LegacyAsyncLookupFunction AsyncLookup, SymbolNameSet Names, - bool WaitUntilReady, RegisterDependenciesFunction RegisterDependencies) { + SymbolState RequiredState, + RegisterDependenciesFunction RegisterDependencies) { #if LLVM_ENABLE_THREADS // In the threaded case we use promises to return the results. std::promise<SymbolMap> PromisedResult; - std::mutex ErrMutex; Error ResolutionError = Error::success(); - std::promise<void> PromisedReady; - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { + auto NotifyComplete = [&](Expected<SymbolMap> R) { if (R) PromisedResult.set_value(std::move(*R)); else { - { - ErrorAsOutParameter _(&ResolutionError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ResolutionError = R.takeError(); - } + ErrorAsOutParameter _(&ResolutionError); + ResolutionError = R.takeError(); PromisedResult.set_value(SymbolMap()); } }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - if (Err) { - ErrorAsOutParameter _(&ReadyError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ReadyError = std::move(Err); - } - PromisedReady.set_value(); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } - #else SymbolMap Result; Error ResolutionError = Error::success(); - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { + auto NotifyComplete = [&](Expected<SymbolMap> R) { ErrorAsOutParameter _(&ResolutionError); if (R) Result = std::move(*R); else ResolutionError = R.takeError(); }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - ErrorAsOutParameter _(&ReadyError); - if (Err) - ReadyError = std::move(Err); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } #endif auto Query = std::make_shared<AsynchronousSymbolQuery>( - Names, std::move(OnResolve), std::move(OnReady)); + Names, RequiredState, std::move(NotifyComplete)); // FIXME: This should be run session locked along with the registration code // and error reporting below. SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); @@ -1701,39 +1687,13 @@ Expected<SymbolMap> ExecutionSession::legacyLookup( #if LLVM_ENABLE_THREADS auto ResultFuture = PromisedResult.get_future(); auto Result = ResultFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - } - - if (WaitUntilReady) { - auto ReadyFuture = PromisedReady.get_future(); - ReadyFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ReadyError) - return std::move(ReadyError); - } - } else - cantFail(std::move(ReadyError)); - + if (ResolutionError) + return std::move(ResolutionError); return std::move(Result); #else - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); + if (ResolutionError) return std::move(ResolutionError); - } - - if (ReadyError) - return std::move(ReadyError); return Result; #endif @@ -1741,9 +1701,16 @@ Expected<SymbolMap> ExecutionSession::legacyLookup( void ExecutionSession::lookup( const JITDylibSearchList &SearchOrder, SymbolNameSet Symbols, - SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady, + SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies) { + LLVM_DEBUG({ + runSessionLocked([&]() { + dbgs() << "Looking up " << Symbols << " in " << SearchOrder + << " (required state: " << RequiredState << ")\n"; + }); + }); + // lookup can be re-entered recursively if running on a single thread. Run any // outstanding MUs in case this query depends on them, otherwise this lookup // will starve waiting for a result from an MU that is stuck in the queue. @@ -1751,38 +1718,32 @@ void ExecutionSession::lookup( auto Unresolved = std::move(Symbols); std::map<JITDylib *, MaterializationUnitList> CollectedMUsMap; - auto Q = std::make_shared<AsynchronousSymbolQuery>( - Unresolved, std::move(OnResolve), std::move(OnReady)); - bool QueryIsFullyResolved = false; - bool QueryIsFullyReady = false; - bool QueryFailed = false; - - runSessionLocked([&]() { - for (auto &KV : SearchOrder) { - assert(KV.first && "JITDylibList entries must not be null"); - assert(!CollectedMUsMap.count(KV.first) && - "JITDylibList should not contain duplicate entries"); - - auto &JD = *KV.first; - auto MatchNonExported = KV.second; - JD.lodgeQuery(Q, Unresolved, MatchNonExported, CollectedMUsMap[&JD]); - } + auto Q = std::make_shared<AsynchronousSymbolQuery>(Unresolved, RequiredState, + std::move(NotifyComplete)); + bool QueryComplete = false; + + auto LodgingErr = runSessionLocked([&]() -> Error { + auto LodgeQuery = [&]() -> Error { + for (auto &KV : SearchOrder) { + assert(KV.first && "JITDylibList entries must not be null"); + assert(!CollectedMUsMap.count(KV.first) && + "JITDylibList should not contain duplicate entries"); + + auto &JD = *KV.first; + auto MatchNonExported = KV.second; + if (auto Err = JD.lodgeQuery(Q, Unresolved, MatchNonExported, + CollectedMUsMap[&JD])) + return Err; + } - if (Unresolved.empty()) { - // Query lodged successfully. + if (!Unresolved.empty()) + return make_error<SymbolsNotFound>(std::move(Unresolved)); - // Record whether this query is fully ready / resolved. We will use - // this to call handleFullyResolved/handleFullyReady outside the session - // lock. - QueryIsFullyResolved = Q->isFullyResolved(); - QueryIsFullyReady = Q->isFullyReady(); + return Error::success(); + }; - // Call the register dependencies function. - if (RegisterDependencies && !Q->QueryRegistrations.empty()) - RegisterDependencies(Q->QueryRegistrations); - } else { - // Query failed due to unresolved symbols. - QueryFailed = true; + if (auto Err = LodgeQuery()) { + // Query failed. // Disconnect the query from its dependencies. Q->detach(); @@ -1791,19 +1752,32 @@ void ExecutionSession::lookup( for (auto &KV : CollectedMUsMap) for (auto &MU : KV.second) KV.first->replace(std::move(MU)); + + return Err; } + + // Query lodged successfully. + + // Record whether this query is fully ready / resolved. We will use + // this to call handleFullyResolved/handleFullyReady outside the session + // lock. + QueryComplete = Q->isComplete(); + + // Call the register dependencies function. + if (RegisterDependencies && !Q->QueryRegistrations.empty()) + RegisterDependencies(Q->QueryRegistrations); + + return Error::success(); }); - if (QueryFailed) { - Q->handleFailed(make_error<SymbolsNotFound>(std::move(Unresolved))); + if (LodgingErr) { + Q->handleFailed(std::move(LodgingErr)); return; - } else { - if (QueryIsFullyResolved) - Q->handleFullyResolved(); - if (QueryIsFullyReady) - Q->handleFullyReady(); } + if (QueryComplete) + Q->handleComplete(); + // Move the MUs to the OutstandingMUs list, then materialize. { std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); @@ -1816,113 +1790,55 @@ void ExecutionSession::lookup( runOutstandingMUs(); } -Expected<SymbolMap> ExecutionSession::lookup( - const JITDylibSearchList &SearchOrder, const SymbolNameSet &Symbols, - RegisterDependenciesFunction RegisterDependencies, bool WaitUntilReady) { +Expected<SymbolMap> +ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, + const SymbolNameSet &Symbols, + SymbolState RequiredState, + RegisterDependenciesFunction RegisterDependencies) { #if LLVM_ENABLE_THREADS // In the threaded case we use promises to return the results. std::promise<SymbolMap> PromisedResult; - std::mutex ErrMutex; Error ResolutionError = Error::success(); - std::promise<void> PromisedReady; - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { + + auto NotifyComplete = [&](Expected<SymbolMap> R) { if (R) PromisedResult.set_value(std::move(*R)); else { - { - ErrorAsOutParameter _(&ResolutionError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ResolutionError = R.takeError(); - } + ErrorAsOutParameter _(&ResolutionError); + ResolutionError = R.takeError(); PromisedResult.set_value(SymbolMap()); } }; - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - if (Err) { - ErrorAsOutParameter _(&ReadyError); - std::lock_guard<std::mutex> Lock(ErrMutex); - ReadyError = std::move(Err); - } - PromisedReady.set_value(); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } - #else SymbolMap Result; Error ResolutionError = Error::success(); - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected<SymbolMap> R) { + auto NotifyComplete = [&](Expected<SymbolMap> R) { ErrorAsOutParameter _(&ResolutionError); if (R) Result = std::move(*R); else ResolutionError = R.takeError(); }; - - std::function<void(Error)> OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - ErrorAsOutParameter _(&ReadyError); - if (Err) - ReadyError = std::move(Err); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } #endif // Perform the asynchronous lookup. - lookup(SearchOrder, Symbols, OnResolve, OnReady, RegisterDependencies); + lookup(SearchOrder, Symbols, RequiredState, NotifyComplete, + RegisterDependencies); #if LLVM_ENABLE_THREADS auto ResultFuture = PromisedResult.get_future(); auto Result = ResultFuture.get(); - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - } - - if (WaitUntilReady) { - auto ReadyFuture = PromisedReady.get_future(); - ReadyFuture.get(); - - { - std::lock_guard<std::mutex> Lock(ErrMutex); - if (ReadyError) - return std::move(ReadyError); - } - } else - cantFail(std::move(ReadyError)); + if (ResolutionError) + return std::move(ResolutionError); return std::move(Result); #else - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); + if (ResolutionError) return std::move(ResolutionError); - } - - if (ReadyError) - return std::move(ReadyError); return Result; #endif @@ -1933,8 +1849,8 @@ ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, SymbolStringPtr Name) { SymbolNameSet Names({Name}); - if (auto ResultMap = lookup(SearchOrder, std::move(Names), - NoDependenciesToRegister, true)) { + if (auto ResultMap = lookup(SearchOrder, std::move(Names), SymbolState::Ready, + NoDependenciesToRegister)) { assert(ResultMap->size() == 1 && "Unexpected number of results"); assert(ResultMap->count(Name) && "Missing result for symbol"); return std::move(ResultMap->begin()->second); diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 7c3c50b4d6e5..f7fc5f8f1797 100644 --- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -1,9 +1,8 @@ //===---- ExecutionUtils.cpp - Utilities for executing functions in Orc ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -130,8 +129,7 @@ Error CtorDtorRunner::run() { auto &ES = JD.getExecutionSession(); if (auto CtorDtorMap = - ES.lookup(JITDylibSearchList({{&JD, true}}), std::move(Names), - NoDependenciesToRegister, true)) { + ES.lookup(JITDylibSearchList({{&JD, true}}), std::move(Names))) { for (auto &KV : CtorDtorsByPriority) { for (auto &Name : KV.second) { assert(CtorDtorMap->count(Name) && "No entry for Name"); @@ -140,13 +138,10 @@ Error CtorDtorRunner::run() { CtorDtor(); } } + CtorDtorsByPriority.clear(); return Error::success(); } else return CtorDtorMap.takeError(); - - CtorDtorsByPriority.clear(); - - return Error::success(); } void LocalCXXRuntimeOverridesBase::runDestructors() { @@ -179,22 +174,24 @@ Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, } DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( - sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow) + sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), - GlobalPrefix(DL.getGlobalPrefix()) {} + GlobalPrefix(GlobalPrefix) {} Expected<DynamicLibrarySearchGenerator> -DynamicLibrarySearchGenerator::Load(const char *FileName, const DataLayout &DL, +DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow) { std::string ErrMsg; auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); if (!Lib.isValid()) return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); - return DynamicLibrarySearchGenerator(std::move(Lib), DL, std::move(Allow)); + return DynamicLibrarySearchGenerator(std::move(Lib), GlobalPrefix, + std::move(Allow)); } -SymbolNameSet DynamicLibrarySearchGenerator:: -operator()(JITDylib &JD, const SymbolNameSet &Names) { +Expected<SymbolNameSet> +DynamicLibrarySearchGenerator::operator()(JITDylib &JD, + const SymbolNameSet &Names) { orc::SymbolNameSet Added; orc::SymbolMap NewSymbols; @@ -210,7 +207,8 @@ operator()(JITDylib &JD, const SymbolNameSet &Names) { if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) continue; - std::string Tmp((*Name).data() + (HasGlobalPrefix ? 1 : 0), (*Name).size()); + std::string Tmp((*Name).data() + HasGlobalPrefix, + (*Name).size() - HasGlobalPrefix); if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { Added.insert(Name); NewSymbols[Name] = JITEvaluatedSymbol( diff --git a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp index d952d1be70da..81dfc02f55b2 100644 --- a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp +++ b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -1,9 +1,8 @@ //===--------------- IRCompileLayer.cpp - IR Compiling Layer --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp index 7bc0d696e3ac..e3519284613e 100644 --- a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp +++ b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -1,9 +1,8 @@ //===-------------- IRTransformLayer.cpp - IR Transform Layer -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 82000ec5b32b..cc3656fe5dc5 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -1,9 +1,8 @@ //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -38,8 +37,8 @@ private: void materialize(MaterializationResponsibility R) override { SymbolMap Result; Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); - R.resolve(Result); - R.emit(); + R.notifyResolved(Result); + R.notifyEmitted(); } void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { @@ -238,11 +237,11 @@ void makeStub(Function &F, Value &ImplPointer) { Module &M = *F.getParent(); BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); IRBuilder<> Builder(EntryBlock); - LoadInst *ImplAddr = Builder.CreateLoad(&ImplPointer); + LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); std::vector<Value*> CallArgs; for (auto &A : F.args()) CallArgs.push_back(&A); - CallInst *Call = Builder.CreateCall(ImplAddr, CallArgs); + CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); Call->setTailCall(); Call->setAttributes(F.getAttributes()); if (F.getReturnType()->isVoidTy()) diff --git a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index 4af09d196ff9..df23547a9de3 100644 --- a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -1,9 +1,8 @@ //===----- JITTargetMachineBuilder.cpp - Build TargetMachines for JIT -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp index e2089f9106bd..b120691faf07 100644 --- a/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -1,58 +1,37 @@ //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/Mangler.h" -namespace { +namespace llvm { +namespace orc { - // A SimpleCompiler that owns its TargetMachine. - class TMOwningSimpleCompiler : public llvm::orc::SimpleCompiler { - public: - TMOwningSimpleCompiler(std::unique_ptr<llvm::TargetMachine> TM) - : llvm::orc::SimpleCompiler(*TM), TM(std::move(TM)) {} - private: - // FIXME: shared because std::functions (and thus - // IRCompileLayer::CompileFunction) are not moveable. - std::shared_ptr<llvm::TargetMachine> TM; - }; +Error LLJITBuilderState::prepareForConstruction() { -} // end anonymous namespace + if (!JTMB) { + if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost()) + JTMB = std::move(*JTMBOrErr); + else + return JTMBOrErr.takeError(); + } -namespace llvm { -namespace orc { + return Error::success(); +} LLJIT::~LLJIT() { if (CompileThreads) CompileThreads->wait(); } -Expected<std::unique_ptr<LLJIT>> -LLJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL, - unsigned NumCompileThreads) { - - if (NumCompileThreads == 0) { - // If NumCompileThreads == 0 then create a single-threaded LLJIT instance. - auto TM = JTMB.createTargetMachine(); - if (!TM) - return TM.takeError(); - return std::unique_ptr<LLJIT>(new LLJIT(llvm::make_unique<ExecutionSession>(), - std::move(*TM), std::move(DL))); - } - - return std::unique_ptr<LLJIT>(new LLJIT(llvm::make_unique<ExecutionSession>(), - std::move(JTMB), std::move(DL), - NumCompileThreads)); -} - Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { auto InternedName = ES->intern(Name); SymbolMap Symbols({{InternedName, Sym}}); @@ -65,13 +44,13 @@ Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { if (auto Err = applyDataLayout(*TSM.getModule())) return Err; - return CompileLayer.add(JD, std::move(TSM), ES->allocateVModule()); + return CompileLayer->add(JD, std::move(TSM), ES->allocateVModule()); } Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { assert(Obj && "Can not add null object"); - return ObjLinkingLayer.add(JD, std::move(Obj), ES->allocateVModule()); + return ObjLinkingLayer->add(JD, std::move(Obj), ES->allocateVModule()); } Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, @@ -79,42 +58,76 @@ Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, return ES->lookup(JITDylibSearchList({{&JD, true}}), ES->intern(Name)); } -LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES, - std::unique_ptr<TargetMachine> TM, DataLayout DL) - : ES(std::move(ES)), Main(this->ES->getMainJITDylib()), DL(std::move(DL)), - ObjLinkingLayer( - *this->ES, - []() { return llvm::make_unique<SectionMemoryManager>(); }), - CompileLayer(*this->ES, ObjLinkingLayer, - TMOwningSimpleCompiler(std::move(TM))), - CtorRunner(Main), DtorRunner(Main) {} - -LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES, JITTargetMachineBuilder JTMB, - DataLayout DL, unsigned NumCompileThreads) - : ES(std::move(ES)), Main(this->ES->getMainJITDylib()), DL(std::move(DL)), - ObjLinkingLayer( - *this->ES, - []() { return llvm::make_unique<SectionMemoryManager>(); }), - CompileLayer(*this->ES, ObjLinkingLayer, - ConcurrentIRCompiler(std::move(JTMB))), - CtorRunner(Main), DtorRunner(Main) { - assert(NumCompileThreads != 0 && - "Multithreaded LLJIT instance can not be created with 0 threads"); - - // Move modules to new contexts when they're emitted so that we can compile - // them in parallel. - CompileLayer.setCloneToNewContextOnEmit(true); - - // Create a thread pool to compile on and set the execution session - // dispatcher to use the thread pool. - CompileThreads = llvm::make_unique<ThreadPool>(NumCompileThreads); - this->ES->setDispatchMaterialization( - [this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) { - // FIXME: Switch to move capture once we have c++14. - auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU)); - auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); }; - CompileThreads->async(std::move(Work)); - }); +std::unique_ptr<ObjectLayer> +LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { + + // If the config state provided an ObjectLinkingLayer factory then use it. + if (S.CreateObjectLinkingLayer) + return S.CreateObjectLinkingLayer(ES); + + // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs + // a new SectionMemoryManager for each object. + auto GetMemMgr = []() { return llvm::make_unique<SectionMemoryManager>(); }; + return llvm::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); +} + +Expected<IRCompileLayer::CompileFunction> +LLJIT::createCompileFunction(LLJITBuilderState &S, + JITTargetMachineBuilder JTMB) { + + /// If there is a custom compile function creator set then use it. + if (S.CreateCompileFunction) + return S.CreateCompileFunction(std::move(JTMB)); + + // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler, + // depending on the number of threads requested. + if (S.NumCompileThreads > 0) + return ConcurrentIRCompiler(std::move(JTMB)); + + auto TM = JTMB.createTargetMachine(); + if (!TM) + return TM.takeError(); + + return TMOwningSimpleCompiler(std::move(*TM)); +} + +LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) + : ES(S.ES ? std::move(S.ES) : llvm::make_unique<ExecutionSession>()), + Main(this->ES->getMainJITDylib()), DL(""), CtorRunner(Main), + DtorRunner(Main) { + + ErrorAsOutParameter _(&Err); + + ObjLinkingLayer = createObjectLinkingLayer(S, *ES); + + if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) + DL = std::move(*DLOrErr); + else { + Err = DLOrErr.takeError(); + return; + } + + { + auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB)); + if (!CompileFunction) { + Err = CompileFunction.takeError(); + return; + } + CompileLayer = llvm::make_unique<IRCompileLayer>( + *ES, *ObjLinkingLayer, std::move(*CompileFunction)); + } + + if (S.NumCompileThreads > 0) { + CompileLayer->setCloneToNewContextOnEmit(true); + CompileThreads = llvm::make_unique<ThreadPool>(S.NumCompileThreads); + ES->setDispatchMaterialization( + [this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) { + // FIXME: Switch to move capture once we have c++14. + auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU)); + auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); }; + CompileThreads->async(std::move(Work)); + }); + } } std::string LLJIT::mangle(StringRef UnmangledName) { @@ -143,35 +156,11 @@ void LLJIT::recordCtorDtors(Module &M) { DtorRunner.add(getDestructors(M)); } -Expected<std::unique_ptr<LLLazyJIT>> -LLLazyJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL, - JITTargetAddress ErrorAddr, unsigned NumCompileThreads) { - auto ES = llvm::make_unique<ExecutionSession>(); - - const Triple &TT = JTMB.getTargetTriple(); - - auto LCTMgr = createLocalLazyCallThroughManager(TT, *ES, ErrorAddr); - if (!LCTMgr) - return LCTMgr.takeError(); - - auto ISMBuilder = createLocalIndirectStubsManagerBuilder(TT); - if (!ISMBuilder) - return make_error<StringError>( - std::string("No indirect stubs manager builder for ") + TT.str(), - inconvertibleErrorCode()); - - if (NumCompileThreads == 0) { - auto TM = JTMB.createTargetMachine(); - if (!TM) - return TM.takeError(); - return std::unique_ptr<LLLazyJIT>( - new LLLazyJIT(std::move(ES), std::move(*TM), std::move(DL), - std::move(*LCTMgr), std::move(ISMBuilder))); - } - - return std::unique_ptr<LLLazyJIT>(new LLLazyJIT( - std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads, - std::move(*LCTMgr), std::move(ISMBuilder))); +Error LLLazyJITBuilderState::prepareForConstruction() { + if (auto Err = LLJITBuilderState::prepareForConstruction()) + return Err; + TT = JTMB->getTargetTriple(); + return Error::success(); } Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { @@ -182,28 +171,55 @@ Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { recordCtorDtors(*TSM.getModule()); - return CODLayer.add(JD, std::move(TSM), ES->allocateVModule()); + return CODLayer->add(JD, std::move(TSM), ES->allocateVModule()); } -LLLazyJIT::LLLazyJIT( - std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM, - DataLayout DL, std::unique_ptr<LazyCallThroughManager> LCTMgr, - std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder) - : LLJIT(std::move(ES), std::move(TM), std::move(DL)), - LCTMgr(std::move(LCTMgr)), TransformLayer(*this->ES, CompileLayer), - CODLayer(*this->ES, TransformLayer, *this->LCTMgr, - std::move(ISMBuilder)) {} - -LLLazyJIT::LLLazyJIT( - std::unique_ptr<ExecutionSession> ES, JITTargetMachineBuilder JTMB, - DataLayout DL, unsigned NumCompileThreads, - std::unique_ptr<LazyCallThroughManager> LCTMgr, - std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder) - : LLJIT(std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads), - LCTMgr(std::move(LCTMgr)), TransformLayer(*this->ES, CompileLayer), - CODLayer(*this->ES, TransformLayer, *this->LCTMgr, - std::move(ISMBuilder)) { - CODLayer.setCloneToNewContextOnEmit(true); +LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { + + // If LLJIT construction failed then bail out. + if (Err) + return; + + ErrorAsOutParameter _(&Err); + + /// Take/Create the lazy-compile callthrough manager. + if (S.LCTMgr) + LCTMgr = std::move(S.LCTMgr); + else { + if (auto LCTMgrOrErr = createLocalLazyCallThroughManager( + S.TT, *ES, S.LazyCompileFailureAddr)) + LCTMgr = std::move(*LCTMgrOrErr); + else { + Err = LCTMgrOrErr.takeError(); + return; + } + } + + // Take/Create the indirect stubs manager builder. + auto ISMBuilder = std::move(S.ISMBuilder); + + // If none was provided, try to build one. + if (!ISMBuilder) + ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT); + + // No luck. Bail out. + if (!ISMBuilder) { + Err = make_error<StringError>("Could not construct " + "IndirectStubsManagerBuilder for target " + + S.TT.str(), + inconvertibleErrorCode()); + return; + } + + // Create the transform layer. + TransformLayer = llvm::make_unique<IRTransformLayer>(*ES, *CompileLayer); + + // Create the COD layer. + CODLayer = llvm::make_unique<CompileOnDemandLayer>( + *ES, *TransformLayer, *LCTMgr, std::move(ISMBuilder)); + + if (S.NumCompileThreads > 0) + CODLayer->setCloneToNewContextOnEmit(true); } } // End namespace orc. diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp index 11af76825e9f..3ed2dabf4545 100644 --- a/lib/ExecutionEngine/Orc/Layer.cpp +++ b/lib/ExecutionEngine/Orc/Layer.cpp @@ -1,9 +1,8 @@ //===-------------------- Layer.cpp - Layer interfaces --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -88,17 +87,15 @@ void BasicIRLayerMaterializationUnit::materialize( #ifndef NDEBUG auto &ES = R.getTargetJITDylib().getExecutionSession(); + auto &N = R.getTargetJITDylib().getName(); #endif // NDEBUG auto Lock = TSM.getContextLock(); - LLVM_DEBUG(ES.runSessionLocked([&]() { - dbgs() << "Emitting, for " << R.getTargetJITDylib().getName() << ", " - << *this << "\n"; - });); + LLVM_DEBUG(ES.runSessionLocked( + [&]() { dbgs() << "Emitting, for " << N << ", " << *this << "\n"; });); L.emit(std::move(R), std::move(TSM)); LLVM_DEBUG(ES.runSessionLocked([&]() { - dbgs() << "Finished emitting, for " << R.getTargetJITDylib().getName() - << ", " << *this << "\n"; + dbgs() << "Finished emitting, for " << N << ", " << *this << "\n"; });); } diff --git a/lib/ExecutionEngine/Orc/LazyReexports.cpp b/lib/ExecutionEngine/Orc/LazyReexports.cpp index 55f4a7c5afce..fc8205845654 100644 --- a/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -1,9 +1,8 @@ //===---------- LazyReexports.cpp - Utilities for lazy reexports ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -52,18 +51,15 @@ LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) { SymbolName = I->second.second; } - auto LookupResult = ES.lookup(JITDylibSearchList({{SourceJD, true}}), - {SymbolName}, NoDependenciesToRegister, true); + auto LookupResult = + ES.lookup(JITDylibSearchList({{SourceJD, true}}), SymbolName); if (!LookupResult) { ES.reportError(LookupResult.takeError()); return ErrorHandlerAddr; } - assert(LookupResult->size() == 1 && "Unexpected number of results"); - assert(LookupResult->count(SymbolName) && "Unexpected result"); - - auto ResolvedAddr = LookupResult->begin()->second.getAddress(); + auto ResolvedAddr = LookupResult->getAddress(); std::shared_ptr<NotifyResolvedFunction> NotifyResolved = nullptr; { @@ -182,8 +178,8 @@ void LazyReexportsMaterializationUnit::materialize( for (auto &Alias : RequestedAliases) Stubs[Alias.first] = ISManager.findStub(*Alias.first, false); - R.resolve(Stubs); - R.emit(); + R.notifyResolved(Stubs); + R.notifyEmitted(); } void LazyReexportsMaterializationUnit::discard(const JITDylib &JD, diff --git a/lib/ExecutionEngine/Orc/Legacy.cpp b/lib/ExecutionEngine/Orc/Legacy.cpp index ddb72544b770..ce6368b57a89 100644 --- a/lib/ExecutionEngine/Orc/Legacy.cpp +++ b/lib/ExecutionEngine/Orc/Legacy.cpp @@ -1,9 +1,8 @@ //===------- Legacy.cpp - Adapters for ExecutionEngine API interop --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -37,8 +36,7 @@ void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols, }; auto Q = std::make_shared<AsynchronousSymbolQuery>( - InternedSymbols, OnResolvedWithUnwrap, - [this](Error Err) { ES.reportError(std::move(Err)); }); + InternedSymbols, SymbolState::Resolved, OnResolvedWithUnwrap); auto Unresolved = R.lookup(Q, InternedSymbols); if (Unresolved.empty()) { diff --git a/lib/ExecutionEngine/Orc/NullResolver.cpp b/lib/ExecutionEngine/Orc/NullResolver.cpp index 922fc6f021ce..5b4345b870bb 100644 --- a/lib/ExecutionEngine/Orc/NullResolver.cpp +++ b/lib/ExecutionEngine/Orc/NullResolver.cpp @@ -1,9 +1,8 @@ //===---------- NullResolver.cpp - Reject symbol lookup requests ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp new file mode 100644 index 000000000000..def0b300eca1 --- /dev/null +++ b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -0,0 +1,483 @@ +//===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" + +#include <vector> + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { +public: + ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer &Layer, + MaterializationResponsibility MR, + std::unique_ptr<MemoryBuffer> ObjBuffer) + : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} + + JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } + + MemoryBufferRef getObjectBuffer() const override { + return ObjBuffer->getMemBufferRef(); + } + + void notifyFailed(Error Err) override { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + } + + void lookup(const DenseSet<StringRef> &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation) override { + + JITDylibSearchList SearchOrder; + MR.getTargetJITDylib().withSearchOrderDo( + [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); + + auto &ES = Layer.getExecutionSession(); + + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.intern(S)); + + // OnResolve -- De-intern the symbols and pass the result to the linker. + // FIXME: Capture LookupContinuation by move once we have c++14. + auto SharedLookupContinuation = + std::make_shared<JITLinkAsyncLookupContinuation>( + std::move(LookupContinuation)); + auto OnResolve = [SharedLookupContinuation](Expected<SymbolMap> Result) { + if (!Result) + (*SharedLookupContinuation)(Result.takeError()); + else { + AsyncLookupResult LR; + for (auto &KV : *Result) + LR[*KV.first] = KV.second; + (*SharedLookupContinuation)(std::move(LR)); + } + }; + + ES.lookup(SearchOrder, std::move(InternedSymbols), SymbolState::Resolved, + std::move(OnResolve), [this](const SymbolDependenceMap &Deps) { + registerDependencies(Deps); + }); + } + + void notifyResolved(AtomGraph &G) override { + auto &ES = Layer.getExecutionSession(); + + SymbolFlagsMap ExtraSymbolsToClaim; + bool AutoClaim = Layer.AutoClaimObjectSymbols; + + SymbolMap InternedResult; + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && DA->isGlobal()) { + auto InternedName = ES.intern(DA->getName()); + JITSymbolFlags Flags; + + if (DA->isExported()) + Flags |= JITSymbolFlags::Exported; + if (DA->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (DA->isCallable()) + Flags |= JITSymbolFlags::Callable; + if (DA->isCommon()) + Flags |= JITSymbolFlags::Common; + + InternedResult[InternedName] = + JITEvaluatedSymbol(DA->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + for (auto *A : G.absolute_atoms()) + if (A->hasName()) { + auto InternedName = ES.intern(A->getName()); + JITSymbolFlags Flags; + Flags |= JITSymbolFlags::Absolute; + if (A->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (A->isCallable()) + Flags |= JITSymbolFlags::Callable; + InternedResult[InternedName] = + JITEvaluatedSymbol(A->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + if (!ExtraSymbolsToClaim.empty()) + if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) + return notifyFailed(std::move(Err)); + + MR.notifyResolved(InternedResult); + + Layer.notifyLoaded(MR); + } + + void notifyFinalized( + std::unique_ptr<JITLinkMemoryManager::Allocation> A) override { + + if (auto Err = Layer.notifyEmitted(MR, std::move(A))) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + + return; + } + MR.notifyEmitted(); + } + + AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override { + return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); }; + } + + Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { + // Add passes to mark duplicate defs as should-discard, and to walk the + // atom graph to build the symbol dependence graph. + Config.PrePrunePasses.push_back( + [this](AtomGraph &G) { return markSymbolsToDiscard(G); }); + Config.PostPrunePasses.push_back( + [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); }); + + Layer.modifyPassConfig(MR, TT, Config); + + return Error::success(); + } + +private: + using AnonAtomNamedDependenciesMap = + DenseMap<const DefinedAtom *, SymbolNameSet>; + + Error markSymbolsToDiscard(AtomGraph &G) { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->isWeak() && DA->hasName()) { + auto S = ES.intern(DA->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + DA->setShouldDiscard(true); + } + + for (auto *A : G.absolute_atoms()) + if (A->isWeak() && A->hasName()) { + auto S = ES.intern(A->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + A->setShouldDiscard(true); + } + + return Error::success(); + } + + Error markResponsibilitySymbolsLive(AtomGraph &G) const { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && + MR.getSymbols().count(ES.intern(DA->getName()))) + DA->setLive(true); + return Error::success(); + } + + Error computeNamedSymbolDependencies(AtomGraph &G) { + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + auto AnonDeps = computeAnonDeps(G); + + for (auto *DA : G.defined_atoms()) { + + // Skip anonymous and non-global atoms: we do not need dependencies for + // these. + if (!DA->hasName() || !DA->isGlobal()) + continue; + + auto DAName = ES.intern(DA->getName()); + SymbolNameSet &DADeps = NamedSymbolDeps[DAName]; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + + if (TA.hasName()) + DADeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + auto &DTA = static_cast<DefinedAtom &>(TA); + auto I = AnonDeps.find(&DTA); + if (I != AnonDeps.end()) + for (auto &S : I->second) + DADeps.insert(S); + } + } + } + + return Error::success(); + } + + AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) { + + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + AnonAtomNamedDependenciesMap DepMap; + + // For all anonymous atoms: + // (1) Add their named dependencies. + // (2) Add them to the worklist for further iteration if they have any + // depend on any other anonymous atoms. + struct WorklistEntry { + WorklistEntry(DefinedAtom *DA, DenseSet<DefinedAtom *> DAAnonDeps) + : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {} + + DefinedAtom *DA = nullptr; + DenseSet<DefinedAtom *> DAAnonDeps; + }; + std::vector<WorklistEntry> Worklist; + for (auto *DA : G.defined_atoms()) + if (!DA->hasName()) { + auto &DANamedDeps = DepMap[DA]; + DenseSet<DefinedAtom *> DAAnonDeps; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + if (TA.hasName()) + DANamedDeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + DAAnonDeps.insert(static_cast<DefinedAtom *>(&TA)); + } + } + + if (!DAAnonDeps.empty()) + Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps))); + } + + // Loop over all anonymous atoms with anonymous dependencies, propagating + // their respective *named* dependencies. Iterate until we hit a stable + // state. + bool Changed; + do { + Changed = false; + for (auto &WLEntry : Worklist) { + auto *DA = WLEntry.DA; + auto &DANamedDeps = DepMap[DA]; + auto &DAAnonDeps = WLEntry.DAAnonDeps; + + for (auto *TA : DAAnonDeps) { + auto I = DepMap.find(TA); + if (I != DepMap.end()) + for (const auto &S : I->second) + Changed |= DANamedDeps.insert(S).second; + } + } + } while (Changed); + + return DepMap; + } + + void registerDependencies(const SymbolDependenceMap &QueryDeps) { + for (auto &NamedDepsEntry : NamedSymbolDeps) { + auto &Name = NamedDepsEntry.first; + auto &NameDeps = NamedDepsEntry.second; + SymbolDependenceMap SymbolDeps; + + for (const auto &QueryDepsEntry : QueryDeps) { + JITDylib &SourceJD = *QueryDepsEntry.first; + const SymbolNameSet &Symbols = QueryDepsEntry.second; + auto &DepsForJD = SymbolDeps[&SourceJD]; + + for (const auto &S : Symbols) + if (NameDeps.count(S)) + DepsForJD.insert(S); + + if (DepsForJD.empty()) + SymbolDeps.erase(&SourceJD); + } + + MR.addDependencies(Name, SymbolDeps); + } + } + + ObjectLinkingLayer &Layer; + MaterializationResponsibility MR; + std::unique_ptr<MemoryBuffer> ObjBuffer; + DenseMap<SymbolStringPtr, SymbolNameSet> NamedSymbolDeps; +}; + +ObjectLinkingLayer::Plugin::~Plugin() {} + +ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, + JITLinkMemoryManager &MemMgr) + : ObjectLayer(ES), MemMgr(MemMgr) {} + +ObjectLinkingLayer::~ObjectLinkingLayer() { + if (auto Err = removeAllModules()) + getExecutionSession().reportError(std::move(Err)); +} + +void ObjectLinkingLayer::emit(MaterializationResponsibility R, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Object must not be null"); + jitLink(llvm::make_unique<ObjectLinkingLayerJITLinkContext>( + *this, std::move(R), std::move(O))); +} + +void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, + const Triple &TT, + PassConfiguration &PassConfig) { + for (auto &P : Plugins) + P->modifyPassConfig(MR, TT, PassConfig); +} + +void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { + for (auto &P : Plugins) + P->notifyLoaded(MR); +} + +Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, + AllocPtr Alloc) { + Error Err = Error::success(); + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyEmitted(MR)); + + if (Err) + return Err; + + { + std::lock_guard<std::mutex> Lock(LayerMutex); + UntrackedAllocs.push_back(std::move(Alloc)); + } + + return Error::success(); +} + +Error ObjectLinkingLayer::removeModule(VModuleKey K) { + Error Err = Error::success(); + + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingModule(K)); + + AllocPtr Alloc; + + { + std::lock_guard<std::mutex> Lock(LayerMutex); + auto AllocItr = TrackedAllocs.find(K); + Alloc = std::move(AllocItr->second); + TrackedAllocs.erase(AllocItr); + } + + assert(Alloc && "No allocation for key K"); + + return joinErrors(std::move(Err), Alloc->deallocate()); +} + +Error ObjectLinkingLayer::removeAllModules() { + + Error Err = Error::success(); + + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingAllModules()); + + std::vector<AllocPtr> Allocs; + { + std::lock_guard<std::mutex> Lock(LayerMutex); + Allocs = std::move(UntrackedAllocs); + + for (auto &KV : TrackedAllocs) + Allocs.push_back(std::move(KV.second)); + + TrackedAllocs.clear(); + } + + while (!Allocs.empty()) { + Err = joinErrors(std::move(Err), Allocs.back()->deallocate()); + Allocs.pop_back(); + } + + return Err; +} + +EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( + jitlink::EHFrameRegistrar &Registrar) + : Registrar(Registrar) {} + +void EHFrameRegistrationPlugin::modifyPassConfig( + MaterializationResponsibility &MR, const Triple &TT, + PassConfiguration &PassConfig) { + assert(!InProcessLinks.count(&MR) && "Link for MR already being tracked?"); + + PassConfig.PostFixupPasses.push_back( + createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr) { + if (Addr) + InProcessLinks[&MR] = Addr; + })); +} + +Error EHFrameRegistrationPlugin::notifyEmitted( + MaterializationResponsibility &MR) { + + auto EHFrameAddrItr = InProcessLinks.find(&MR); + if (EHFrameAddrItr == InProcessLinks.end()) + return Error::success(); + + auto EHFrameAddr = EHFrameAddrItr->second; + assert(EHFrameAddr && "eh-frame addr to register can not be null"); + + InProcessLinks.erase(EHFrameAddrItr); + if (auto Key = MR.getVModuleKey()) + TrackedEHFrameAddrs[Key] = EHFrameAddr; + else + UntrackedEHFrameAddrs.push_back(EHFrameAddr); + + return Registrar.registerEHFrames(EHFrameAddr); +} + +Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) { + auto EHFrameAddrItr = TrackedEHFrameAddrs.find(K); + if (EHFrameAddrItr == TrackedEHFrameAddrs.end()) + return Error::success(); + + auto EHFrameAddr = EHFrameAddrItr->second; + assert(EHFrameAddr && "Tracked eh-frame addr must not be null"); + + TrackedEHFrameAddrs.erase(EHFrameAddrItr); + + return Registrar.deregisterEHFrames(EHFrameAddr); +} + +Error EHFrameRegistrationPlugin::notifyRemovingAllModules() { + + std::vector<JITTargetAddress> EHFrameAddrs = std::move(UntrackedEHFrameAddrs); + EHFrameAddrs.reserve(EHFrameAddrs.size() + TrackedEHFrameAddrs.size()); + + for (auto &KV : TrackedEHFrameAddrs) + EHFrameAddrs.push_back(KV.second); + + TrackedEHFrameAddrs.clear(); + + Error Err = Error::success(); + + while (!EHFrameAddrs.empty()) { + auto EHFrameAddr = EHFrameAddrs.back(); + assert(EHFrameAddr && "Untracked eh-frame addr must not be null"); + EHFrameAddrs.pop_back(); + Err = joinErrors(std::move(Err), Registrar.deregisterEHFrames(EHFrameAddr)); + } + + return Err; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp index 825f53204736..815517321b76 100644 --- a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp +++ b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -1,9 +1,8 @@ //===---------- ObjectTransformLayer.cpp - Object Transform Layer ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/lib/ExecutionEngine/Orc/OrcABISupport.cpp index aa4055542426..8ed23de419d1 100644 --- a/lib/ExecutionEngine/Orc/OrcABISupport.cpp +++ b/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -1,9 +1,8 @@ //===------------- OrcABISupport.cpp - ABI specific support code ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -148,7 +147,7 @@ Error OrcAArch64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -230,7 +229,7 @@ Error OrcX86_64_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -498,7 +497,7 @@ Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -684,7 +683,7 @@ Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -930,7 +929,7 @@ Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; diff --git a/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/lib/ExecutionEngine/Orc/OrcCBindings.cpp index 6dea64a6e78f..28c8479abba4 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ b/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -1,9 +1,8 @@ //===----------- OrcCBindings.cpp - C bindings for the Orc APIs -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index 817a4b89bfb0..98129e1690d2 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -1,9 +1,8 @@ //===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -154,8 +153,8 @@ private: for (auto &S : Symbols) { if (auto Sym = findSymbol(*S)) { if (auto Addr = Sym.getAddress()) { - Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); - Query->notifySymbolReady(); + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); } else { Stack.ES.legacyFailQuery(*Query, Addr.takeError()); return orc::SymbolNameSet(); @@ -167,11 +166,8 @@ private: UnresolvedSymbols.insert(S); } - if (Query->isFullyResolved()) - Query->handleFullyResolved(); - - if (Query->isFullyReady()) - Query->handleFullyReady(); + if (Query->isComplete()) + Query->handleComplete(); return UnresolvedSymbols; } @@ -215,28 +211,31 @@ public: IndirectStubsManagerBuilder IndirectStubsMgrBuilder) : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), - ObjectLayer(ES, - [this](orc::VModuleKey K) { - auto ResolverI = Resolvers.find(K); - assert(ResolverI != Resolvers.end() && - "No resolver for module K"); - auto Resolver = std::move(ResolverI->second); - Resolvers.erase(ResolverI); - return ObjLayerT::Resources{ - std::make_shared<SectionMemoryManager>(), Resolver}; - }, - nullptr, - [this](orc::VModuleKey K, const object::ObjectFile &Obj, - const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { - this->notifyFinalized(K, Obj, LoadedObjInfo); - }, - [this](orc::VModuleKey K, const object::ObjectFile &Obj) { - this->notifyFreed(K, Obj); - }), - CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), + ObjectLayer( + AcknowledgeORCv1Deprecation, ES, + [this](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && + "No resolver for module K"); + auto Resolver = std::move(ResolverI->second); + Resolvers.erase(ResolverI); + return ObjLayerT::Resources{ + std::make_shared<SectionMemoryManager>(), Resolver}; + }, + nullptr, + [this](orc::VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + this->notifyFinalized(K, Obj, LoadedObjInfo); + }, + [this](orc::VModuleKey K, const object::ObjectFile &Obj) { + this->notifyFreed(K, Obj); + }), + CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, + orc::SimpleCompiler(TM)), CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(), std::move(IndirectStubsMgrBuilder), Resolvers)), CXXRuntimeOverrides( + AcknowledgeORCv1Deprecation, [this](const std::string &S) { return mangle(S); }) {} Error shutdown() { @@ -312,7 +311,8 @@ public: // Run the static constructors, and save the static destructor runner for // execution when the JIT is torn down. - orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner(std::move(CtorNames), K); + orc::LegacyCtorDtorRunner<OrcCBindingsStack> CtorRunner( + AcknowledgeORCv1Deprecation, std::move(CtorNames), K); if (auto Err = CtorRunner.runViaLayer(*this)) return std::move(Err); @@ -469,7 +469,7 @@ private: return nullptr; return llvm::make_unique<CODLayerT>( - ES, CompileLayer, + AcknowledgeORCv1Deprecation, ES, CompileLayer, [&Resolvers](orc::VModuleKey K) { auto ResolverI = Resolvers.find(K); assert(ResolverI != Resolvers.end() && "No resolver for module K"); diff --git a/lib/ExecutionEngine/Orc/OrcError.cpp b/lib/ExecutionEngine/Orc/OrcError.cpp index f4102b359a6b..e6e9a095319c 100644 --- a/lib/ExecutionEngine/Orc/OrcError.cpp +++ b/lib/ExecutionEngine/Orc/OrcError.cpp @@ -1,9 +1,8 @@ //===---------------- OrcError.cpp - Error codes for ORC ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp index 617bc2fc64b5..772a9c2c4ab2 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp @@ -1,9 +1,8 @@ //===-------- OrcMCJITReplacement.cpp - Orc-based MCJIT replacement -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -128,7 +127,8 @@ void OrcMCJITReplacement::runStaticConstructorsDestructors(bool isDtors) { auto &CtorDtorsMap = isDtors ? UnexecutedDestructors : UnexecutedConstructors; for (auto &KV : CtorDtorsMap) - cantFail(LegacyCtorDtorRunner<LazyEmitLayerT>(std::move(KV.second), KV.first) + cantFail(LegacyCtorDtorRunner<LazyEmitLayerT>( + AcknowledgeORCv1Deprecation, std::move(KV.second), KV.first) .runViaLayer(LazyEmitLayer)); CtorDtorsMap.clear(); diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 36e7e83a8bab..169dc8f1d02b 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -1,9 +1,8 @@ //===- OrcMCJITReplacement.h - Orc based MCJIT replacement ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -177,8 +176,8 @@ class OrcMCJITReplacement : public ExecutionEngine { for (auto &S : Symbols) { if (auto Sym = M.findMangledSymbol(*S)) { if (auto Addr = Sym.getAddress()) { - Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); - Query->notifySymbolReady(); + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); NewSymbolsResolved = true; } else { M.ES.legacyFailQuery(*Query, Addr.takeError()); @@ -190,8 +189,8 @@ class OrcMCJITReplacement : public ExecutionEngine { } else { if (auto Sym2 = M.ClientResolver->findSymbol(*S)) { if (auto Addr = Sym2.getAddress()) { - Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); - Query->notifySymbolReady(); + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); NewSymbolsResolved = true; } else { M.ES.legacyFailQuery(*Query, Addr.takeError()); @@ -205,11 +204,8 @@ class OrcMCJITReplacement : public ExecutionEngine { } } - if (NewSymbolsResolved && Query->isFullyResolved()) - Query->handleFullyResolved(); - - if (NewSymbolsResolved && Query->isFullyReady()) - Query->handleFullyReady(); + if (NewSymbolsResolved && Query->isComplete()) + Query->handleComplete(); return UnresolvedSymbols; } @@ -236,24 +232,24 @@ public: OrcMCJITReplacement(std::shared_ptr<MCJITMemoryManager> MemMgr, std::shared_ptr<LegacyJITSymbolResolver> ClientResolver, std::unique_ptr<TargetMachine> TM) - : ExecutionEngine(TM->createDataLayout()), - TM(std::move(TM)), + : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), MemMgr( std::make_shared<MCJITReplacementMemMgr>(*this, std::move(MemMgr))), Resolver(std::make_shared<LinkingORCResolver>(*this)), ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), NotifyFinalized(*this), ObjectLayer( - ES, + AcknowledgeORCv1Deprecation, ES, [this](VModuleKey K) { return ObjectLayerT::Resources{this->MemMgr, this->Resolver}; }, NotifyObjectLoaded, NotifyFinalized), - CompileLayer(ObjectLayer, SimpleCompiler(*this->TM), + CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, + SimpleCompiler(*this->TM), [this](VModuleKey K, std::unique_ptr<Module> M) { Modules.push_back(std::move(M)); }), - LazyEmitLayer(CompileLayer) {} + LazyEmitLayer(AcknowledgeORCv1Deprecation, CompileLayer) {} static void Register() { OrcMCJITReplacementCtor = createOrcMCJITReplacement; diff --git a/lib/ExecutionEngine/Orc/RPCUtils.cpp b/lib/ExecutionEngine/Orc/RPCUtils.cpp index 2a7ab5ca8180..367b3639f841 100644 --- a/lib/ExecutionEngine/Orc/RPCUtils.cpp +++ b/lib/ExecutionEngine/Orc/RPCUtils.cpp @@ -1,9 +1,8 @@ //===--------------- RPCUtils.cpp - RPCUtils implementation ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 299d76183cd4..b22ecd5f80a1 100644 --- a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -1,9 +1,8 @@ //===-- RTDyldObjectLinkingLayer.cpp - RuntimeDyld backed ORC ObjectLayer -===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -42,9 +41,6 @@ public: OnResolved(Result); }; - // We're not waiting for symbols to be ready. Just log any errors. - auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; - // Register dependencies for all symbols contained in this set. auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { MR.addDependenciesForAll(Deps); @@ -53,8 +49,8 @@ public: JITDylibSearchList SearchOrder; MR.getTargetJITDylib().withSearchOrderDo( [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); - ES.lookup(SearchOrder, InternedSymbols, OnResolvedWithUnwrap, OnReady, - RegisterDependencies); + ES.lookup(SearchOrder, InternedSymbols, SymbolState::Resolved, + OnResolvedWithUnwrap, RegisterDependencies); } Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) { @@ -78,11 +74,8 @@ namespace llvm { namespace orc { RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( - ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, - NotifyLoadedFunction NotifyLoaded, NotifyEmittedFunction NotifyEmitted) - : ObjectLayer(ES), GetMemoryManager(GetMemoryManager), - NotifyLoaded(std::move(NotifyLoaded)), - NotifyEmitted(std::move(NotifyEmitted)) {} + ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager) + : ObjectLayer(ES), GetMemoryManager(GetMemoryManager) {} void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, std::unique_ptr<MemoryBuffer> O) { @@ -96,7 +89,13 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, auto &ES = getExecutionSession(); - auto Obj = object::ObjectFile::createObjectFile(*O); + // Create a MemoryBufferRef backed MemoryBuffer (i.e. shallow) copy of the + // the underlying buffer to pass into RuntimeDyld. This allows us to hold + // ownership of the real underlying buffer and return it to the user once + // the object has been emitted. + auto ObjBuffer = MemoryBuffer::getMemBuffer(O->getMemBufferRef(), false); + + auto Obj = object::ObjectFile::createObjectFile(*ObjBuffer); if (!Obj) { getExecutionSession().reportError(Obj.takeError()); @@ -134,13 +133,8 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, JITDylibSearchOrderResolver Resolver(*SharedR); - /* Thoughts on proper cross-dylib weak symbol handling: - * - * Change selection of canonical defs to be a manually triggered process, and - * add a 'canonical' bit to symbol definitions. When canonical def selection - * is triggered, sweep the JITDylibs to mark defs as canonical, discard - * duplicate defs. - */ + // FIXME: Switch to move-capture for the 'O' buffer once we have c++14. + MemoryBuffer *UnownedObjBuffer = O.release(); jitLinkForORC( **Obj, std::move(O), *MemMgr, Resolver, ProcessAllSections, [this, K, SharedR, &Obj, InternalSymbols]( @@ -149,8 +143,9 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, return onObjLoad(K, *SharedR, **Obj, std::move(LoadedObjInfo), ResolvedSymbols, *InternalSymbols); }, - [this, K, SharedR](Error Err) { - onObjEmit(K, *SharedR, std::move(Err)); + [this, K, SharedR, UnownedObjBuffer](Error Err) { + std::unique_ptr<MemoryBuffer> ObjBuffer(UnownedObjBuffer); + onObjEmit(K, std::move(ObjBuffer), *SharedR, std::move(Err)); }); } @@ -177,7 +172,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( auto I = R.getSymbols().find(InternedName); if (OverrideObjectFlags && I != R.getSymbols().end()) - Flags = JITSymbolFlags::stripTransientFlags(I->second); + Flags = I->second; else if (AutoClaimObjectSymbols && I == R.getSymbols().end()) ExtraSymbolsToClaim[InternedName] = Flags; } @@ -189,7 +184,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( if (auto Err = R.defineMaterializing(ExtraSymbolsToClaim)) return Err; - R.resolve(Symbols); + R.notifyResolved(Symbols); if (NotifyLoaded) NotifyLoaded(K, Obj, *LoadedObjInfo); @@ -197,20 +192,29 @@ Error RTDyldObjectLinkingLayer::onObjLoad( return Error::success(); } -void RTDyldObjectLinkingLayer::onObjEmit(VModuleKey K, - MaterializationResponsibility &R, - Error Err) { +void RTDyldObjectLinkingLayer::onObjEmit( + VModuleKey K, std::unique_ptr<MemoryBuffer> ObjBuffer, + MaterializationResponsibility &R, Error Err) { if (Err) { getExecutionSession().reportError(std::move(Err)); R.failMaterialization(); return; } - R.emit(); + R.notifyEmitted(); if (NotifyEmitted) - NotifyEmitted(K); + NotifyEmitted(K, std::move(ObjBuffer)); } +LegacyRTDyldObjectLinkingLayer::LegacyRTDyldObjectLinkingLayer( + ExecutionSession &ES, ResourcesGetter GetResources, + NotifyLoadedFtor NotifyLoaded, NotifyFinalizedFtor NotifyFinalized, + NotifyFreedFtor NotifyFreed) + : ES(ES), GetResources(std::move(GetResources)), + NotifyLoaded(std::move(NotifyLoaded)), + NotifyFinalized(std::move(NotifyFinalized)), + NotifyFreed(std::move(NotifyFreed)), ProcessAllSections(false) {} + } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp index 9525b168fbd3..4cb7376758a7 100644 --- a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp +++ b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp @@ -1,10 +1,9 @@ //===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities //h-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// |