diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine/Orc')
22 files changed, 3578 insertions, 1072 deletions
diff --git a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index f26835ff8a085..9e38dc36faae7 100644 --- a/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -7,8 +7,12 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" + +#include "llvm/ADT/Hashing.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" +#include "llvm/Support/FormatVariadic.h" using namespace llvm; using namespace llvm::orc; @@ -35,7 +39,7 @@ static ThreadSafeModule extractSubModule(ThreadSafeModule &TSM, Constant *Aliasee = A.getAliasee(); assert(A.hasName() && "Anonymous alias?"); assert(Aliasee->hasName() && "Anonymous aliasee"); - std::string AliasName = A.getName(); + std::string AliasName = std::string(A.getName()); if (isa<Function>(Aliasee)) { auto *F = cloneFunctionDecl(*A.getParent(), *cast<Function>(Aliasee)); @@ -67,17 +71,19 @@ namespace orc { class PartitioningIRMaterializationUnit : public IRMaterializationUnit { public: - PartitioningIRMaterializationUnit(ExecutionSession &ES, ThreadSafeModule TSM, - VModuleKey K, CompileOnDemandLayer &Parent) - : IRMaterializationUnit(ES, std::move(TSM), std::move(K)), + PartitioningIRMaterializationUnit(ExecutionSession &ES, + const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM, VModuleKey K, + CompileOnDemandLayer &Parent) + : IRMaterializationUnit(ES, MO, std::move(TSM), std::move(K)), Parent(Parent) {} PartitioningIRMaterializationUnit( - ThreadSafeModule TSM, SymbolFlagsMap SymbolFlags, - SymbolNameToDefinitionMap SymbolToDefinition, + ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, + SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition, CompileOnDemandLayer &Parent) : IRMaterializationUnit(std::move(TSM), std::move(K), - std::move(SymbolFlags), + std::move(SymbolFlags), std::move(InitSymbol), std::move(SymbolToDefinition)), Parent(Parent) {} @@ -111,7 +117,8 @@ CompileOnDemandLayer::compileWholeModule(GlobalValueSet Requested) { CompileOnDemandLayer::CompileOnDemandLayer( ExecutionSession &ES, IRLayer &BaseLayer, LazyCallThroughManager &LCTMgr, IndirectStubsManagerBuilder BuildIndirectStubsManager) - : IRLayer(ES), BaseLayer(BaseLayer), LCTMgr(LCTMgr), + : IRLayer(ES, BaseLayer.getManglingOptions()), BaseLayer(BaseLayer), + LCTMgr(LCTMgr), BuildIndirectStubsManager(std::move(BuildIndirectStubsManager)) {} void CompileOnDemandLayer::setPartitionFunction(PartitionFunction Partition) { @@ -136,36 +143,34 @@ void CompileOnDemandLayer::emit(MaterializationResponsibility R, TSM.withModuleDo([&](Module &M) { // First, do some cleanup on the module: cleanUpModule(M); - - MangleAndInterner Mangle(ES, M.getDataLayout()); - for (auto &GV : M.global_values()) { - if (GV.isDeclaration() || GV.hasLocalLinkage() || - GV.hasAppendingLinkage()) - continue; - - auto Name = Mangle(GV.getName()); - auto Flags = JITSymbolFlags::fromGlobalValue(GV); - if (Flags.isCallable()) - Callables[Name] = SymbolAliasMapEntry(Name, Flags); - else - NonCallables[Name] = SymbolAliasMapEntry(Name, Flags); - } }); + for (auto &KV : R.getSymbols()) { + auto &Name = KV.first; + auto &Flags = KV.second; + if (Flags.isCallable()) + Callables[Name] = SymbolAliasMapEntry(Name, Flags); + else + NonCallables[Name] = SymbolAliasMapEntry(Name, Flags); + } + // Create a partitioning materialization unit and lodge it with the // implementation dylib. if (auto Err = PDR.getImplDylib().define( std::make_unique<PartitioningIRMaterializationUnit>( - ES, std::move(TSM), R.getVModuleKey(), *this))) { + ES, *getManglingOptions(), std::move(TSM), R.getVModuleKey(), + *this))) { ES.reportError(std::move(Err)); R.failMaterialization(); return; } - R.replace(reexports(PDR.getImplDylib(), std::move(NonCallables), - JITDylibLookupFlags::MatchAllSymbols)); - R.replace(lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(), - std::move(Callables), AliaseeImpls)); + if (!NonCallables.empty()) + R.replace(reexports(PDR.getImplDylib(), std::move(NonCallables), + JITDylibLookupFlags::MatchAllSymbols)); + if (!Callables.empty()) + R.replace(lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(), + std::move(Callables), AliaseeImpls)); } CompileOnDemandLayer::PerDylibResources & @@ -173,21 +178,22 @@ CompileOnDemandLayer::getPerDylibResources(JITDylib &TargetD) { auto I = DylibResources.find(&TargetD); if (I == DylibResources.end()) { auto &ImplD = - getExecutionSession().createJITDylib(TargetD.getName() + ".impl"); - TargetD.withSearchOrderDo( - [&](const JITDylibSearchOrder &TargetSearchOrder) { - auto NewSearchOrder = TargetSearchOrder; - assert( - !NewSearchOrder.empty() && - NewSearchOrder.front().first == &TargetD && - NewSearchOrder.front().second == - JITDylibLookupFlags::MatchAllSymbols && - "TargetD must be at the front of its own search order and match " - "non-exported symbol"); - NewSearchOrder.insert(std::next(NewSearchOrder.begin()), - {&ImplD, JITDylibLookupFlags::MatchAllSymbols}); - ImplD.setSearchOrder(std::move(NewSearchOrder), false); - }); + getExecutionSession().createBareJITDylib(TargetD.getName() + ".impl"); + JITDylibSearchOrder NewLinkOrder; + TargetD.withLinkOrderDo([&](const JITDylibSearchOrder &TargetLinkOrder) { + NewLinkOrder = TargetLinkOrder; + }); + + assert(!NewLinkOrder.empty() && NewLinkOrder.front().first == &TargetD && + NewLinkOrder.front().second == + JITDylibLookupFlags::MatchAllSymbols && + "TargetD must be at the front of its own search order and match " + "non-exported symbol"); + NewLinkOrder.insert(std::next(NewLinkOrder.begin()), + {&ImplD, JITDylibLookupFlags::MatchAllSymbols}); + ImplD.setLinkOrder(NewLinkOrder, false); + TargetD.setLinkOrder(std::move(NewLinkOrder), false); + PerDylibResources PDR(ImplD, BuildIndirectStubsManager()); I = DylibResources.insert(std::make_pair(&TargetD, std::move(PDR))).first; } @@ -252,8 +258,15 @@ void CompileOnDemandLayer::emitPartition( auto &ES = getExecutionSession(); GlobalValueSet RequestedGVs; for (auto &Name : R.getRequestedSymbols()) { - assert(Defs.count(Name) && "No definition for symbol"); - RequestedGVs.insert(Defs[Name]); + if (Name == R.getInitializerSymbol()) + TSM.withModuleDo([&](Module &M) { + for (auto &GV : getStaticInitGVs(M)) + RequestedGVs.insert(&GV); + }); + else { + assert(Defs.count(Name) && "No definition for symbol"); + RequestedGVs.insert(Defs[Name]); + } } /// Perform partitioning with the context lock held, since the partition @@ -273,7 +286,8 @@ void CompileOnDemandLayer::emitPartition( // If the partition is empty, return the whole module to the symbol table. if (GVsToExtract->empty()) { R.replace(std::make_unique<PartitioningIRMaterializationUnit>( - std::move(TSM), R.getSymbols(), std::move(Defs), *this)); + std::move(TSM), R.getVModuleKey(), R.getSymbols(), + R.getInitializerSymbol(), std::move(Defs), *this)); return; } @@ -284,29 +298,52 @@ void CompileOnDemandLayer::emitPartition( // // FIXME: We apply this promotion once per partitioning. It's safe, but // overkill. - auto ExtractedTSM = TSM.withModuleDo([&](Module &M) -> Expected<ThreadSafeModule> { auto PromotedGlobals = PromoteSymbols(M); if (!PromotedGlobals.empty()) { + MangleAndInterner Mangle(ES, M.getDataLayout()); SymbolFlagsMap SymbolFlags; - for (auto &GV : PromotedGlobals) - SymbolFlags[Mangle(GV->getName())] = - JITSymbolFlags::fromGlobalValue(*GV); + IRSymbolMapper::add(ES, *getManglingOptions(), + PromotedGlobals, SymbolFlags); + if (auto Err = R.defineMaterializing(SymbolFlags)) return std::move(Err); } expandPartition(*GVsToExtract); + // Submodule name is given by hashing the names of the globals. + std::string SubModuleName; + { + std::vector<const GlobalValue*> HashGVs; + HashGVs.reserve(GVsToExtract->size()); + for (auto *GV : *GVsToExtract) + HashGVs.push_back(GV); + llvm::sort(HashGVs, [](const GlobalValue *LHS, const GlobalValue *RHS) { + return LHS->getName() < RHS->getName(); + }); + hash_code HC(0); + for (auto *GV : HashGVs) { + assert(GV->hasName() && "All GVs to extract should be named by now"); + auto GVName = GV->getName(); + HC = hash_combine(HC, hash_combine_range(GVName.begin(), GVName.end())); + } + raw_string_ostream(SubModuleName) + << ".submodule." + << formatv(sizeof(size_t) == 8 ? "{0:x16}" : "{0:x8}", + static_cast<size_t>(HC)) + << ".ll"; + } + // Extract the requested partiton (plus any necessary aliases) and // put the rest back into the impl dylib. auto ShouldExtract = [&](const GlobalValue &GV) -> bool { return GVsToExtract->count(&GV); }; - return extractSubModule(TSM, ".submodule", ShouldExtract); + return extractSubModule(TSM, SubModuleName , ShouldExtract); }); if (!ExtractedTSM) { @@ -316,7 +353,7 @@ void CompileOnDemandLayer::emitPartition( } R.replace(std::make_unique<PartitioningIRMaterializationUnit>( - ES, std::move(TSM), R.getVModuleKey(), *this)); + ES, *getManglingOptions(), std::move(TSM), R.getVModuleKey(), *this)); BaseLayer.emit(std::move(R), std::move(*ExtractedTSM)); } diff --git a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp index f5671d90420a7..f8efed15edea3 100644 --- a/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -24,11 +24,20 @@ namespace llvm { namespace orc { +IRSymbolMapper::ManglingOptions +irManglingOptionsFromTargetOptions(const TargetOptions &Opts) { + IRSymbolMapper::ManglingOptions MO; + + MO.EmulatedTLS = Opts.EmulatedTLS; + + return MO; +} + /// Compile a Module to an ObjectFile. -SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) { +Expected<SimpleCompiler::CompileResult> SimpleCompiler::operator()(Module &M) { CompileResult CachedObject = tryToLoadFromObjectCache(M); if (CachedObject) - return CachedObject; + return std::move(CachedObject); SmallVector<char, 0> ObjBufferSV; @@ -38,7 +47,8 @@ SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) { legacy::PassManager PM; MCContext *Ctx; if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) - llvm_unreachable("Target does not support MC emission."); + return make_error<StringError>("Target does not support MC emission", + inconvertibleErrorCode()); PM.run(M); } @@ -47,14 +57,11 @@ SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) { auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); - if (Obj) { - notifyObjectCompiled(M, *ObjBuffer); - return std::move(ObjBuffer); - } + if (!Obj) + return Obj.takeError(); - // TODO: Actually report errors helpfully. - consumeError(Obj.takeError()); - return nullptr; + notifyObjectCompiled(M, *ObjBuffer); + return std::move(ObjBuffer); } SimpleCompiler::CompileResult @@ -73,9 +80,11 @@ void SimpleCompiler::notifyObjectCompiled(const Module &M, ConcurrentIRCompiler::ConcurrentIRCompiler(JITTargetMachineBuilder JTMB, ObjectCache *ObjCache) - : JTMB(std::move(JTMB)), ObjCache(ObjCache) {} + : IRCompiler(irManglingOptionsFromTargetOptions(JTMB.getOptions())), + JTMB(std::move(JTMB)), ObjCache(ObjCache) {} -std::unique_ptr<MemoryBuffer> ConcurrentIRCompiler::operator()(Module &M) { +Expected<std::unique_ptr<MemoryBuffer>> +ConcurrentIRCompiler::operator()(Module &M) { auto TM = cantFail(JTMB.createTargetMachine()); SimpleCompiler C(*TM, ObjCache); return C(M); diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index 63ef889dae464..bad13cfebbc6b 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -10,302 +10,30 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/Config/llvm-config.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" -#include "llvm/IR/Mangler.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/Format.h" +#include <condition_variable> #if LLVM_ENABLE_THREADS #include <future> #endif #define DEBUG_TYPE "orc" -using namespace llvm; - -namespace { - -#ifndef NDEBUG - -cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(true), - cl::desc("debug print hidden symbols defined by " - "materialization units"), - cl::Hidden); - -cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(true), - cl::desc("debug print callable symbols defined by " - "materialization units"), - cl::Hidden); - -cl::opt<bool> PrintData("debug-orc-print-data", cl::init(true), - cl::desc("debug print data symbols defined by " - "materialization units"), - cl::Hidden); - -#endif // NDEBUG - -// SetPrinter predicate that prints every element. -template <typename T> struct PrintAll { - bool operator()(const T &E) { return true; } -}; - -bool anyPrintSymbolOptionSet() { -#ifndef NDEBUG - return PrintHidden || PrintCallable || PrintData; -#else - return false; -#endif // NDEBUG -} - -bool flagsMatchCLOpts(const JITSymbolFlags &Flags) { -#ifndef NDEBUG - // Bail out early if this is a hidden symbol and we're not printing hiddens. - if (!PrintHidden && !Flags.isExported()) - return false; - - // Return true if this is callable and we're printing callables. - if (PrintCallable && Flags.isCallable()) - return true; - - // Return true if this is data and we're printing data. - if (PrintData && !Flags.isCallable()) - return true; - - // otherwise return false. - return false; -#else - return false; -#endif // NDEBUG -} - -// Prints a sequence of items, filtered by an user-supplied predicate. -template <typename Sequence, - typename Pred = PrintAll<typename Sequence::value_type>> -class SequencePrinter { -public: - SequencePrinter(const Sequence &S, char OpenSeq, char CloseSeq, - Pred ShouldPrint = Pred()) - : S(S), OpenSeq(OpenSeq), CloseSeq(CloseSeq), - ShouldPrint(std::move(ShouldPrint)) {} - - void printTo(llvm::raw_ostream &OS) const { - bool PrintComma = false; - OS << OpenSeq; - for (auto &E : S) { - if (ShouldPrint(E)) { - if (PrintComma) - OS << ','; - OS << ' ' << E; - PrintComma = true; - } - } - OS << ' ' << CloseSeq; - } - -private: - const Sequence &S; - char OpenSeq; - char CloseSeq; - mutable Pred ShouldPrint; -}; - -template <typename Sequence, typename Pred> -SequencePrinter<Sequence, Pred> printSequence(const Sequence &S, char OpenSeq, - char CloseSeq, Pred P = Pred()) { - return SequencePrinter<Sequence, Pred>(S, OpenSeq, CloseSeq, std::move(P)); -} - -// Render a SequencePrinter by delegating to its printTo method. -template <typename Sequence, typename Pred> -llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, - const SequencePrinter<Sequence, Pred> &Printer) { - Printer.printTo(OS); - return OS; -} - -struct PrintSymbolFlagsMapElemsMatchingCLOpts { - bool operator()(const orc::SymbolFlagsMap::value_type &KV) { - return flagsMatchCLOpts(KV.second); - } -}; - -struct PrintSymbolMapElemsMatchingCLOpts { - bool operator()(const orc::SymbolMap::value_type &KV) { - return flagsMatchCLOpts(KV.second.getFlags()); - } -}; - -} // end anonymous namespace - namespace llvm { namespace orc { char FailedToMaterialize::ID = 0; char SymbolsNotFound::ID = 0; char SymbolsCouldNotBeRemoved::ID = 0; +char MissingSymbolDefinitions::ID = 0; +char UnexpectedSymbolDefinitions::ID = 0; RegisterDependenciesFunction NoDependenciesToRegister = RegisterDependenciesFunction(); void MaterializationUnit::anchor() {} -raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym) { - return OS << *Sym; -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { - return OS << printSequence(Symbols, '{', '}', PrintAll<SymbolStringPtr>()); -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols) { - return OS << printSequence(Symbols, '[', ']', PrintAll<SymbolStringPtr>()); -} - -raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { - if (Flags.hasError()) - OS << "[*ERROR*]"; - if (Flags.isCallable()) - OS << "[Callable]"; - else - OS << "[Data]"; - if (Flags.isWeak()) - OS << "[Weak]"; - else if (Flags.isCommon()) - OS << "[Common]"; - - if (!Flags.isExported()) - OS << "[Hidden]"; - - return OS; -} - -raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) { - return OS << format("0x%016" PRIx64, Sym.getAddress()) << " " - << Sym.getFlags(); -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV) { - return OS << "(\"" << KV.first << "\", " << KV.second << ")"; -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV) { - return OS << "(\"" << KV.first << "\": " << KV.second << ")"; -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) { - return OS << printSequence(SymbolFlags, '{', '}', - PrintSymbolFlagsMapElemsMatchingCLOpts()); -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) { - return OS << printSequence(Symbols, '{', '}', - PrintSymbolMapElemsMatchingCLOpts()); -} - -raw_ostream &operator<<(raw_ostream &OS, - const SymbolDependenceMap::value_type &KV) { - return OS << "(" << KV.first << ", " << KV.second << ")"; -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) { - return OS << printSequence(Deps, '{', '}', - PrintAll<SymbolDependenceMap::value_type>()); -} - -raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU) { - OS << "MU@" << &MU << " (\"" << MU.getName() << "\""; - if (anyPrintSymbolOptionSet()) - OS << ", " << MU.getSymbols(); - return OS << ")"; -} - -raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K) { - switch (K) { - case LookupKind::Static: - return OS << "Static"; - case LookupKind::DLSym: - return OS << "DLSym"; - } - llvm_unreachable("Invalid lookup kind"); -} - -raw_ostream &operator<<(raw_ostream &OS, - const JITDylibLookupFlags &JDLookupFlags) { - switch (JDLookupFlags) { - case JITDylibLookupFlags::MatchExportedSymbolsOnly: - return OS << "MatchExportedSymbolsOnly"; - case JITDylibLookupFlags::MatchAllSymbols: - return OS << "MatchAllSymbols"; - } - llvm_unreachable("Invalid JITDylib lookup flags"); -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags) { - switch (LookupFlags) { - case SymbolLookupFlags::RequiredSymbol: - return OS << "RequiredSymbol"; - case SymbolLookupFlags::WeaklyReferencedSymbol: - return OS << "WeaklyReferencedSymbol"; - } - llvm_unreachable("Invalid symbol lookup flags"); -} - -raw_ostream &operator<<(raw_ostream &OS, - const SymbolLookupSet::value_type &KV) { - return OS << "(" << KV.first << ", " << KV.second << ")"; -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet &LookupSet) { - return OS << printSequence(LookupSet, '{', '}', - PrintAll<SymbolLookupSet::value_type>()); -} - -raw_ostream &operator<<(raw_ostream &OS, - const JITDylibSearchOrder &SearchOrder) { - OS << "["; - if (!SearchOrder.empty()) { - assert(SearchOrder.front().first && - "JITDylibList entries must not be null"); - OS << " (\"" << SearchOrder.front().first->getName() << "\", " - << SearchOrder.begin()->second << ")"; - for (auto &KV : - make_range(std::next(SearchOrder.begin(), 1), SearchOrder.end())) { - assert(KV.first && "JITDylibList entries must not be null"); - OS << ", (\"" << KV.first->getName() << "\", " << KV.second << ")"; - } - } - OS << " ]"; - return OS; -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) { - OS << "{"; - for (auto &KV : Aliases) - OS << " " << *KV.first << ": " << KV.second.Aliasee << " " - << KV.second.AliasFlags; - OS << " }"; - return OS; -} - -raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { - switch (S) { - case SymbolState::Invalid: - return OS << "Invalid"; - case SymbolState::NeverSearched: - return OS << "Never-Searched"; - case SymbolState::Materializing: - return OS << "Materializing"; - case SymbolState::Resolved: - return OS << "Resolved"; - case SymbolState::Emitted: - return OS << "Emitted"; - case SymbolState::Ready: - return OS << "Ready"; - } - llvm_unreachable("Invalid state"); -} - FailedToMaterialize::FailedToMaterialize( std::shared_ptr<SymbolDependenceMap> Symbols) : Symbols(std::move(Symbols)) { @@ -352,6 +80,24 @@ void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const { OS << "Symbols could not be removed: " << Symbols; } +std::error_code MissingSymbolDefinitions::convertToErrorCode() const { + return orcError(OrcErrorCode::MissingSymbolDefinitions); +} + +void MissingSymbolDefinitions::log(raw_ostream &OS) const { + OS << "Missing definitions in module " << ModuleName + << ": " << Symbols; +} + +std::error_code UnexpectedSymbolDefinitions::convertToErrorCode() const { + return orcError(OrcErrorCode::UnexpectedSymbolDefinitions); +} + +void UnexpectedSymbolDefinitions::log(raw_ostream &OS) const { + OS << "Unexpected definitions in module " << ModuleName + << ": " << Symbols; +} + AsynchronousSymbolQuery::AsynchronousSymbolQuery( const SymbolLookupSet &Symbols, SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete) @@ -372,7 +118,13 @@ void AsynchronousSymbolQuery::notifySymbolMetRequiredState( assert(I != ResolvedSymbols.end() && "Resolving symbol outside the requested set"); assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); - I->second = std::move(Sym); + + // If this is a materialization-side-effects-only symbol then drop it, + // otherwise update its map entry with its resolved address. + if (Sym.getFlags().hasMaterializationSideEffectsOnly()) + ResolvedSymbols.erase(I); + else + I->second = std::move(Sym); --OutstandingSymbolsCount; } @@ -413,6 +165,14 @@ void AsynchronousSymbolQuery::removeQueryDependence( QueryRegistrations.erase(QRI); } +void AsynchronousSymbolQuery::dropSymbol(const SymbolStringPtr &Name) { + auto I = ResolvedSymbols.find(Name); + assert(I != ResolvedSymbols.end() && + "Redundant removal of weakly-referenced symbol"); + ResolvedSymbols.erase(I); + --OutstandingSymbolsCount; +} + void AsynchronousSymbolQuery::detach() { ResolvedSymbols.clear(); OutstandingSymbolsCount = 0; @@ -421,24 +181,18 @@ void AsynchronousSymbolQuery::detach() { QueryRegistrations.clear(); } -MaterializationResponsibility::MaterializationResponsibility( - JITDylib &JD, SymbolFlagsMap SymbolFlags, VModuleKey K) - : JD(JD), SymbolFlags(std::move(SymbolFlags)), K(std::move(K)) { - assert(!this->SymbolFlags.empty() && "Materializing nothing?"); -} - MaterializationResponsibility::~MaterializationResponsibility() { assert(SymbolFlags.empty() && "All symbols should have been explicitly materialized or failed"); } SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { - return JD.getRequestedSymbols(SymbolFlags); + return JD->getRequestedSymbols(SymbolFlags); } Error MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { LLVM_DEBUG({ - dbgs() << "In " << JD.getName() << " resolving " << Symbols << "\n"; + dbgs() << "In " << JD->getName() << " resolving " << Symbols << "\n"; }); #ifndef NDEBUG for (auto &KV : Symbols) { @@ -446,21 +200,23 @@ Error MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { auto I = SymbolFlags.find(KV.first); assert(I != SymbolFlags.end() && "Resolving symbol outside this responsibility set"); + assert(!I->second.hasMaterializationSideEffectsOnly() && + "Can't resolve materialization-side-effects-only symbol"); assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) && "Resolving symbol with incorrect flags"); } #endif - return JD.resolve(Symbols); + return JD->resolve(Symbols); } Error MaterializationResponsibility::notifyEmitted() { LLVM_DEBUG({ - dbgs() << "In " << JD.getName() << " emitting " << SymbolFlags << "\n"; + dbgs() << "In " << JD->getName() << " emitting " << SymbolFlags << "\n"; }); - if (auto Err = JD.emit(SymbolFlags)) + if (auto Err = JD->emit(SymbolFlags)) return Err; SymbolFlags.clear(); @@ -468,44 +224,59 @@ Error MaterializationResponsibility::notifyEmitted() { } Error MaterializationResponsibility::defineMaterializing( - const SymbolFlagsMap &NewSymbolFlags) { - // Add the given symbols to this responsibility object. - // It's ok if we hit a duplicate here: In that case the new version will be - // discarded, and the JITDylib::defineMaterializing method will return a - // duplicate symbol error. - for (auto &KV : NewSymbolFlags) - SymbolFlags.insert(KV); + SymbolFlagsMap NewSymbolFlags) { - return JD.defineMaterializing(NewSymbolFlags); + LLVM_DEBUG({ + dbgs() << "In " << JD->getName() << " defining materializing symbols " + << NewSymbolFlags << "\n"; + }); + if (auto AcceptedDefs = JD->defineMaterializing(std::move(NewSymbolFlags))) { + // Add all newly accepted symbols to this responsibility object. + for (auto &KV : *AcceptedDefs) + SymbolFlags.insert(KV); + return Error::success(); + } else + return AcceptedDefs.takeError(); } void MaterializationResponsibility::failMaterialization() { LLVM_DEBUG({ - dbgs() << "In " << JD.getName() << " failing materialization for " + dbgs() << "In " << JD->getName() << " failing materialization for " << SymbolFlags << "\n"; }); JITDylib::FailedSymbolsWorklist Worklist; for (auto &KV : SymbolFlags) - Worklist.push_back(std::make_pair(&JD, KV.first)); + Worklist.push_back(std::make_pair(JD.get(), KV.first)); SymbolFlags.clear(); - JD.notifyFailed(std::move(Worklist)); + JD->notifyFailed(std::move(Worklist)); } void MaterializationResponsibility::replace( std::unique_ptr<MaterializationUnit> MU) { - for (auto &KV : MU->getSymbols()) + + // If the replacement MU is empty then return. + if (MU->getSymbols().empty()) + return; + + for (auto &KV : MU->getSymbols()) { + assert(SymbolFlags.count(KV.first) && + "Replacing definition outside this responsibility set"); SymbolFlags.erase(KV.first); + } - LLVM_DEBUG(JD.getExecutionSession().runSessionLocked([&]() { - dbgs() << "In " << JD.getName() << " replacing symbols with " << *MU + if (MU->getInitializerSymbol() == InitSymbol) + InitSymbol = nullptr; + + LLVM_DEBUG(JD->getExecutionSession().runSessionLocked([&]() { + dbgs() << "In " << JD->getName() << " replacing symbols with " << *MU << "\n"; });); - JD.replace(std::move(MU)); + JD->replace(std::move(MU)); } MaterializationResponsibility @@ -515,6 +286,7 @@ MaterializationResponsibility::delegate(const SymbolNameSet &Symbols, if (NewKey == VModuleKey()) NewKey = K; + SymbolStringPtr DelegatedInitSymbol; SymbolFlagsMap DelegatedFlags; for (auto &Name : Symbols) { @@ -524,29 +296,41 @@ MaterializationResponsibility::delegate(const SymbolNameSet &Symbols, "instance"); DelegatedFlags[Name] = std::move(I->second); + if (Name == InitSymbol) + std::swap(InitSymbol, DelegatedInitSymbol); + SymbolFlags.erase(I); } return MaterializationResponsibility(JD, std::move(DelegatedFlags), + std::move(DelegatedInitSymbol), std::move(NewKey)); } void MaterializationResponsibility::addDependencies( const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { + LLVM_DEBUG({ + dbgs() << "Adding dependencies for " << Name << ": " << Dependencies + << "\n"; + }); assert(SymbolFlags.count(Name) && "Symbol not covered by this MaterializationResponsibility instance"); - JD.addDependencies(Name, Dependencies); + JD->addDependencies(Name, Dependencies); } void MaterializationResponsibility::addDependenciesForAll( const SymbolDependenceMap &Dependencies) { + LLVM_DEBUG({ + dbgs() << "Adding dependencies for all symbols in " << SymbolFlags << ": " + << Dependencies << "\n"; + }); for (auto &KV : SymbolFlags) - JD.addDependencies(KV.first, Dependencies); + JD->addDependencies(KV.first, Dependencies); } AbsoluteSymbolsMaterializationUnit::AbsoluteSymbolsMaterializationUnit( SymbolMap Symbols, VModuleKey K) - : MaterializationUnit(extractFlags(Symbols), std::move(K)), + : MaterializationUnit(extractFlags(Symbols), nullptr, std::move(K)), Symbols(std::move(Symbols)) {} StringRef AbsoluteSymbolsMaterializationUnit::getName() const { @@ -577,7 +361,7 @@ AbsoluteSymbolsMaterializationUnit::extractFlags(const SymbolMap &Symbols) { ReExportsMaterializationUnit::ReExportsMaterializationUnit( JITDylib *SourceJD, JITDylibLookupFlags SourceJDLookupFlags, SymbolAliasMap Aliases, VModuleKey K) - : MaterializationUnit(extractFlags(Aliases), std::move(K)), + : MaterializationUnit(extractFlags(Aliases), nullptr, std::move(K)), SourceJD(SourceJD), SourceJDLookupFlags(SourceJDLookupFlags), Aliases(std::move(Aliases)) {} @@ -630,11 +414,13 @@ void ReExportsMaterializationUnit::materialize( SymbolAliasMap Aliases; }; - // Build a list of queries to issue. In each round we build the largest set of - // aliases that we can resolve without encountering a chain definition of the - // form Foo -> Bar, Bar -> Baz. Such a form would deadlock as the query would - // be waitin on a symbol that it itself had to resolve. Usually this will just - // involve one round and a single query. + // Build a list of queries to issue. In each round we build a query for the + // largest set of aliases that we can resolve without encountering a chain of + // aliases (e.g. Foo -> Bar, Bar -> Baz). Such a chain would deadlock as the + // query would be waiting on a symbol that it itself had to resolve. Creating + // a new query for each link in such a chain eliminates the possibility of + // deadlock. In practice chains are likely to be rare, and this algorithm will + // usually result in a single query to issue. std::vector<std::pair<SymbolLookupSet, std::shared_ptr<OnResolveInfo>>> QueryInfos; @@ -651,7 +437,10 @@ void ReExportsMaterializationUnit::materialize( continue; ResponsibilitySymbols.insert(KV.first); - QuerySymbols.add(KV.second.Aliasee); + QuerySymbols.add(KV.second.Aliasee, + KV.second.AliasFlags.hasMaterializationSideEffectsOnly() + ? SymbolLookupFlags::WeaklyReferencedSymbol + : SymbolLookupFlags::RequiredSymbol); QueryAliases[KV.first] = std::move(KV.second); } @@ -700,8 +489,13 @@ void ReExportsMaterializationUnit::materialize( if (Result) { SymbolMap ResolutionMap; for (auto &KV : QueryInfo->Aliases) { - assert(Result->count(KV.second.Aliasee) && + assert((KV.second.AliasFlags.hasMaterializationSideEffectsOnly() || + Result->count(KV.second.Aliasee)) && "Result map missing entry?"); + // Don't try to resolve materialization-side-effects-only symbols. + if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly()) + continue; + ResolutionMap[KV.first] = JITEvaluatedSymbol( (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); } @@ -809,31 +603,52 @@ void JITDylib::removeGenerator(DefinitionGenerator &G) { }); } -Error JITDylib::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { - return ES.runSessionLocked([&]() -> Error { +Expected<SymbolFlagsMap> +JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { + + return ES.runSessionLocked([&]() -> Expected<SymbolFlagsMap> { std::vector<SymbolTable::iterator> AddedSyms; + std::vector<SymbolFlagsMap::iterator> RejectedWeakDefs; - for (auto &KV : SymbolFlags) { - SymbolTable::iterator EntryItr; - bool Added; + for (auto SFItr = SymbolFlags.begin(), SFEnd = SymbolFlags.end(); + SFItr != SFEnd; ++SFItr) { - std::tie(EntryItr, Added) = - Symbols.insert(std::make_pair(KV.first, SymbolTableEntry(KV.second))); + auto &Name = SFItr->first; + auto &Flags = SFItr->second; - if (Added) { - AddedSyms.push_back(EntryItr); - EntryItr->second.setState(SymbolState::Materializing); - } else { - // Remove any symbols already added. - for (auto &SI : AddedSyms) - Symbols.erase(SI); + auto EntryItr = Symbols.find(Name); - // FIXME: Return all duplicates. - return make_error<DuplicateDefinition>(*KV.first); - } + // If the entry already exists... + if (EntryItr != Symbols.end()) { + + // If this is a strong definition then error out. + if (!Flags.isWeak()) { + // Remove any symbols already added. + for (auto &SI : AddedSyms) + Symbols.erase(SI); + + // FIXME: Return all duplicates. + return make_error<DuplicateDefinition>(std::string(*Name)); + } + + // Otherwise just make a note to discard this symbol after the loop. + RejectedWeakDefs.push_back(SFItr); + continue; + } else + EntryItr = + Symbols.insert(std::make_pair(Name, SymbolTableEntry(Flags))).first; + + AddedSyms.push_back(EntryItr); + EntryItr->second.setState(SymbolState::Materializing); } - return Error::success(); + // Remove any rejected weak definitions from the SymbolFlags map. + while (!RejectedWeakDefs.empty()) { + SymbolFlags.erase(RejectedWeakDefs.back()); + RejectedWeakDefs.pop_back(); + } + + return SymbolFlags; }); } @@ -847,8 +662,8 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { for (auto &KV : MU->getSymbols()) { auto SymI = Symbols.find(KV.first); assert(SymI != Symbols.end() && "Replacing unknown symbol"); - assert(SymI->second.isInMaterializationPhase() && - "Can not call replace on a symbol that is not materializing"); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace a symbol that ha is not materializing"); assert(!SymI->second.hasMaterializerAttached() && "Symbol should not have materializer attached already"); assert(UnmaterializedInfos.count(KV.first) == 0 && @@ -878,14 +693,21 @@ void JITDylib::replace(std::unique_ptr<MaterializationUnit> MU) { "Unexpected materializer entry in map"); SymI->second.setAddress(SymI->second.getAddress()); SymI->second.setMaterializerAttached(true); - UnmaterializedInfos[KV.first] = UMI; + + auto &UMIEntry = UnmaterializedInfos[KV.first]; + assert((!UMIEntry || !UMIEntry->MU) && + "Replacing symbol with materializer still attached"); + UMIEntry = UMI; } return nullptr; }); - if (MustRunMU) - ES.dispatchMaterialization(*this, std::move(MustRunMU)); + if (MustRunMU) { + auto MR = + MustRunMU->createMaterializationResponsibility(shared_from_this()); + ES.dispatchMaterialization(std::move(MustRunMU), std::move(MR)); + } } SymbolNameSet @@ -895,7 +717,9 @@ JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { for (auto &KV : SymbolFlags) { assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?"); - assert(Symbols.find(KV.first)->second.isInMaterializationPhase() && + assert(Symbols.find(KV.first)->second.getState() != + SymbolState::NeverSearched && + Symbols.find(KV.first)->second.getState() != SymbolState::Ready && "getRequestedSymbols can only be called for symbols that have " "started materializing"); auto I = MaterializingInfos.find(KV.first); @@ -913,9 +737,14 @@ JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { void JITDylib::addDependencies(const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { assert(Symbols.count(Name) && "Name not in symbol table"); - assert(Symbols[Name].isInMaterializationPhase() && + assert(Symbols[Name].getState() < SymbolState::Emitted && "Can not add dependencies for a symbol that is not materializing"); + LLVM_DEBUG({ + dbgs() << "In " << getName() << " adding dependencies for " + << *Name << ": " << Dependencies << "\n"; + }); + // If Name is already in an error state then just bail out. if (Symbols[Name].getFlags().hasError()) return; @@ -938,16 +767,18 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, // Check the sym entry for the dependency. auto OtherSymI = OtherJITDylib.Symbols.find(OtherSymbol); -#ifndef NDEBUG // Assert that this symbol exists and has not reached the ready state // already. assert(OtherSymI != OtherJITDylib.Symbols.end() && - (OtherSymI->second.getState() != SymbolState::Ready && - "Dependency on emitted/ready symbol")); -#endif + "Dependency on unknown symbol"); auto &OtherSymEntry = OtherSymI->second; + // If the other symbol is already in the Ready state then there's no + // dependency to add. + if (OtherSymEntry.getState() == SymbolState::Ready) + continue; + // If the dependency is in an error state then note this and continue, // we will move this symbol to the error state below. if (OtherSymEntry.getFlags().hasError()) { @@ -957,8 +788,6 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, // If the dependency was not in the error state then add it to // our list of dependencies. - assert(OtherJITDylib.MaterializingInfos.count(OtherSymbol) && - "No MaterializingInfo for dependency"); auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; if (OtherSymEntry.getState() == SymbolState::Emitted) @@ -1039,7 +868,11 @@ Error JITDylib::resolve(const SymbolMap &Resolved) { SymI->second.setFlags(ResolvedFlags); SymI->second.setState(SymbolState::Resolved); - auto &MI = MaterializingInfos[Name]; + auto MII = MaterializingInfos.find(Name); + if (MII == MaterializingInfos.end()) + continue; + + auto &MI = MII->second; for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { Q->notifySymbolMetRequiredState(Name, ResolvedSym); Q->removeQueryDependence(*this, Name); @@ -1071,6 +904,7 @@ Error JITDylib::resolve(const SymbolMap &Resolved) { Error JITDylib::emit(const SymbolFlagsMap &Emitted) { AsynchronousSymbolQuerySet CompletedQueries; SymbolNameSet SymbolsInErrorState; + DenseMap<JITDylib *, SymbolNameVector> ReadySymbols; ES.runSessionLocked([&, this]() { std::vector<SymbolTable::iterator> Worklist; @@ -1101,13 +935,21 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { auto &SymEntry = SymI->second; // Move symbol to the emitted state. - assert(SymEntry.getState() == SymbolState::Resolved && + assert(((SymEntry.getFlags().hasMaterializationSideEffectsOnly() && + SymEntry.getState() == SymbolState::Materializing) || + SymEntry.getState() == SymbolState::Resolved) && "Emitting from state other than Resolved"); SymEntry.setState(SymbolState::Emitted); auto MII = MaterializingInfos.find(Name); - assert(MII != MaterializingInfos.end() && - "Missing MaterializingInfo entry"); + + // If this symbol has no MaterializingInfo then it's trivially ready. + // Update its state and continue. + if (MII == MaterializingInfos.end()) { + SymEntry.setState(SymbolState::Ready); + continue; + } + auto &MI = MII->second; // For each dependant, transfer this node's emitted dependencies to @@ -1115,6 +957,7 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { // dependencies) then notify any pending queries. for (auto &KV : MI.Dependants) { auto &DependantJD = *KV.first; + auto &DependantJDReadySymbols = ReadySymbols[&DependantJD]; for (auto &DependantName : KV.second) { auto DependantMII = DependantJD.MaterializingInfos.find(DependantName); @@ -1154,6 +997,7 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { // Since this dependant is now ready, we erase its MaterializingInfo // and update its materializing state. DependantSymEntry.setState(SymbolState::Ready); + DependantJDReadySymbols.push_back(DependantName); for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { Q->notifySymbolMetRequiredState( @@ -1162,22 +1006,21 @@ Error JITDylib::emit(const SymbolFlagsMap &Emitted) { CompletedQueries.insert(Q); Q->removeQueryDependence(DependantJD, DependantName); } - - DependantJD.MaterializingInfos.erase(DependantMII); } } } + auto &ThisJDReadySymbols = ReadySymbols[this]; MI.Dependants.clear(); if (MI.UnemittedDependencies.empty()) { SymI->second.setState(SymbolState::Ready); + ThisJDReadySymbols.push_back(Name); for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); if (Q->isComplete()) CompletedQueries.insert(Q); Q->removeQueryDependence(*this, Name); } - MaterializingInfos.erase(MII); } } }); @@ -1317,30 +1160,29 @@ void JITDylib::notifyFailed(FailedSymbolsWorklist Worklist) { Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbolsMap)); } -void JITDylib::setSearchOrder(JITDylibSearchOrder NewSearchOrder, - bool SearchThisJITDylibFirst) { +void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder, + bool LinkAgainstThisJITDylibFirst) { ES.runSessionLocked([&]() { - if (SearchThisJITDylibFirst) { - SearchOrder.clear(); - if (NewSearchOrder.empty() || NewSearchOrder.front().first != this) - SearchOrder.push_back( + if (LinkAgainstThisJITDylibFirst) { + LinkOrder.clear(); + if (NewLinkOrder.empty() || NewLinkOrder.front().first != this) + LinkOrder.push_back( std::make_pair(this, JITDylibLookupFlags::MatchAllSymbols)); - SearchOrder.insert(SearchOrder.end(), NewSearchOrder.begin(), - NewSearchOrder.end()); + LinkOrder.insert(LinkOrder.end(), NewLinkOrder.begin(), + NewLinkOrder.end()); } else - SearchOrder = std::move(NewSearchOrder); + LinkOrder = std::move(NewLinkOrder); }); } -void JITDylib::addToSearchOrder(JITDylib &JD, - JITDylibLookupFlags JDLookupFlags) { - ES.runSessionLocked([&]() { SearchOrder.push_back({&JD, JDLookupFlags}); }); +void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) { + ES.runSessionLocked([&]() { LinkOrder.push_back({&JD, JDLookupFlags}); }); } -void JITDylib::replaceInSearchOrder(JITDylib &OldJD, JITDylib &NewJD, - JITDylibLookupFlags JDLookupFlags) { +void JITDylib::replaceInLinkOrder(JITDylib &OldJD, JITDylib &NewJD, + JITDylibLookupFlags JDLookupFlags) { ES.runSessionLocked([&]() { - for (auto &KV : SearchOrder) + for (auto &KV : LinkOrder) if (KV.first == &OldJD) { KV = {&NewJD, JDLookupFlags}; break; @@ -1348,14 +1190,14 @@ void JITDylib::replaceInSearchOrder(JITDylib &OldJD, JITDylib &NewJD, }); } -void JITDylib::removeFromSearchOrder(JITDylib &JD) { +void JITDylib::removeFromLinkOrder(JITDylib &JD) { ES.runSessionLocked([&]() { - auto I = std::find_if(SearchOrder.begin(), SearchOrder.end(), + auto I = std::find_if(LinkOrder.begin(), LinkOrder.end(), [&](const JITDylibSearchOrder::value_type &KV) { return KV.first == &JD; }); - if (I != SearchOrder.end()) - SearchOrder.erase(I); + if (I != LinkOrder.end()) + LinkOrder.erase(I); }); } @@ -1377,7 +1219,8 @@ Error JITDylib::remove(const SymbolNameSet &Names) { } // Note symbol materializing. - if (I->second.isInMaterializationPhase()) { + if (I->second.getState() != SymbolState::NeverSearched && + I->second.getState() != SymbolState::Ready) { Materializing.insert(Name); continue; } @@ -1498,6 +1341,12 @@ Error JITDylib::lodgeQueryImpl(MaterializationUnitList &MUs, if (SymI == Symbols.end()) return false; + // If we match against a materialization-side-effects only symbol then + // make sure it is weakly-referenced. Otherwise bail out with an error. + if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && + SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) + return make_error<SymbolsNotFound>(SymbolNameVector({Name})); + // If this is a non exported symbol and we're matching exported symbols // only then skip this symbol without removal. if (!SymI->second.getFlags().isExported() && @@ -1545,7 +1394,8 @@ Error JITDylib::lodgeQueryImpl(MaterializationUnitList &MUs, // Add the query to the PendingQueries list and continue, deleting the // element. - assert(SymI->second.isInMaterializationPhase() && + assert(SymI->second.getState() != SymbolState::NeverSearched && + SymI->second.getState() != SymbolState::Ready && "By this line the symbol should be materializing"); auto &MI = MaterializingInfos[Name]; MI.addQuery(Q); @@ -1601,8 +1451,11 @@ JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q, // Add MUs to the OutstandingMUs list. { std::lock_guard<std::recursive_mutex> Lock(ES.OutstandingMUsMutex); - for (auto &MU : MUs) - ES.OutstandingMUs.push_back(make_pair(this, std::move(MU))); + auto ThisJD = shared_from_this(); + for (auto &MU : MUs) { + auto MR = MU->createMaterializationResponsibility(ThisJD); + ES.OutstandingMUs.push_back(make_pair(std::move(MU), std::move(MR))); + } } ES.runOutstandingMUs(); @@ -1665,7 +1518,8 @@ bool JITDylib::lookupImpl( } // Add the query to the PendingQueries list. - assert(SymI->second.isInMaterializationPhase() && + assert(SymI->second.getState() != SymbolState::NeverSearched && + SymI->second.getState() != SymbolState::Ready && "By this line the symbol should be materializing"); auto &MI = MaterializingInfos[Name]; MI.addQuery(Q); @@ -1680,7 +1534,7 @@ void JITDylib::dump(raw_ostream &OS) { ES.runSessionLocked([&, this]() { OS << "JITDylib \"" << JITDylibName << "\" (ES: " << format("0x%016" PRIx64, reinterpret_cast<uintptr_t>(&ES)) << "):\n" - << "Search order: " << SearchOrder << "\n" + << "Link order: " << LinkOrder << "\n" << "Symbol table:\n"; for (auto &KV : Symbols) { @@ -1691,14 +1545,14 @@ void JITDylib::dump(raw_ostream &OS) { else OS << "<not resolved> "; - OS << KV.second.getState(); + OS << KV.second.getFlags() << " " << KV.second.getState(); if (KV.second.hasMaterializerAttached()) { OS << " (Materializer "; auto I = UnmaterializedInfos.find(KV.first); assert(I != UnmaterializedInfos.end() && "Lazy symbol should have UnmaterializedInfo"); - OS << I->second->MU.get() << ")\n"; + OS << I->second->MU.get() << ", " << I->second->MU->getName() << ")\n"; } else OS << "\n"; } @@ -1761,10 +1615,13 @@ JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) { JITDylib::JITDylib(ExecutionSession &ES, std::string Name) : ES(ES), JITDylibName(std::move(Name)) { - SearchOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols}); + LinkOrder.push_back({this, JITDylibLookupFlags::MatchAllSymbols}); } Error JITDylib::defineImpl(MaterializationUnit &MU) { + + LLVM_DEBUG({ dbgs() << " " << MU.getSymbols() << "\n"; }); + SymbolNameSet Duplicates; std::vector<SymbolStringPtr> ExistingDefsOverridden; std::vector<SymbolStringPtr> MUDefsOverridden; @@ -1789,14 +1646,26 @@ Error JITDylib::defineImpl(MaterializationUnit &MU) { } // If there were any duplicate definitions then bail out. - if (!Duplicates.empty()) - return make_error<DuplicateDefinition>(**Duplicates.begin()); + if (!Duplicates.empty()) { + LLVM_DEBUG( + { dbgs() << " Error: Duplicate symbols " << Duplicates << "\n"; }); + return make_error<DuplicateDefinition>(std::string(**Duplicates.begin())); + } // Discard any overridden defs in this MU. + LLVM_DEBUG({ + if (!MUDefsOverridden.empty()) + dbgs() << " Defs in this MU overridden: " << MUDefsOverridden << "\n"; + }); for (auto &S : MUDefsOverridden) MU.doDiscard(*this, S); // Discard existing overridden defs. + LLVM_DEBUG({ + if (!ExistingDefsOverridden.empty()) + dbgs() << " Existing defs overridden by this MU: " << MUDefsOverridden + << "\n"; + }); for (auto &S : ExistingDefsOverridden) { auto UMII = UnmaterializedInfos.find(S); @@ -1852,6 +1721,57 @@ void JITDylib::transferEmittedNodeDependencies( } } +Platform::~Platform() {} + +Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols( + ExecutionSession &ES, + const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) { + + DenseMap<JITDylib *, SymbolMap> CompoundResult; + Error CompoundErr = Error::success(); + std::mutex LookupMutex; + std::condition_variable CV; + uint64_t Count = InitSyms.size(); + + LLVM_DEBUG({ + dbgs() << "Issuing init-symbol lookup:\n"; + for (auto &KV : InitSyms) + dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n"; + }); + + for (auto &KV : InitSyms) { + auto *JD = KV.first; + auto Names = std::move(KV.second); + ES.lookup( + LookupKind::Static, + JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}), + std::move(Names), SymbolState::Ready, + [&, JD](Expected<SymbolMap> Result) { + { + std::lock_guard<std::mutex> Lock(LookupMutex); + --Count; + if (Result) { + assert(!CompoundResult.count(JD) && + "Duplicate JITDylib in lookup?"); + CompoundResult[JD] = std::move(*Result); + } else + CompoundErr = + joinErrors(std::move(CompoundErr), Result.takeError()); + } + CV.notify_one(); + }, + NoDependenciesToRegister); + } + + std::unique_lock<std::mutex> Lock(LookupMutex); + CV.wait(Lock, [&] { return Count == 0 || CompoundErr; }); + + if (CompoundErr) + return std::move(CompoundErr); + + return std::move(CompoundResult); +} + ExecutionSession::ExecutionSession(std::shared_ptr<SymbolStringPool> SSP) : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) { } @@ -1865,15 +1785,23 @@ JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { }); } -JITDylib &ExecutionSession::createJITDylib(std::string Name) { +JITDylib &ExecutionSession::createBareJITDylib(std::string Name) { assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { JDs.push_back( - std::unique_ptr<JITDylib>(new JITDylib(*this, std::move(Name)))); + std::shared_ptr<JITDylib>(new JITDylib(*this, std::move(Name)))); return *JDs.back(); }); } +Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) { + auto &JD = createBareJITDylib(Name); + if (P) + if (auto Err = P->setupJITDylib(JD)) + return std::move(Err); + return JD; +} + void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { assert(!!Err && "Error should be in failure state"); @@ -2050,9 +1978,13 @@ void ExecutionSession::lookup( { std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); - for (auto &KV : CollectedMUsMap) - for (auto &MU : KV.second) - OutstandingMUs.push_back(std::make_pair(KV.first, std::move(MU))); + for (auto &KV : CollectedMUsMap) { + auto JD = KV.first->shared_from_this(); + for (auto &MU : KV.second) { + auto MR = MU->createMaterializationResponsibility(JD); + OutstandingMUs.push_back(std::make_pair(std::move(MU), std::move(MR))); + } + } } runOutstandingMUs(); @@ -2114,11 +2046,11 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, Expected<JITEvaluatedSymbol> ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, - SymbolStringPtr Name) { + SymbolStringPtr Name, SymbolState RequiredState) { SymbolLookupSet Names({Name}); if (auto ResultMap = lookup(SearchOrder, std::move(Names), LookupKind::Static, - SymbolState::Ready, NoDependenciesToRegister)) { + RequiredState, NoDependenciesToRegister)) { assert(ResultMap->size() == 1 && "Unexpected number of results"); assert(ResultMap->count(Name) && "Missing result for symbol"); return std::move(ResultMap->begin()->second); @@ -2127,14 +2059,15 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, } Expected<JITEvaluatedSymbol> -ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, - SymbolStringPtr Name) { - return lookup(makeJITDylibSearchOrder(SearchOrder), Name); +ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name, + SymbolState RequiredState) { + return lookup(makeJITDylibSearchOrder(SearchOrder), Name, RequiredState); } Expected<JITEvaluatedSymbol> -ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name) { - return lookup(SearchOrder, intern(Name)); +ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name, + SymbolState RequiredState) { + return lookup(SearchOrder, intern(Name), RequiredState); } void ExecutionSession::dump(raw_ostream &OS) { @@ -2146,36 +2079,33 @@ void ExecutionSession::dump(raw_ostream &OS) { void ExecutionSession::runOutstandingMUs() { while (1) { - std::pair<JITDylib *, std::unique_ptr<MaterializationUnit>> JITDylibAndMU; + Optional<std::pair<std::unique_ptr<MaterializationUnit>, + MaterializationResponsibility>> + JMU; { std::lock_guard<std::recursive_mutex> Lock(OutstandingMUsMutex); if (!OutstandingMUs.empty()) { - JITDylibAndMU = std::move(OutstandingMUs.back()); + JMU.emplace(std::move(OutstandingMUs.back())); OutstandingMUs.pop_back(); } } - if (JITDylibAndMU.first) { - assert(JITDylibAndMU.second && "JITDylib, but no MU?"); - dispatchMaterialization(*JITDylibAndMU.first, - std::move(JITDylibAndMU.second)); - } else + if (!JMU) break; + + assert(JMU->first && "No MU?"); + dispatchMaterialization(std::move(JMU->first), std::move(JMU->second)); } } -MangleAndInterner::MangleAndInterner(ExecutionSession &ES, const DataLayout &DL) - : ES(ES), DL(DL) {} - -SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { - std::string MangledName; - { - raw_string_ostream MangledNameStream(MangledName); - Mangler::getNameWithPrefix(MangledNameStream, Name, DL); - } - return ES.intern(MangledName); +#ifndef NDEBUG +void ExecutionSession::dumpDispatchInfo(JITDylib &JD, MaterializationUnit &MU) { + runSessionLocked([&]() { + dbgs() << "Dispatching " << MU << " for " << JD.getName() << "\n"; + }); } +#endif // NDEBUG } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp index c9e87ff737fcb..6247158919fa1 100644 --- a/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp @@ -7,16 +7,297 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/DebugUtils.h" + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/Format.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" #define DEBUG_TYPE "orc" +using namespace llvm; + +namespace { + +#ifndef NDEBUG + +cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(true), + cl::desc("debug print hidden symbols defined by " + "materialization units"), + cl::Hidden); + +cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(true), + cl::desc("debug print callable symbols defined by " + "materialization units"), + cl::Hidden); + +cl::opt<bool> PrintData("debug-orc-print-data", cl::init(true), + cl::desc("debug print data symbols defined by " + "materialization units"), + cl::Hidden); + +#endif // NDEBUG + +// SetPrinter predicate that prints every element. +template <typename T> struct PrintAll { + bool operator()(const T &E) { return true; } +}; + +bool anyPrintSymbolOptionSet() { +#ifndef NDEBUG + return PrintHidden || PrintCallable || PrintData; +#else + return false; +#endif // NDEBUG +} + +bool flagsMatchCLOpts(const JITSymbolFlags &Flags) { +#ifndef NDEBUG + // Bail out early if this is a hidden symbol and we're not printing hiddens. + if (!PrintHidden && !Flags.isExported()) + return false; + + // Return true if this is callable and we're printing callables. + if (PrintCallable && Flags.isCallable()) + return true; + + // Return true if this is data and we're printing data. + if (PrintData && !Flags.isCallable()) + return true; + + // otherwise return false. + return false; +#else + return false; +#endif // NDEBUG +} + +// Prints a sequence of items, filtered by an user-supplied predicate. +template <typename Sequence, + typename Pred = PrintAll<typename Sequence::value_type>> +class SequencePrinter { +public: + SequencePrinter(const Sequence &S, char OpenSeq, char CloseSeq, + Pred ShouldPrint = Pred()) + : S(S), OpenSeq(OpenSeq), CloseSeq(CloseSeq), + ShouldPrint(std::move(ShouldPrint)) {} + + void printTo(llvm::raw_ostream &OS) const { + bool PrintComma = false; + OS << OpenSeq; + for (auto &E : S) { + if (ShouldPrint(E)) { + if (PrintComma) + OS << ','; + OS << ' ' << E; + PrintComma = true; + } + } + OS << ' ' << CloseSeq; + } + +private: + const Sequence &S; + char OpenSeq; + char CloseSeq; + mutable Pred ShouldPrint; +}; + +template <typename Sequence, typename Pred> +SequencePrinter<Sequence, Pred> printSequence(const Sequence &S, char OpenSeq, + char CloseSeq, Pred P = Pred()) { + return SequencePrinter<Sequence, Pred>(S, OpenSeq, CloseSeq, std::move(P)); +} + +// Render a SequencePrinter by delegating to its printTo method. +template <typename Sequence, typename Pred> +llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, + const SequencePrinter<Sequence, Pred> &Printer) { + Printer.printTo(OS); + return OS; +} + +struct PrintSymbolFlagsMapElemsMatchingCLOpts { + bool operator()(const orc::SymbolFlagsMap::value_type &KV) { + return flagsMatchCLOpts(KV.second); + } +}; + +struct PrintSymbolMapElemsMatchingCLOpts { + bool operator()(const orc::SymbolMap::value_type &KV) { + return flagsMatchCLOpts(KV.second.getFlags()); + } +}; + +} // end anonymous namespace + namespace llvm { namespace orc { +raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym) { + return OS << *Sym; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) { + return OS << printSequence(Symbols, '{', '}', PrintAll<SymbolStringPtr>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolNameVector &Symbols) { + return OS << printSequence(Symbols, '[', ']', PrintAll<SymbolStringPtr>()); +} + +raw_ostream &operator<<(raw_ostream &OS, ArrayRef<SymbolStringPtr> Symbols) { + return OS << printSequence(Symbols, '[', ']', PrintAll<SymbolStringPtr>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { + if (Flags.hasError()) + OS << "[*ERROR*]"; + if (Flags.isCallable()) + OS << "[Callable]"; + else + OS << "[Data]"; + if (Flags.isWeak()) + OS << "[Weak]"; + else if (Flags.isCommon()) + OS << "[Common]"; + + if (!Flags.isExported()) + OS << "[Hidden]"; + + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) { + return OS << format("0x%016" PRIx64, Sym.getAddress()) << " " + << Sym.getFlags(); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV) { + return OS << "(\"" << KV.first << "\", " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap::value_type &KV) { + return OS << "(\"" << KV.first << "\": " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap &SymbolFlags) { + return OS << printSequence(SymbolFlags, '{', '}', + PrintSymbolFlagsMapElemsMatchingCLOpts()); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolMap &Symbols) { + return OS << printSequence(Symbols, '{', '}', + PrintSymbolMapElemsMatchingCLOpts()); +} + +raw_ostream &operator<<(raw_ostream &OS, + const SymbolDependenceMap::value_type &KV) { + return OS << "(" << KV.first->getName() << ", " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolDependenceMap &Deps) { + return OS << printSequence(Deps, '{', '}', + PrintAll<SymbolDependenceMap::value_type>()); +} + +raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU) { + OS << "MU@" << &MU << " (\"" << MU.getName() << "\""; + if (anyPrintSymbolOptionSet()) + OS << ", " << MU.getSymbols(); + return OS << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const LookupKind &K) { + switch (K) { + case LookupKind::Static: + return OS << "Static"; + case LookupKind::DLSym: + return OS << "DLSym"; + } + llvm_unreachable("Invalid lookup kind"); +} + +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibLookupFlags &JDLookupFlags) { + switch (JDLookupFlags) { + case JITDylibLookupFlags::MatchExportedSymbolsOnly: + return OS << "MatchExportedSymbolsOnly"; + case JITDylibLookupFlags::MatchAllSymbols: + return OS << "MatchAllSymbols"; + } + llvm_unreachable("Invalid JITDylib lookup flags"); +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LookupFlags) { + switch (LookupFlags) { + case SymbolLookupFlags::RequiredSymbol: + return OS << "RequiredSymbol"; + case SymbolLookupFlags::WeaklyReferencedSymbol: + return OS << "WeaklyReferencedSymbol"; + } + llvm_unreachable("Invalid symbol lookup flags"); +} + +raw_ostream &operator<<(raw_ostream &OS, + const SymbolLookupSet::value_type &KV) { + return OS << "(" << KV.first << ", " << KV.second << ")"; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupSet &LookupSet) { + return OS << printSequence(LookupSet, '{', '}', + PrintAll<SymbolLookupSet::value_type>()); +} + +raw_ostream &operator<<(raw_ostream &OS, + const JITDylibSearchOrder &SearchOrder) { + OS << "["; + if (!SearchOrder.empty()) { + assert(SearchOrder.front().first && + "JITDylibList entries must not be null"); + OS << " (\"" << SearchOrder.front().first->getName() << "\", " + << SearchOrder.begin()->second << ")"; + for (auto &KV : + make_range(std::next(SearchOrder.begin(), 1), SearchOrder.end())) { + assert(KV.first && "JITDylibList entries must not be null"); + OS << ", (\"" << KV.first->getName() << "\", " << KV.second << ")"; + } + } + OS << " ]"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) { + OS << "{"; + for (auto &KV : Aliases) + OS << " " << *KV.first << ": " << KV.second.Aliasee << " " + << KV.second.AliasFlags; + OS << " }"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { + switch (S) { + case SymbolState::Invalid: + return OS << "Invalid"; + case SymbolState::NeverSearched: + return OS << "Never-Searched"; + case SymbolState::Materializing: + return OS << "Materializing"; + case SymbolState::Resolved: + return OS << "Resolved"; + case SymbolState::Emitted: + return OS << "Emitted"; + case SymbolState::Ready: + return OS << "Ready"; + } + llvm_unreachable("Invalid state"); +} + DumpObjects::DumpObjects(std::string DumpDir, std::string IdentifierOverride) : DumpDir(std::move(DumpDir)), IdentifierOverride(std::move(IdentifierOverride)) { diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 3d97fe9eeab17..4d255cd66c1be 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -13,6 +13,8 @@ #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Module.h" +#include "llvm/Object/MachOUniversal.h" +#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Target/TargetMachine.h" @@ -113,6 +115,26 @@ iterator_range<CtorDtorIterator> getDestructors(const Module &M) { CtorDtorIterator(DtorsList, true)); } +bool StaticInitGVIterator::isStaticInitGlobal(GlobalValue &GV) { + if (GV.isDeclaration()) + return false; + + if (GV.hasName() && (GV.getName() == "llvm.global_ctors" || + GV.getName() == "llvm.global_dtors")) + return true; + + if (ObjFmt == Triple::MachO) { + // FIXME: These section checks are too strict: We should match first and + // second word split by comma. + if (GV.hasSection() && + (GV.getSection().startswith("__DATA,__objc_classlist") || + GV.getSection().startswith("__DATA,__objc_selrefs"))) + return true; + } + + return false; +} + void CtorDtorRunner::add(iterator_range<CtorDtorIterator> CtorDtors) { if (CtorDtors.empty()) return; @@ -198,6 +220,30 @@ Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, return JD.define(absoluteSymbols(std::move(RuntimeInterposes))); } +void ItaniumCXAAtExitSupport::registerAtExit(void (*F)(void *), void *Ctx, + void *DSOHandle) { + std::lock_guard<std::mutex> Lock(AtExitsMutex); + AtExitRecords[DSOHandle].push_back({F, Ctx}); +} + +void ItaniumCXAAtExitSupport::runAtExits(void *DSOHandle) { + std::vector<AtExitRecord> AtExitsToRun; + + { + std::lock_guard<std::mutex> Lock(AtExitsMutex); + auto I = AtExitRecords.find(DSOHandle); + if (I != AtExitRecords.end()) { + AtExitsToRun = std::move(I->second); + AtExitRecords.erase(I); + } + } + + while (!AtExitsToRun.empty()) { + AtExitsToRun.back().F(AtExitsToRun.back().Ctx); + AtExitsToRun.pop_back(); + } +} + DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), @@ -259,6 +305,51 @@ StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName) { } Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> +StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName, + const Triple &TT) { + auto B = object::createBinary(FileName); + if (!B) + return B.takeError(); + + // If this is a regular archive then create an instance from it. + if (isa<object::Archive>(B->getBinary())) + return Create(L, std::move(B->takeBinary().second)); + + // If this is a universal binary then search for a slice matching the given + // Triple. + if (auto *UB = cast<object::MachOUniversalBinary>(B->getBinary())) { + for (const auto &Obj : UB->objects()) { + auto ObjTT = Obj.getTriple(); + if (ObjTT.getArch() == TT.getArch() && + ObjTT.getSubArch() == TT.getSubArch() && + ObjTT.getVendor() == TT.getVendor()) { + // We found a match. Create an instance from a buffer covering this + // slice. + auto SliceBuffer = MemoryBuffer::getFileSlice(FileName, Obj.getSize(), + Obj.getOffset()); + if (!SliceBuffer) + return make_error<StringError>( + Twine("Could not create buffer for ") + TT.str() + " slice of " + + FileName + ": [ " + formatv("{0:x}", Obj.getOffset()) + + " .. " + formatv("{0:x}", Obj.getOffset() + Obj.getSize()) + + ": " + SliceBuffer.getError().message(), + SliceBuffer.getError()); + return Create(L, std::move(*SliceBuffer)); + } + } + + return make_error<StringError>(Twine("Universal binary ") + FileName + + " does not contain a slice for " + + TT.str(), + inconvertibleErrorCode()); + } + + return make_error<StringError>(Twine("Unrecognized file type for ") + + FileName, + inconvertibleErrorCode()); +} + +Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> StaticLibraryDefinitionGenerator::Create( ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer) { Error Err = Error::success(); @@ -305,8 +396,8 @@ Error StaticLibraryDefinitionGenerator::tryToGenerate( MemoryBufferRef ChildBufferRef(ChildBufferInfo.first, ChildBufferInfo.second); - if (auto Err = - L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef), VModuleKey())) + if (auto Err = L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef, false), + VModuleKey())) return Err; } diff --git a/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp index d311f34179c7c..023940dc82982 100644 --- a/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -11,9 +11,14 @@ namespace llvm { namespace orc { +IRCompileLayer::IRCompiler::~IRCompiler() {} + IRCompileLayer::IRCompileLayer(ExecutionSession &ES, ObjectLayer &BaseLayer, - CompileFunction Compile) - : IRLayer(ES), BaseLayer(BaseLayer), Compile(std::move(Compile)) {} + std::unique_ptr<IRCompiler> Compile) + : IRLayer(ES, ManglingOpts), BaseLayer(BaseLayer), + Compile(std::move(Compile)) { + ManglingOpts = &this->Compile->getManglingOptions(); +} void IRCompileLayer::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) { std::lock_guard<std::mutex> Lock(IRLayerMutex); @@ -24,7 +29,7 @@ void IRCompileLayer::emit(MaterializationResponsibility R, ThreadSafeModule TSM) { assert(TSM && "Module must not be null"); - if (auto Obj = TSM.withModuleDo(Compile)) { + if (auto Obj = TSM.withModuleDo(*Compile)) { { std::lock_guard<std::mutex> Lock(IRLayerMutex); if (NotifyCompiled) diff --git a/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp index 845ecc71eb870..511248f83b259 100644 --- a/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -12,10 +12,10 @@ namespace llvm { namespace orc { -IRTransformLayer::IRTransformLayer(ExecutionSession &ES, - IRLayer &BaseLayer, - TransformFunction Transform) - : IRLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} +IRTransformLayer::IRTransformLayer(ExecutionSession &ES, IRLayer &BaseLayer, + TransformFunction Transform) + : IRLayer(ES, BaseLayer.getManglingOptions()), BaseLayer(BaseLayer), + Transform(std::move(Transform)) {} void IRTransformLayer::emit(MaterializationResponsibility R, ThreadSafeModule TSM) { diff --git a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 1ac9a58aeaef2..031b1afefc9da 100644 --- a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -10,7 +10,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/Format.h" #include "llvm/Transforms/Utils/Cloning.h" @@ -28,7 +27,7 @@ public: CompileCallbackMaterializationUnit(SymbolStringPtr Name, CompileFunction Compile, VModuleKey K) : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}}), - std::move(K)), + nullptr, std::move(K)), Name(std::move(Name)), Compile(std::move(Compile)) {} StringRef getName() const override { return "<Compile Callbacks>"; } diff --git a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index 114e81e41771b..8cf66c9e759a3 100644 --- a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -10,6 +10,7 @@ #include "llvm/Support/Host.h" #include "llvm/Support/TargetRegistry.h" +#include "llvm/Support/raw_ostream.h" namespace llvm { namespace orc { @@ -33,7 +34,7 @@ Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { for (auto &Feature : FeatureMap) TMBuilder.getFeatures().AddFeature(Feature.first(), Feature.second); - TMBuilder.setCPU(llvm::sys::getHostCPUName()); + TMBuilder.setCPU(std::string(llvm::sys::getHostCPUName())); return TMBuilder; } @@ -63,5 +64,78 @@ JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures( return *this; } +#ifndef NDEBUG +raw_ostream &operator<<(raw_ostream &OS, const JITTargetMachineBuilder &JTMB) { + OS << "{ Triple = \"" << JTMB.TT.str() << "\", CPU = \"" << JTMB.CPU + << "\", Options = <not-printable>, Relocation Model = "; + + if (JTMB.RM) { + switch (*JTMB.RM) { + case Reloc::Static: + OS << "Static"; + break; + case Reloc::PIC_: + OS << "PIC_"; + break; + case Reloc::DynamicNoPIC: + OS << "DynamicNoPIC"; + break; + case Reloc::ROPI: + OS << "ROPI"; + break; + case Reloc::RWPI: + OS << "RWPI"; + break; + case Reloc::ROPI_RWPI: + OS << "ROPI_RWPI"; + break; + } + } else + OS << "unspecified"; + + OS << ", Code Model = "; + + if (JTMB.CM) { + switch (*JTMB.CM) { + case CodeModel::Tiny: + OS << "Tiny"; + break; + case CodeModel::Small: + OS << "Small"; + break; + case CodeModel::Kernel: + OS << "Kernel"; + break; + case CodeModel::Medium: + OS << "Medium"; + break; + case CodeModel::Large: + OS << "Large"; + break; + } + } else + OS << "unspecified"; + + OS << ", Optimization Level = "; + switch (JTMB.OptLevel) { + case CodeGenOpt::None: + OS << "None"; + break; + case CodeGenOpt::Less: + OS << "Less"; + break; + case CodeGenOpt::Default: + OS << "Default"; + break; + case CodeGenOpt::Aggressive: + OS << "Aggressive"; + break; + } + + OS << " }"; + return OS; +} +#endif // NDEBUG + } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index 54473ab46423c..713a48fbf3eb8 100644 --- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -7,30 +7,965 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" +#include "llvm/IR/GlobalVariable.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Mangler.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/DynamicLibrary.h" + +#include <map> + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; + +namespace { + +/// Adds helper function decls and wrapper functions that call the helper with +/// some additional prefix arguments. +/// +/// E.g. For wrapper "foo" with type i8(i8, i64), helper "bar", and prefix +/// args i32 4 and i16 12345, this function will add: +/// +/// declare i8 @bar(i32, i16, i8, i64) +/// +/// define i8 @foo(i8, i64) { +/// entry: +/// %2 = call i8 @bar(i32 4, i16 12345, i8 %0, i64 %1) +/// ret i8 %2 +/// } +/// +Function *addHelperAndWrapper(Module &M, StringRef WrapperName, + FunctionType *WrapperFnType, + GlobalValue::VisibilityTypes WrapperVisibility, + StringRef HelperName, + ArrayRef<Value *> HelperPrefixArgs) { + std::vector<Type *> HelperArgTypes; + for (auto *Arg : HelperPrefixArgs) + HelperArgTypes.push_back(Arg->getType()); + for (auto *T : WrapperFnType->params()) + HelperArgTypes.push_back(T); + auto *HelperFnType = + FunctionType::get(WrapperFnType->getReturnType(), HelperArgTypes, false); + auto *HelperFn = Function::Create(HelperFnType, GlobalValue::ExternalLinkage, + HelperName, M); + + auto *WrapperFn = Function::Create( + WrapperFnType, GlobalValue::ExternalLinkage, WrapperName, M); + WrapperFn->setVisibility(WrapperVisibility); + + auto *EntryBlock = BasicBlock::Create(M.getContext(), "entry", WrapperFn); + IRBuilder<> IB(EntryBlock); + + std::vector<Value *> HelperArgs; + for (auto *Arg : HelperPrefixArgs) + HelperArgs.push_back(Arg); + for (auto &Arg : WrapperFn->args()) + HelperArgs.push_back(&Arg); + auto *HelperResult = IB.CreateCall(HelperFn, HelperArgs); + if (HelperFn->getReturnType()->isVoidTy()) + IB.CreateRetVoid(); + else + IB.CreateRet(HelperResult); + + return WrapperFn; +} + +class GenericLLVMIRPlatformSupport; + +/// orc::Platform component of Generic LLVM IR Platform support. +/// Just forwards calls to the GenericLLVMIRPlatformSupport class below. +class GenericLLVMIRPlatform : public Platform { +public: + GenericLLVMIRPlatform(GenericLLVMIRPlatformSupport &S) : S(S) {} + Error setupJITDylib(JITDylib &JD) override; + Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) override; + Error notifyRemoving(JITDylib &JD, VModuleKey K) override { + // Noop -- Nothing to do (yet). + return Error::success(); + } + +private: + GenericLLVMIRPlatformSupport &S; +}; + +/// This transform parses llvm.global_ctors to produce a single initialization +/// function for the module, records the function, then deletes +/// llvm.global_ctors. +class GlobalCtorDtorScraper { +public: + + GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS, + StringRef InitFunctionPrefix) + : PS(PS), InitFunctionPrefix(InitFunctionPrefix) {} + Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM, + MaterializationResponsibility &R); + +private: + GenericLLVMIRPlatformSupport &PS; + StringRef InitFunctionPrefix; +}; + +/// Generic IR Platform Support +/// +/// Scrapes llvm.global_ctors and llvm.global_dtors and replaces them with +/// specially named 'init' and 'deinit'. Injects definitions / interposes for +/// some runtime API, including __cxa_atexit, dlopen, and dlclose. +class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { +public: + // GenericLLVMIRPlatform &P) : P(P) { + GenericLLVMIRPlatformSupport(LLJIT &J) + : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")) { + + getExecutionSession().setPlatform( + std::make_unique<GenericLLVMIRPlatform>(*this)); + + setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix)); + + SymbolMap StdInterposes; + + StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(this), + JITSymbolFlags::Exported); + StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper), + JITSymbolFlags()); + + cantFail( + J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes)))); + cantFail(setupJITDylib(J.getMainJITDylib())); + cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule())); + } + + ExecutionSession &getExecutionSession() { return J.getExecutionSession(); } + + /// Adds a module that defines the __dso_handle global. + Error setupJITDylib(JITDylib &JD) { + + // Add per-jitdylib standard interposes. + SymbolMap PerJDInterposes; + PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper), + JITSymbolFlags()); + cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes)))); + + auto Ctx = std::make_unique<LLVMContext>(); + auto M = std::make_unique<Module>("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *Int64Ty = Type::getInt64Ty(*Ctx); + auto *DSOHandle = new GlobalVariable( + *M, Int64Ty, true, GlobalValue::ExternalLinkage, + ConstantInt::get(Int64Ty, reinterpret_cast<uintptr_t>(&JD)), + "__dso_handle"); + DSOHandle->setVisibility(GlobalValue::DefaultVisibility); + DSOHandle->setInitializer( + ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD))); + + auto *GenericIRPlatformSupportTy = + StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage, + nullptr, "__lljit.platform_support_instance"); + + auto *VoidTy = Type::getVoidTy(*Ctx); + addHelperAndWrapper( + *M, "__lljit_run_atexits", FunctionType::get(VoidTy, {}, false), + GlobalValue::HiddenVisibility, "__lljit.run_atexits_helper", + {PlatformInstanceDecl, DSOHandle}); + + return J.addIRModule(JD, ThreadSafeModule(std::move(M), std::move(Ctx))); + } + + Error notifyAdding(JITDylib &JD, const MaterializationUnit &MU) { + if (auto &InitSym = MU.getInitializerSymbol()) + InitSymbols[&JD].add(InitSym, SymbolLookupFlags::WeaklyReferencedSymbol); + else { + // If there's no identified init symbol attached, but there is a symbol + // with the GenericIRPlatform::InitFunctionPrefix, then treat that as + // an init function. Add the symbol to both the InitSymbols map (which + // will trigger a lookup to materialize the module) and the InitFunctions + // map (which holds the names of the symbols to execute). + for (auto &KV : MU.getSymbols()) + if ((*KV.first).startswith(InitFunctionPrefix)) { + InitSymbols[&JD].add(KV.first, + SymbolLookupFlags::WeaklyReferencedSymbol); + InitFunctions[&JD].add(KV.first); + } + } + return Error::success(); + } + + Error initialize(JITDylib &JD) override { + LLVM_DEBUG({ + dbgs() << "GenericLLVMIRPlatformSupport getting initializers to run\n"; + }); + if (auto Initializers = getInitializers(JD)) { + LLVM_DEBUG( + { dbgs() << "GenericLLVMIRPlatformSupport running initializers\n"; }); + for (auto InitFnAddr : *Initializers) { + LLVM_DEBUG({ + dbgs() << " Running init " << formatv("{0:x16}", InitFnAddr) + << "...\n"; + }); + auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr); + InitFn(); + } + } else + return Initializers.takeError(); + return Error::success(); + } + + Error deinitialize(JITDylib &JD) override { + LLVM_DEBUG({ + dbgs() << "GenericLLVMIRPlatformSupport getting deinitializers to run\n"; + }); + if (auto Deinitializers = getDeinitializers(JD)) { + LLVM_DEBUG({ + dbgs() << "GenericLLVMIRPlatformSupport running deinitializers\n"; + }); + for (auto DeinitFnAddr : *Deinitializers) { + LLVM_DEBUG({ + dbgs() << " Running init " << formatv("{0:x16}", DeinitFnAddr) + << "...\n"; + }); + auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr); + DeinitFn(); + } + } else + return Deinitializers.takeError(); + + return Error::success(); + } + + void registerInitFunc(JITDylib &JD, SymbolStringPtr InitName) { + getExecutionSession().runSessionLocked([&]() { + InitFunctions[&JD].add(InitName); + }); + } + +private: + + Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) { + if (auto Err = issueInitLookups(JD)) + return std::move(Err); + + DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols; + std::vector<JITDylib *> DFSLinkOrder; + + getExecutionSession().runSessionLocked([&]() { + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *NextJD : DFSLinkOrder) { + auto IFItr = InitFunctions.find(NextJD); + if (IFItr != InitFunctions.end()) { + LookupSymbols[NextJD] = std::move(IFItr->second); + InitFunctions.erase(IFItr); + } + } + }); + + LLVM_DEBUG({ + dbgs() << "JITDylib init order is [ "; + for (auto *JD : llvm::reverse(DFSLinkOrder)) + dbgs() << "\"" << JD->getName() << "\" "; + dbgs() << "]\n"; + dbgs() << "Looking up init functions:\n"; + for (auto &KV : LookupSymbols) + dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; + }); + + auto &ES = getExecutionSession(); + auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); + + if (!LookupResult) + return LookupResult.takeError(); + + std::vector<JITTargetAddress> Initializers; + while (!DFSLinkOrder.empty()) { + auto &NextJD = *DFSLinkOrder.back(); + DFSLinkOrder.pop_back(); + auto InitsItr = LookupResult->find(&NextJD); + if (InitsItr == LookupResult->end()) + continue; + for (auto &KV : InitsItr->second) + Initializers.push_back(KV.second.getAddress()); + } + + return Initializers; + } + + Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) { + auto &ES = getExecutionSession(); + + auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits"); + + DenseMap<JITDylib *, SymbolLookupSet> LookupSymbols; + std::vector<JITDylib *> DFSLinkOrder; + + ES.runSessionLocked([&]() { + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *NextJD : DFSLinkOrder) { + auto &JDLookupSymbols = LookupSymbols[NextJD]; + auto DIFItr = DeInitFunctions.find(NextJD); + if (DIFItr != DeInitFunctions.end()) { + LookupSymbols[NextJD] = std::move(DIFItr->second); + DeInitFunctions.erase(DIFItr); + } + JDLookupSymbols.add(LLJITRunAtExits, + SymbolLookupFlags::WeaklyReferencedSymbol); + } + }); + + LLVM_DEBUG({ + dbgs() << "JITDylib deinit order is [ "; + for (auto *JD : DFSLinkOrder) + dbgs() << "\"" << JD->getName() << "\" "; + dbgs() << "]\n"; + dbgs() << "Looking up deinit functions:\n"; + for (auto &KV : LookupSymbols) + dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; + }); + + auto LookupResult = Platform::lookupInitSymbols(ES, LookupSymbols); + + if (!LookupResult) + return LookupResult.takeError(); + + std::vector<JITTargetAddress> DeInitializers; + for (auto *NextJD : DFSLinkOrder) { + auto DeInitsItr = LookupResult->find(NextJD); + assert(DeInitsItr != LookupResult->end() && + "Every JD should have at least __lljit_run_atexits"); + + auto RunAtExitsItr = DeInitsItr->second.find(LLJITRunAtExits); + if (RunAtExitsItr != DeInitsItr->second.end()) + DeInitializers.push_back(RunAtExitsItr->second.getAddress()); + + for (auto &KV : DeInitsItr->second) + if (KV.first != LLJITRunAtExits) + DeInitializers.push_back(KV.second.getAddress()); + } + + return DeInitializers; + } + + // Returns a DFS traversal order of the JITDylibs reachable (via + // links-against edges) from JD, starting with JD itself. + static std::vector<JITDylib *> getDFSLinkOrder(JITDylib &JD) { + std::vector<JITDylib *> DFSLinkOrder; + std::vector<JITDylib *> WorkStack({&JD}); + DenseSet<JITDylib *> Visited; + + while (!WorkStack.empty()) { + auto &NextJD = *WorkStack.back(); + WorkStack.pop_back(); + if (Visited.count(&NextJD)) + continue; + Visited.insert(&NextJD); + DFSLinkOrder.push_back(&NextJD); + NextJD.withLinkOrderDo([&](const JITDylibSearchOrder &LinkOrder) { + for (auto &KV : LinkOrder) + WorkStack.push_back(KV.first); + }); + } + + return DFSLinkOrder; + } + + /// Issue lookups for all init symbols required to initialize JD (and any + /// JITDylibs that it depends on). + Error issueInitLookups(JITDylib &JD) { + DenseMap<JITDylib *, SymbolLookupSet> RequiredInitSymbols; + std::vector<JITDylib *> DFSLinkOrder; + + getExecutionSession().runSessionLocked([&]() { + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *NextJD : DFSLinkOrder) { + auto ISItr = InitSymbols.find(NextJD); + if (ISItr != InitSymbols.end()) { + RequiredInitSymbols[NextJD] = std::move(ISItr->second); + InitSymbols.erase(ISItr); + } + } + }); + + return Platform::lookupInitSymbols(getExecutionSession(), + RequiredInitSymbols) + .takeError(); + } + + static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, + void *DSOHandle) { + LLVM_DEBUG({ + dbgs() << "Registering atexit function " << (void *)F << " for JD " + << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n"; + }); + static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.registerAtExit( + F, Ctx, DSOHandle); + } + + static void runAtExitsHelper(void *Self, void *DSOHandle) { + LLVM_DEBUG({ + dbgs() << "Running atexit functions for JD " + << (*static_cast<JITDylib **>(DSOHandle))->getName() << "\n"; + }); + static_cast<GenericLLVMIRPlatformSupport *>(Self)->AtExitMgr.runAtExits( + DSOHandle); + } + + // Constructs an LLVM IR module containing platform runtime globals, + // functions, and interposes. + ThreadSafeModule createPlatformRuntimeModule() { + auto Ctx = std::make_unique<LLVMContext>(); + auto M = std::make_unique<Module>("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *GenericIRPlatformSupportTy = + StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, GenericIRPlatformSupportTy, true, GlobalValue::ExternalLinkage, + nullptr, "__lljit.platform_support_instance"); + + auto *Int8Ty = Type::getInt8Ty(*Ctx); + auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); + auto *VoidTy = Type::getVoidTy(*Ctx); + auto *BytePtrTy = PointerType::getUnqual(Int8Ty); + auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); + auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); + + addHelperAndWrapper( + *M, "__cxa_atexit", + FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, + false), + GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper", + {PlatformInstanceDecl}); + + return ThreadSafeModule(std::move(M), std::move(Ctx)); + } + + LLJIT &J; + std::string InitFunctionPrefix; + DenseMap<JITDylib *, SymbolLookupSet> InitSymbols; + DenseMap<JITDylib *, SymbolLookupSet> InitFunctions; + DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions; + ItaniumCXAAtExitSupport AtExitMgr; +}; + +Error GenericLLVMIRPlatform::setupJITDylib(JITDylib &JD) { + return S.setupJITDylib(JD); +} + +Error GenericLLVMIRPlatform::notifyAdding(JITDylib &JD, + const MaterializationUnit &MU) { + return S.notifyAdding(JD, MU); +} + +Expected<ThreadSafeModule> +GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM, + MaterializationResponsibility &R) { + auto Err = TSM.withModuleDo([&](Module &M) -> Error { + auto &Ctx = M.getContext(); + auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors"); + + // If there's no llvm.global_ctors or it's just a decl then skip. + if (!GlobalCtors || GlobalCtors->isDeclaration()) + return Error::success(); + + std::string InitFunctionName; + raw_string_ostream(InitFunctionName) + << InitFunctionPrefix << M.getModuleIdentifier(); + + MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout()); + auto InternedName = Mangle(InitFunctionName); + if (auto Err = + R.defineMaterializing({{InternedName, JITSymbolFlags::Callable}})) + return Err; + + auto *InitFunc = + Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false), + GlobalValue::ExternalLinkage, InitFunctionName, &M); + InitFunc->setVisibility(GlobalValue::HiddenVisibility); + std::vector<std::pair<Function *, unsigned>> Inits; + for (auto E : getConstructors(M)) + Inits.push_back(std::make_pair(E.Func, E.Priority)); + llvm::sort(Inits, [](const std::pair<Function *, unsigned> &LHS, + const std::pair<Function *, unsigned> &RHS) { + return LHS.first < RHS.first; + }); + auto *EntryBlock = BasicBlock::Create(Ctx, "entry", InitFunc); + IRBuilder<> IB(EntryBlock); + for (auto &KV : Inits) + IB.CreateCall(KV.first); + IB.CreateRetVoid(); + + PS.registerInitFunc(R.getTargetJITDylib(), InternedName); + GlobalCtors->eraseFromParent(); + return Error::success(); + }); + + if (Err) + return std::move(Err); + + return std::move(TSM); +} + +class MachOPlatformSupport : public LLJIT::PlatformSupport { +public: + using DLOpenType = void *(*)(const char *Name, int Mode); + using DLCloseType = int (*)(void *Handle); + using DLSymType = void *(*)(void *Handle, const char *Name); + using DLErrorType = const char *(*)(); + + struct DlFcnValues { + Optional<void *> RTLDDefault; + DLOpenType dlopen = nullptr; + DLCloseType dlclose = nullptr; + DLSymType dlsym = nullptr; + DLErrorType dlerror = nullptr; + }; + + static Expected<std::unique_ptr<MachOPlatformSupport>> + Create(LLJIT &J, JITDylib &PlatformJITDylib) { + + // Make process symbols visible. + { + std::string ErrMsg; + auto Lib = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg); + if (!Lib.isValid()) + return make_error<StringError>(std::move(ErrMsg), + inconvertibleErrorCode()); + } + + DlFcnValues DlFcn; + + // Add support for RTLDDefault on known platforms. +#ifdef __APPLE__ + DlFcn.RTLDDefault = reinterpret_cast<void *>(-2); +#endif // __APPLE__ + + if (auto Err = hookUpFunction(DlFcn.dlopen, "dlopen")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlclose, "dlclose")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlsym, "dlsym")) + return std::move(Err); + if (auto Err = hookUpFunction(DlFcn.dlerror, "dlerror")) + return std::move(Err); + + std::unique_ptr<MachOPlatformSupport> MP( + new MachOPlatformSupport(J, PlatformJITDylib, DlFcn)); + return std::move(MP); + } + + Error initialize(JITDylib &JD) override { + LLVM_DEBUG({ + dbgs() << "MachOPlatformSupport initializing \"" << JD.getName() + << "\"\n"; + }); + + auto InitSeq = MP.getInitializerSequence(JD); + if (!InitSeq) + return InitSeq.takeError(); + + // If ObjC is not enabled but there are JIT'd ObjC inits then return + // an error. + if (!objCRegistrationEnabled()) + for (auto &KV : *InitSeq) { + if (!KV.second.getObjCSelRefsSections().empty() || + !KV.second.getObjCClassListSections().empty()) + return make_error<StringError>("JITDylib " + KV.first->getName() + + " contains objc metadata but objc" + " is not enabled", + inconvertibleErrorCode()); + } + + // Run the initializers. + for (auto &KV : *InitSeq) { + if (objCRegistrationEnabled()) { + KV.second.registerObjCSelectors(); + if (auto Err = KV.second.registerObjCClasses()) { + // FIXME: Roll back registrations on error? + return Err; + } + } + KV.second.runModInits(); + } + + return Error::success(); + } + + Error deinitialize(JITDylib &JD) override { + auto &ES = J.getExecutionSession(); + if (auto DeinitSeq = MP.getDeinitializerSequence(JD)) { + for (auto &KV : *DeinitSeq) { + auto DSOHandleName = ES.intern("___dso_handle"); + + // FIXME: Run DeInits here. + auto Result = ES.lookup( + {{KV.first, JITDylibLookupFlags::MatchAllSymbols}}, + SymbolLookupSet(DSOHandleName, + SymbolLookupFlags::WeaklyReferencedSymbol)); + if (!Result) + return Result.takeError(); + if (Result->empty()) + continue; + assert(Result->count(DSOHandleName) && + "Result does not contain __dso_handle"); + auto *DSOHandle = jitTargetAddressToPointer<void *>( + Result->begin()->second.getAddress()); + AtExitMgr.runAtExits(DSOHandle); + } + } else + return DeinitSeq.takeError(); + return Error::success(); + } + +private: + template <typename FunctionPtrTy> + static Error hookUpFunction(FunctionPtrTy &Fn, const char *Name) { + if (auto *FnAddr = sys::DynamicLibrary::SearchForAddressOfSymbol(Name)) { + Fn = reinterpret_cast<FunctionPtrTy>(Fn); + return Error::success(); + } + + return make_error<StringError>((Twine("Can not enable MachO JIT Platform: " + "missing function: ") + + Name) + .str(), + inconvertibleErrorCode()); + } + + MachOPlatformSupport(LLJIT &J, JITDylib &PlatformJITDylib, DlFcnValues DlFcn) + : J(J), MP(setupPlatform(J)), DlFcn(std::move(DlFcn)) { + + SymbolMap HelperSymbols; + + // platform and atexit helpers. + HelperSymbols[J.mangleAndIntern("__lljit.platform_support_instance")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper), + JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.run_atexits_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper), + JITSymbolFlags()); + + // dlfcn helpers. + HelperSymbols[J.mangleAndIntern("__lljit.dlopen_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(dlopenHelper), + JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.dlclose_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(dlcloseHelper), + JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.dlsym_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(dlsymHelper), + JITSymbolFlags()); + HelperSymbols[J.mangleAndIntern("__lljit.dlerror_helper")] = + JITEvaluatedSymbol(pointerToJITTargetAddress(dlerrorHelper), + JITSymbolFlags()); + + cantFail( + PlatformJITDylib.define(absoluteSymbols(std::move(HelperSymbols)))); + cantFail(MP.setupJITDylib(J.getMainJITDylib())); + cantFail(J.addIRModule(PlatformJITDylib, createPlatformRuntimeModule())); + } + + static MachOPlatform &setupPlatform(LLJIT &J) { + auto Tmp = std::make_unique<MachOPlatform>( + J.getExecutionSession(), + static_cast<ObjectLinkingLayer &>(J.getObjLinkingLayer()), + createStandardSymbolsObject(J)); + auto &MP = *Tmp; + J.getExecutionSession().setPlatform(std::move(Tmp)); + return MP; + } + + static std::unique_ptr<MemoryBuffer> createStandardSymbolsObject(LLJIT &J) { + LLVMContext Ctx; + Module M("__standard_symbols", Ctx); + M.setDataLayout(J.getDataLayout()); + + auto *Int64Ty = Type::getInt64Ty(Ctx); + + auto *DSOHandle = + new GlobalVariable(M, Int64Ty, true, GlobalValue::ExternalLinkage, + ConstantInt::get(Int64Ty, 0), "__dso_handle"); + DSOHandle->setVisibility(GlobalValue::DefaultVisibility); + + return cantFail(J.getIRCompileLayer().getCompiler()(M)); + } + + ThreadSafeModule createPlatformRuntimeModule() { + auto Ctx = std::make_unique<LLVMContext>(); + auto M = std::make_unique<Module>("__standard_lib", *Ctx); + M->setDataLayout(J.getDataLayout()); + + auto *MachOPlatformSupportTy = + StructType::create(*Ctx, "lljit.MachOPlatformSupport"); + + auto *PlatformInstanceDecl = new GlobalVariable( + *M, MachOPlatformSupportTy, true, GlobalValue::ExternalLinkage, nullptr, + "__lljit.platform_support_instance"); + + auto *Int8Ty = Type::getInt8Ty(*Ctx); + auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); + auto *VoidTy = Type::getVoidTy(*Ctx); + auto *BytePtrTy = PointerType::getUnqual(Int8Ty); + auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); + auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); + + addHelperAndWrapper( + *M, "__cxa_atexit", + FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, + false), + GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlopen", + FunctionType::get(BytePtrTy, {BytePtrTy, IntTy}, false), + GlobalValue::DefaultVisibility, "__lljit.dlopen_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlclose", + FunctionType::get(IntTy, {BytePtrTy}, false), + GlobalValue::DefaultVisibility, + "__lljit.dlclose_helper", {PlatformInstanceDecl}); + + addHelperAndWrapper( + *M, "dlsym", + FunctionType::get(BytePtrTy, {BytePtrTy, BytePtrTy}, false), + GlobalValue::DefaultVisibility, "__lljit.dlsym_helper", + {PlatformInstanceDecl}); + + addHelperAndWrapper(*M, "dlerror", FunctionType::get(BytePtrTy, {}, false), + GlobalValue::DefaultVisibility, + "__lljit.dlerror_helper", {PlatformInstanceDecl}); + + return ThreadSafeModule(std::move(M), std::move(Ctx)); + } + + static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, + void *DSOHandle) { + static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.registerAtExit( + F, Ctx, DSOHandle); + } + + static void runAtExitsHelper(void *Self, void *DSOHandle) { + static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.runAtExits(DSOHandle); + } + + void *jit_dlopen(const char *Path, int Mode) { + JITDylib *JDToOpen = nullptr; + // FIXME: Do the right thing with Mode flags. + { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + if (auto *JD = J.getExecutionSession().getJITDylibByName(Path)) { + auto I = JDRefCounts.find(JD); + if (I != JDRefCounts.end()) { + ++I->second; + return JD; + } + + JDRefCounts[JD] = 1; + JDToOpen = JD; + } + } + + if (JDToOpen) { + if (auto Err = initialize(*JDToOpen)) { + recordError(std::move(Err)); + return 0; + } + } + + // Fall through to dlopen if no JITDylib found for Path. + return DlFcn.dlopen(Path, Mode); + } + + static void *dlopenHelper(void *Self, const char *Path, int Mode) { + return static_cast<MachOPlatformSupport *>(Self)->jit_dlopen(Path, Mode); + } + + int jit_dlclose(void *Handle) { + JITDylib *JDToClose = nullptr; + + { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + auto I = JDRefCounts.find(Handle); + if (I != JDRefCounts.end()) { + --I->second; + if (I->second == 0) { + JDRefCounts.erase(I); + JDToClose = static_cast<JITDylib *>(Handle); + } else + return 0; + } + } + + if (JDToClose) { + if (auto Err = deinitialize(*JDToClose)) { + recordError(std::move(Err)); + return -1; + } + return 0; + } + + // Fall through to dlclose if no JITDylib found for Path. + return DlFcn.dlclose(Handle); + } + + static int dlcloseHelper(void *Self, void *Handle) { + return static_cast<MachOPlatformSupport *>(Self)->jit_dlclose(Handle); + } + + void *jit_dlsym(void *Handle, const char *Name) { + JITDylibSearchOrder JITSymSearchOrder; + + // FIXME: RTLD_NEXT, RTLD_SELF not supported. + { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + + // Clear any existing error messages. + dlErrorMsgs.erase(std::this_thread::get_id()); + + if (JDRefCounts.count(Handle)) { + JITSymSearchOrder.push_back( + {static_cast<JITDylib *>(Handle), + JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } else if (Handle == DlFcn.RTLDDefault) { + for (auto &KV : JDRefCounts) + JITSymSearchOrder.push_back( + {static_cast<JITDylib *>(KV.first), + JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } + } + + if (!JITSymSearchOrder.empty()) { + auto MangledName = J.mangleAndIntern(Name); + SymbolLookupSet Syms(MangledName, + SymbolLookupFlags::WeaklyReferencedSymbol); + if (auto Result = J.getExecutionSession().lookup(JITSymSearchOrder, Syms, + LookupKind::DLSym)) { + auto I = Result->find(MangledName); + if (I != Result->end()) + return jitTargetAddressToPointer<void *>(I->second.getAddress()); + } else { + recordError(Result.takeError()); + return 0; + } + } + + // Fall through to dlsym. + return DlFcn.dlsym(Handle, Name); + } + + static void *dlsymHelper(void *Self, void *Handle, const char *Name) { + return static_cast<MachOPlatformSupport *>(Self)->jit_dlsym(Handle, Name); + } + + const char *jit_dlerror() { + { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + auto I = dlErrorMsgs.find(std::this_thread::get_id()); + if (I != dlErrorMsgs.end()) + return I->second->c_str(); + } + return DlFcn.dlerror(); + } + + static const char *dlerrorHelper(void *Self) { + return static_cast<MachOPlatformSupport *>(Self)->jit_dlerror(); + } + + void recordError(Error Err) { + std::lock_guard<std::mutex> Lock(PlatformSupportMutex); + dlErrorMsgs[std::this_thread::get_id()] = + std::make_unique<std::string>(toString(std::move(Err))); + } + + std::mutex PlatformSupportMutex; + LLJIT &J; + MachOPlatform &MP; + DlFcnValues DlFcn; + ItaniumCXAAtExitSupport AtExitMgr; + DenseMap<void *, unsigned> JDRefCounts; + std::map<std::thread::id, std::unique_ptr<std::string>> dlErrorMsgs; +}; + +} // end anonymous namespace namespace llvm { namespace orc { +void LLJIT::PlatformSupport::setInitTransform( + LLJIT &J, IRTransformLayer::TransformFunction T) { + J.InitHelperTransformLayer->setTransform(std::move(T)); +} + +LLJIT::PlatformSupport::~PlatformSupport() {} + Error LLJITBuilderState::prepareForConstruction() { + LLVM_DEBUG(dbgs() << "Preparing to create LLIT instance...\n"); + if (!JTMB) { + LLVM_DEBUG({ + dbgs() << " No explicitly set JITTargetMachineBuilder. " + "Detecting host...\n"; + }); if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost()) JTMB = std::move(*JTMBOrErr); else return JTMBOrErr.takeError(); } + LLVM_DEBUG({ + dbgs() << " JITTargetMachineBuilder is " << JTMB << "\n" + << " Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No") + << "\n" + << " DataLayout: "; + if (DL) + dbgs() << DL->getStringRepresentation() << "\n"; + else + dbgs() << "None (will be created by JITTargetMachineBuilder)\n"; + + dbgs() << " Custom object-linking-layer creator: " + << (CreateObjectLinkingLayer ? "Yes" : "No") << "\n" + << " Custom compile-function creator: " + << (CreateCompileFunction ? "Yes" : "No") << "\n" + << " Custom platform-setup function: " + << (SetUpPlatform ? "Yes" : "No") << "\n" + << " Number of compile threads: " << NumCompileThreads; + if (!NumCompileThreads) + dbgs() << " (code will be compiled on the execution thread)\n"; + else + dbgs() << "\n"; + }); + // If the client didn't configure any linker options then auto-configure the // JIT linker. - if (!CreateObjectLinkingLayer && JTMB->getCodeModel() == None && - JTMB->getRelocationModel() == None) { - + if (!CreateObjectLinkingLayer) { auto &TT = JTMB->getTargetTriple(); if (TT.isOSBinFormatMachO() && (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64)) { @@ -40,8 +975,11 @@ Error LLJITBuilderState::prepareForConstruction() { CreateObjectLinkingLayer = [](ExecutionSession &ES, const Triple &) -> std::unique_ptr<ObjectLayer> { - return std::make_unique<ObjectLinkingLayer>( + auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>( ES, std::make_unique<jitlink::InProcessMemoryManager>()); + ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>( + jitlink::InProcessEHFrameRegistrar::getInstance())); + return std::move(ObjLinkingLayer); }; } } @@ -54,12 +992,6 @@ LLJIT::~LLJIT() { CompileThreads->wait(); } -Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { - auto InternedName = ES->intern(Name); - SymbolMap Symbols({{InternedName, Sym}}); - return Main.define(absoluteSymbols(std::move(Symbols))); -} - Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { assert(TSM && "Can not add null module"); @@ -67,7 +999,8 @@ Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); })) return Err; - return CompileLayer->add(JD, std::move(TSM), ES->allocateVModule()); + return InitHelperTransformLayer->add(JD, std::move(TSM), + ES->allocateVModule()); } Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { @@ -77,10 +1010,9 @@ Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr<MemoryBuffer> Obj) { } Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD, - StringRef Name) { + SymbolStringPtr Name) { return ES->lookup( - makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), - ES->intern(Name)); + makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name); } std::unique_ptr<ObjectLayer> @@ -96,8 +1028,10 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { auto ObjLinkingLayer = std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); - if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) + if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) { ObjLinkingLayer->setOverrideObjectFlagsWithResponsibilityFlags(true); + ObjLinkingLayer->setAutoClaimResponsibilityForObjectSymbols(true); + } // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence // errors from some GCC / libstdc++ bots. Remove this conversion (i.e. @@ -105,7 +1039,7 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { return std::unique_ptr<ObjectLayer>(std::move(ObjLinkingLayer)); } -Expected<IRCompileLayer::CompileFunction> +Expected<std::unique_ptr<IRCompileLayer::IRCompiler>> LLJIT::createCompileFunction(LLJITBuilderState &S, JITTargetMachineBuilder JTMB) { @@ -116,25 +1050,33 @@ LLJIT::createCompileFunction(LLJITBuilderState &S, // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler, // depending on the number of threads requested. if (S.NumCompileThreads > 0) - return ConcurrentIRCompiler(std::move(JTMB)); + return std::make_unique<ConcurrentIRCompiler>(std::move(JTMB)); auto TM = JTMB.createTargetMachine(); if (!TM) return TM.takeError(); - return TMOwningSimpleCompiler(std::move(*TM)); + return std::make_unique<TMOwningSimpleCompiler>(std::move(*TM)); } LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) - : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), - Main(this->ES->createJITDylib("<main>")), DL(""), + : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), Main(), + DL(""), TT(S.JTMB->getTargetTriple()), ObjLinkingLayer(createObjectLinkingLayer(S, *ES)), - ObjTransformLayer(*this->ES, *ObjLinkingLayer), CtorRunner(Main), - DtorRunner(Main) { + ObjTransformLayer(*this->ES, *ObjLinkingLayer) { ErrorAsOutParameter _(&Err); - if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) + if (auto MainOrErr = this->ES->createJITDylib("main")) + Main = &*MainOrErr; + else { + Err = MainOrErr.takeError(); + return; + } + + if (S.DL) + DL = std::move(*S.DL); + else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) DL = std::move(*DLOrErr); else { Err = DLOrErr.takeError(); @@ -149,22 +1091,36 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) } CompileLayer = std::make_unique<IRCompileLayer>( *ES, ObjTransformLayer, std::move(*CompileFunction)); + TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer); + InitHelperTransformLayer = + std::make_unique<IRTransformLayer>(*ES, *TransformLayer); } if (S.NumCompileThreads > 0) { - CompileLayer->setCloneToNewContextOnEmit(true); - CompileThreads = std::make_unique<ThreadPool>(S.NumCompileThreads); + InitHelperTransformLayer->setCloneToNewContextOnEmit(true); + CompileThreads = + std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads)); ES->setDispatchMaterialization( - [this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) { - // FIXME: Switch to move capture once we have c++14. + [this](std::unique_ptr<MaterializationUnit> MU, + MaterializationResponsibility MR) { + // FIXME: Switch to move capture once ThreadPool uses unique_function. auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU)); - auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); }; + auto SharedMR = + std::make_shared<MaterializationResponsibility>(std::move(MR)); + auto Work = [SharedMU, SharedMR]() mutable { + SharedMU->materialize(std::move(*SharedMR)); + }; CompileThreads->async(std::move(Work)); }); } + + if (S.SetUpPlatform) + Err = S.SetUpPlatform(*this); + else + setUpGenericLLVMIRPlatform(*this); } -std::string LLJIT::mangle(StringRef UnmangledName) { +std::string LLJIT::mangle(StringRef UnmangledName) const { std::string MangledName; { raw_string_ostream MangledNameStream(MangledName); @@ -179,15 +1135,27 @@ Error LLJIT::applyDataLayout(Module &M) { if (M.getDataLayout() != DL) return make_error<StringError>( - "Added modules have incompatible data layouts", + "Added modules have incompatible data layouts: " + + M.getDataLayout().getStringRepresentation() + " (module) vs " + + DL.getStringRepresentation() + " (jit)", inconvertibleErrorCode()); return Error::success(); } -void LLJIT::recordCtorDtors(Module &M) { - CtorRunner.add(getConstructors(M)); - DtorRunner.add(getDestructors(M)); +void setUpGenericLLVMIRPlatform(LLJIT &J) { + LLVM_DEBUG( + { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; }); + J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J)); +} + +Error setUpMachOPlatform(LLJIT &J) { + LLVM_DEBUG({ dbgs() << "Setting up MachOPlatform support for LLJIT\n"; }); + auto MP = MachOPlatformSupport::Create(J, J.getMainJITDylib()); + if (!MP) + return MP.takeError(); + J.setPlatformSupport(std::move(*MP)); + return Error::success(); } Error LLLazyJITBuilderState::prepareForConstruction() { @@ -200,13 +1168,8 @@ Error LLLazyJITBuilderState::prepareForConstruction() { Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { assert(TSM && "Can not add null module"); - if (auto Err = TSM.withModuleDo([&](Module &M) -> Error { - if (auto Err = applyDataLayout(M)) - return Err; - - recordCtorDtors(M); - return Error::success(); - })) + if (auto Err = TSM.withModuleDo( + [&](Module &M) -> Error { return applyDataLayout(M); })) return Err; return CODLayer->add(JD, std::move(TSM), ES->allocateVModule()); @@ -249,12 +1212,9 @@ LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { return; } - // Create the transform layer. - TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer); - // Create the COD layer. CODLayer = std::make_unique<CompileOnDemandLayer>( - *ES, *TransformLayer, *LCTMgr, std::move(ISMBuilder)); + *ES, *InitHelperTransformLayer, *LCTMgr, std::move(ISMBuilder)); if (S.NumCompileThreads > 0) CODLayer->setCloneToNewContextOnEmit(true); diff --git a/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/llvm/lib/ExecutionEngine/Orc/Layer.cpp index 580e2682ec8c6..61e7ab5ae68b7 100644 --- a/llvm/lib/ExecutionEngine/Orc/Layer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -7,6 +7,11 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Layer.h" + +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/IR/Constants.h" +#include "llvm/Object/MachO.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" @@ -15,28 +20,79 @@ namespace llvm { namespace orc { -IRLayer::IRLayer(ExecutionSession &ES) : ES(ES) {} IRLayer::~IRLayer() {} Error IRLayer::add(JITDylib &JD, ThreadSafeModule TSM, VModuleKey K) { return JD.define(std::make_unique<BasicIRLayerMaterializationUnit>( - *this, std::move(K), std::move(TSM))); + *this, *getManglingOptions(), std::move(TSM), std::move(K))); } -IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, - ThreadSafeModule TSM, VModuleKey K) - : MaterializationUnit(SymbolFlagsMap(), std::move(K)), TSM(std::move(TSM)) { +IRMaterializationUnit::IRMaterializationUnit( + ExecutionSession &ES, const IRSymbolMapper::ManglingOptions &MO, + ThreadSafeModule TSM, VModuleKey K) + : MaterializationUnit(SymbolFlagsMap(), nullptr, std::move(K)), + TSM(std::move(TSM)) { assert(this->TSM && "Module must not be null"); MangleAndInterner Mangle(ES, this->TSM.getModuleUnlocked()->getDataLayout()); this->TSM.withModuleDo([&](Module &M) { for (auto &G : M.global_values()) { - if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() && - !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) { - auto MangledName = Mangle(G.getName()); - SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); - SymbolToDefinition[MangledName] = &G; + // Skip globals that don't generate symbols. + + if (!G.hasName() || G.isDeclaration() || G.hasLocalLinkage() || + G.hasAvailableExternallyLinkage() || G.hasAppendingLinkage()) + continue; + + // thread locals generate different symbols depending on whether or not + // emulated TLS is enabled. + if (G.isThreadLocal() && MO.EmulatedTLS) { + auto &GV = cast<GlobalVariable>(G); + + auto Flags = JITSymbolFlags::fromGlobalValue(GV); + + auto EmuTLSV = Mangle(("__emutls_v." + GV.getName()).str()); + SymbolFlags[EmuTLSV] = Flags; + SymbolToDefinition[EmuTLSV] = &GV; + + // If this GV has a non-zero initializer we'll need to emit an + // __emutls.t symbol too. + if (GV.hasInitializer()) { + const auto *InitVal = GV.getInitializer(); + + // Skip zero-initializers. + if (isa<ConstantAggregateZero>(InitVal)) + continue; + const auto *InitIntValue = dyn_cast<ConstantInt>(InitVal); + if (InitIntValue && InitIntValue->isZero()) + continue; + + auto EmuTLST = Mangle(("__emutls_t." + GV.getName()).str()); + SymbolFlags[EmuTLST] = Flags; + } + continue; + } + + // Otherwise we just need a normal linker mangling. + auto MangledName = Mangle(G.getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); + SymbolToDefinition[MangledName] = &G; + } + + // If we need an init symbol for this module then create one. + if (!llvm::empty(getStaticInitGVs(M))) { + size_t Counter = 0; + + while (true) { + std::string InitSymbolName; + raw_string_ostream(InitSymbolName) + << "$." << M.getModuleIdentifier() << ".__inits." << Counter++; + InitSymbol = ES.intern(InitSymbolName); + if (SymbolFlags.count(InitSymbol)) + continue; + SymbolFlags[InitSymbol] = + JITSymbolFlags::MaterializationSideEffectsOnly; + break; } } }); @@ -44,8 +100,9 @@ IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES, IRMaterializationUnit::IRMaterializationUnit( ThreadSafeModule TSM, VModuleKey K, SymbolFlagsMap SymbolFlags, - SymbolNameToDefinitionMap SymbolToDefinition) - : MaterializationUnit(std::move(SymbolFlags), std::move(K)), + SymbolStringPtr InitSymbol, SymbolNameToDefinitionMap SymbolToDefinition) + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol), + std::move(K)), TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {} StringRef IRMaterializationUnit::getName() const { @@ -72,8 +129,9 @@ void IRMaterializationUnit::discard(const JITDylib &JD, } BasicIRLayerMaterializationUnit::BasicIRLayerMaterializationUnit( - IRLayer &L, VModuleKey K, ThreadSafeModule TSM) - : IRMaterializationUnit(L.getExecutionSession(), std::move(TSM), + IRLayer &L, const IRSymbolMapper::ManglingOptions &MO, ThreadSafeModule TSM, + VModuleKey K) + : IRMaterializationUnit(L.getExecutionSession(), MO, std::move(TSM), std::move(K)), L(L), K(std::move(K)) {} @@ -117,22 +175,26 @@ Error ObjectLayer::add(JITDylib &JD, std::unique_ptr<MemoryBuffer> O, Expected<std::unique_ptr<BasicObjectLayerMaterializationUnit>> BasicObjectLayerMaterializationUnit::Create(ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O) { - auto SymbolFlags = - getObjectSymbolFlags(L.getExecutionSession(), O->getMemBufferRef()); + auto ObjSymInfo = + getObjectSymbolInfo(L.getExecutionSession(), O->getMemBufferRef()); + + if (!ObjSymInfo) + return ObjSymInfo.takeError(); - if (!SymbolFlags) - return SymbolFlags.takeError(); + auto &SymbolFlags = ObjSymInfo->first; + auto &InitSymbol = ObjSymInfo->second; return std::unique_ptr<BasicObjectLayerMaterializationUnit>( - new BasicObjectLayerMaterializationUnit(L, K, std::move(O), - std::move(*SymbolFlags))); + new BasicObjectLayerMaterializationUnit( + L, K, std::move(O), std::move(SymbolFlags), std::move(InitSymbol))); } BasicObjectLayerMaterializationUnit::BasicObjectLayerMaterializationUnit( ObjectLayer &L, VModuleKey K, std::unique_ptr<MemoryBuffer> O, - SymbolFlagsMap SymbolFlags) - : MaterializationUnit(std::move(SymbolFlags), std::move(K)), L(L), - O(std::move(O)) {} + SymbolFlagsMap SymbolFlags, SymbolStringPtr InitSymbol) + : MaterializationUnit(std::move(SymbolFlags), std::move(InitSymbol), + std::move(K)), + L(L), O(std::move(O)) {} StringRef BasicObjectLayerMaterializationUnit::getName() const { if (O) @@ -147,38 +209,8 @@ void BasicObjectLayerMaterializationUnit::materialize( void BasicObjectLayerMaterializationUnit::discard(const JITDylib &JD, const SymbolStringPtr &Name) { - // FIXME: Support object file level discard. This could be done by building a - // filter to pass to the object layer along with the object itself. -} - -Expected<SymbolFlagsMap> getObjectSymbolFlags(ExecutionSession &ES, - MemoryBufferRef ObjBuffer) { - auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); - - if (!Obj) - return Obj.takeError(); - - SymbolFlagsMap SymbolFlags; - for (auto &Sym : (*Obj)->symbols()) { - // Skip symbols not defined in this object file. - if (Sym.getFlags() & object::BasicSymbolRef::SF_Undefined) - continue; - - // Skip symbols that are not global. - if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) - continue; - - auto Name = Sym.getName(); - if (!Name) - return Name.takeError(); - auto InternedName = ES.intern(*Name); - auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); - if (!SymFlags) - return SymFlags.takeError(); - SymbolFlags[InternedName] = std::move(*SymFlags); - } - - return SymbolFlags; + // This is a no-op for object files: Having removed 'Name' from SymbolFlags + // the symbol will be dead-stripped by the JIT linker. } } // End namespace orc. diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp index aab490feb8ea2..153f6b80784f0 100644 --- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -16,70 +16,86 @@ namespace llvm { namespace orc { -void LazyCallThroughManager::NotifyResolvedFunction::anchor() {} - LazyCallThroughManager::LazyCallThroughManager( - ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr, - std::unique_ptr<TrampolinePool> TP) - : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(std::move(TP)) {} + ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr, TrampolinePool *TP) + : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(TP) {} Expected<JITTargetAddress> LazyCallThroughManager::getCallThroughTrampoline( JITDylib &SourceJD, SymbolStringPtr SymbolName, - std::shared_ptr<NotifyResolvedFunction> NotifyResolved) { + NotifyResolvedFunction NotifyResolved) { + assert(TP && "TrampolinePool not set"); + std::lock_guard<std::mutex> Lock(LCTMMutex); auto Trampoline = TP->getTrampoline(); if (!Trampoline) return Trampoline.takeError(); - Reexports[*Trampoline] = std::make_pair(&SourceJD, std::move(SymbolName)); + Reexports[*Trampoline] = ReexportsEntry{&SourceJD, std::move(SymbolName)}; Notifiers[*Trampoline] = std::move(NotifyResolved); return *Trampoline; } -JITTargetAddress -LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) { - JITDylib *SourceJD = nullptr; - SymbolStringPtr SymbolName; - - { - std::lock_guard<std::mutex> Lock(LCTMMutex); - auto I = Reexports.find(TrampolineAddr); - if (I == Reexports.end()) - return ErrorHandlerAddr; - SourceJD = I->second.first; - SymbolName = I->second.second; - } - - auto LookupResult = ES.lookup( - makeJITDylibSearchOrder(SourceJD, JITDylibLookupFlags::MatchAllSymbols), - SymbolName); - - if (!LookupResult) { - ES.reportError(LookupResult.takeError()); - return ErrorHandlerAddr; - } +JITTargetAddress LazyCallThroughManager::reportCallThroughError(Error Err) { + ES.reportError(std::move(Err)); + return ErrorHandlerAddr; +} - auto ResolvedAddr = LookupResult->getAddress(); +Expected<LazyCallThroughManager::ReexportsEntry> +LazyCallThroughManager::findReexport(JITTargetAddress TrampolineAddr) { + std::lock_guard<std::mutex> Lock(LCTMMutex); + auto I = Reexports.find(TrampolineAddr); + if (I == Reexports.end()) + return createStringError(inconvertibleErrorCode(), + "Missing reexport for trampoline address %p", + TrampolineAddr); + return I->second; +} - std::shared_ptr<NotifyResolvedFunction> NotifyResolved = nullptr; +Error LazyCallThroughManager::notifyResolved(JITTargetAddress TrampolineAddr, + JITTargetAddress ResolvedAddr) { + NotifyResolvedFunction NotifyResolved; { std::lock_guard<std::mutex> Lock(LCTMMutex); auto I = Notifiers.find(TrampolineAddr); if (I != Notifiers.end()) { - NotifyResolved = I->second; + NotifyResolved = std::move(I->second); Notifiers.erase(I); } } - if (NotifyResolved) { - if (auto Err = (*NotifyResolved)(*SourceJD, SymbolName, ResolvedAddr)) { - ES.reportError(std::move(Err)); - return ErrorHandlerAddr; - } - } + return NotifyResolved ? NotifyResolved(ResolvedAddr) : Error::success(); +} - return ResolvedAddr; +void LazyCallThroughManager::resolveTrampolineLandingAddress( + JITTargetAddress TrampolineAddr, + NotifyLandingResolvedFunction NotifyLandingResolved) { + + auto Entry = findReexport(TrampolineAddr); + if (!Entry) + return NotifyLandingResolved(reportCallThroughError(Entry.takeError())); + + ES.lookup( + LookupKind::Static, + makeJITDylibSearchOrder(Entry->SourceJD, + JITDylibLookupFlags::MatchAllSymbols), + SymbolLookupSet({Entry->SymbolName}), SymbolState::Ready, + [this, TrampolineAddr, SymbolName = Entry->SymbolName, + NotifyLandingResolved = std::move(NotifyLandingResolved)]( + Expected<SymbolMap> Result) mutable { + if (Result) { + assert(Result->size() == 1 && "Unexpected result size"); + assert(Result->count(SymbolName) && "Unexpected result value"); + JITTargetAddress LandingAddr = (*Result)[SymbolName].getAddress(); + + if (auto Err = notifyResolved(TrampolineAddr, LandingAddr)) + NotifyLandingResolved(reportCallThroughError(std::move(Err))); + else + NotifyLandingResolved(LandingAddr); + } else + NotifyLandingResolved(reportCallThroughError(Result.takeError())); + }, + NoDependenciesToRegister); } Expected<std::unique_ptr<LazyCallThroughManager>> @@ -125,15 +141,9 @@ LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit( LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc, VModuleKey K) - : MaterializationUnit(extractFlags(CallableAliases), std::move(K)), + : MaterializationUnit(extractFlags(CallableAliases), nullptr, std::move(K)), LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD), - CallableAliases(std::move(CallableAliases)), - NotifyResolved(LazyCallThroughManager::createNotifyResolvedFunction( - [&ISManager](JITDylib &JD, const SymbolStringPtr &SymbolName, - JITTargetAddress ResolvedAddr) { - return ISManager.updatePointer(*SymbolName, ResolvedAddr); - })), - AliaseeTable(SrcJDLoc) {} + CallableAliases(std::move(CallableAliases)), AliaseeTable(SrcJDLoc) {} StringRef LazyReexportsMaterializationUnit::getName() const { return "<Lazy Reexports>"; @@ -159,7 +169,11 @@ void LazyReexportsMaterializationUnit::materialize( for (auto &Alias : RequestedAliases) { auto CallThroughTrampoline = LCTManager.getCallThroughTrampoline( - SourceJD, Alias.second.Aliasee, NotifyResolved); + SourceJD, Alias.second.Aliasee, + [&ISManager = this->ISManager, + StubSym = Alias.first](JITTargetAddress ResolvedAddr) -> Error { + return ISManager.updatePointer(*StubSym, ResolvedAddr); + }); if (!CallThroughTrampoline) { SourceJD.getExecutionSession().reportError( diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp new file mode 100644 index 0000000000000..15c3aa79a2a84 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -0,0 +1,506 @@ +//===------ MachOPlatform.cpp - Utilities for executing MachO in Orc ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" + +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/Support/BinaryByteStream.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" + +namespace { + +struct objc_class; +struct objc_image_info; +struct objc_object; +struct objc_selector; + +using Class = objc_class *; +using id = objc_object *; +using SEL = objc_selector *; + +using ObjCMsgSendTy = id (*)(id, SEL, ...); +using ObjCReadClassPairTy = Class (*)(Class, const objc_image_info *); +using SelRegisterNameTy = SEL (*)(const char *); + +enum class ObjCRegistrationAPI { Uninitialized, Unavailable, Initialized }; + +ObjCRegistrationAPI ObjCRegistrationAPIState = + ObjCRegistrationAPI::Uninitialized; +ObjCMsgSendTy objc_msgSend = nullptr; +ObjCReadClassPairTy objc_readClassPair = nullptr; +SelRegisterNameTy sel_registerName = nullptr; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +template <typename FnTy> +static Error setUpObjCRegAPIFunc(FnTy &Target, sys::DynamicLibrary &LibObjC, + const char *Name) { + if (void *Addr = LibObjC.getAddressOfSymbol(Name)) + Target = reinterpret_cast<FnTy>(Addr); + else + return make_error<StringError>( + (Twine("Could not find address for ") + Name).str(), + inconvertibleErrorCode()); + return Error::success(); +} + +Error enableObjCRegistration(const char *PathToLibObjC) { + // If we've already tried to initialize then just bail out. + if (ObjCRegistrationAPIState != ObjCRegistrationAPI::Uninitialized) + return Error::success(); + + ObjCRegistrationAPIState = ObjCRegistrationAPI::Unavailable; + + std::string ErrMsg; + auto LibObjC = + sys::DynamicLibrary::getPermanentLibrary(PathToLibObjC, &ErrMsg); + + if (!LibObjC.isValid()) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + + if (auto Err = setUpObjCRegAPIFunc(objc_msgSend, LibObjC, "objc_msgSend")) + return Err; + if (auto Err = setUpObjCRegAPIFunc(objc_readClassPair, LibObjC, + "objc_readClassPair")) + return Err; + if (auto Err = + setUpObjCRegAPIFunc(sel_registerName, LibObjC, "sel_registerName")) + return Err; + + ObjCRegistrationAPIState = ObjCRegistrationAPI::Initialized; + return Error::success(); +} + +bool objCRegistrationEnabled() { + return ObjCRegistrationAPIState == ObjCRegistrationAPI::Initialized; +} + +void MachOJITDylibInitializers::runModInits() const { + for (const auto &ModInit : ModInitSections) { + for (uint64_t I = 0; I != ModInit.NumPtrs; ++I) { + auto *InitializerAddr = jitTargetAddressToPointer<uintptr_t *>( + ModInit.Address + (I * sizeof(uintptr_t))); + auto *Initializer = + jitTargetAddressToFunction<void (*)()>(*InitializerAddr); + Initializer(); + } + } +} + +void MachOJITDylibInitializers::registerObjCSelectors() const { + assert(objCRegistrationEnabled() && "ObjC registration not enabled."); + + for (const auto &ObjCSelRefs : ObjCSelRefsSections) { + for (uint64_t I = 0; I != ObjCSelRefs.NumPtrs; ++I) { + auto SelEntryAddr = ObjCSelRefs.Address + (I * sizeof(uintptr_t)); + const auto *SelName = + *jitTargetAddressToPointer<const char **>(SelEntryAddr); + auto Sel = sel_registerName(SelName); + *jitTargetAddressToPointer<SEL *>(SelEntryAddr) = Sel; + } + } +} + +Error MachOJITDylibInitializers::registerObjCClasses() const { + assert(objCRegistrationEnabled() && "ObjC registration not enabled."); + + struct ObjCClassCompiled { + void *Metaclass; + void *Parent; + void *Cache1; + void *Cache2; + void *Data; + }; + + auto *ImageInfo = + jitTargetAddressToPointer<const objc_image_info *>(ObjCImageInfoAddr); + auto ClassSelector = sel_registerName("class"); + + for (const auto &ObjCClassList : ObjCClassListSections) { + for (uint64_t I = 0; I != ObjCClassList.NumPtrs; ++I) { + auto ClassPtrAddr = ObjCClassList.Address + (I * sizeof(uintptr_t)); + auto Cls = *jitTargetAddressToPointer<Class *>(ClassPtrAddr); + auto *ClassCompiled = + *jitTargetAddressToPointer<ObjCClassCompiled **>(ClassPtrAddr); + objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent), ClassSelector); + auto Registered = objc_readClassPair(Cls, ImageInfo); + + // FIXME: Improve diagnostic by reporting the failed class's name. + if (Registered != Cls) + return make_error<StringError>("Unable to register Objective-C class", + inconvertibleErrorCode()); + } + } + return Error::success(); +} + +MachOPlatform::MachOPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + std::unique_ptr<MemoryBuffer> StandardSymbolsObject) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), + StandardSymbolsObject(std::move(StandardSymbolsObject)) { + ObjLinkingLayer.addPlugin(std::make_unique<InitScraperPlugin>(*this)); +} + +Error MachOPlatform::setupJITDylib(JITDylib &JD) { + auto ObjBuffer = MemoryBuffer::getMemBuffer( + StandardSymbolsObject->getMemBufferRef(), false); + return ObjLinkingLayer.add(JD, std::move(ObjBuffer)); +} + +Error MachOPlatform::notifyAdding(JITDylib &JD, const MaterializationUnit &MU) { + const auto &InitSym = MU.getInitializerSymbol(); + if (!InitSym) + return Error::success(); + + RegisteredInitSymbols[&JD].add(InitSym, + SymbolLookupFlags::WeaklyReferencedSymbol); + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Registered init symbol " << *InitSym << " for MU " + << MU.getName() << "\n"; + }); + return Error::success(); +} + +Error MachOPlatform::notifyRemoving(JITDylib &JD, VModuleKey K) { + llvm_unreachable("Not supported yet"); +} + +Expected<MachOPlatform::InitializerSequence> +MachOPlatform::getInitializerSequence(JITDylib &JD) { + + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Building initializer sequence for " + << JD.getName() << "\n"; + }); + + std::vector<JITDylib *> DFSLinkOrder; + + while (true) { + + DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; + + ES.runSessionLocked([&]() { + DFSLinkOrder = getDFSLinkOrder(JD); + + for (auto *InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[InitJD] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } + } + }); + + if (NewInitSymbols.empty()) + break; + + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Issuing lookups for new init symbols: " + "(lookup may require multiple rounds)\n"; + for (auto &KV : NewInitSymbols) + dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; + }); + + // Outside the lock, issue the lookup. + if (auto R = lookupInitSymbols(JD.getExecutionSession(), NewInitSymbols)) + ; // Nothing to do in the success case. + else + return R.takeError(); + } + + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Init symbol lookup complete, building init " + "sequence\n"; + }); + + // Lock again to collect the initializers. + InitializerSequence FullInitSeq; + { + std::lock_guard<std::mutex> Lock(InitSeqsMutex); + for (auto *InitJD : reverse(DFSLinkOrder)) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Appending inits for \"" << InitJD->getName() + << "\" to sequence\n"; + }); + auto ISItr = InitSeqs.find(InitJD); + if (ISItr != InitSeqs.end()) { + FullInitSeq.emplace_back(InitJD, std::move(ISItr->second)); + InitSeqs.erase(ISItr); + } + } + } + + return FullInitSeq; +} + +Expected<MachOPlatform::DeinitializerSequence> +MachOPlatform::getDeinitializerSequence(JITDylib &JD) { + std::vector<JITDylib *> DFSLinkOrder = getDFSLinkOrder(JD); + + DeinitializerSequence FullDeinitSeq; + { + std::lock_guard<std::mutex> Lock(InitSeqsMutex); + for (auto *DeinitJD : DFSLinkOrder) { + FullDeinitSeq.emplace_back(DeinitJD, MachOJITDylibDeinitializers()); + } + } + + return FullDeinitSeq; +} + +std::vector<JITDylib *> MachOPlatform::getDFSLinkOrder(JITDylib &JD) { + std::vector<JITDylib *> Result, WorkStack({&JD}); + DenseSet<JITDylib *> Visited; + + while (!WorkStack.empty()) { + auto *NextJD = WorkStack.back(); + WorkStack.pop_back(); + if (Visited.count(NextJD)) + continue; + Visited.insert(NextJD); + Result.push_back(NextJD); + NextJD->withLinkOrderDo([&](const JITDylibSearchOrder &LO) { + for (auto &KV : LO) + WorkStack.push_back(KV.first); + }); + } + + return Result; +} + +void MachOPlatform::registerInitInfo( + JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, + MachOJITDylibInitializers::SectionExtent ModInits, + MachOJITDylibInitializers::SectionExtent ObjCSelRefs, + MachOJITDylibInitializers::SectionExtent ObjCClassList) { + std::lock_guard<std::mutex> Lock(InitSeqsMutex); + + auto &InitSeq = InitSeqs[&JD]; + + InitSeq.setObjCImageInfoAddr(ObjCImageInfoAddr); + + if (ModInits.Address) + InitSeq.addModInitsSection(std::move(ModInits)); + + if (ObjCSelRefs.Address) + InitSeq.addObjCSelRefsSection(std::move(ObjCSelRefs)); + + if (ObjCClassList.Address) + InitSeq.addObjCClassListSection(std::move(ObjCClassList)); +} + +static Expected<MachOJITDylibInitializers::SectionExtent> +getSectionExtent(jitlink::LinkGraph &G, StringRef SectionName) { + auto *Sec = G.findSectionByName(SectionName); + if (!Sec) + return MachOJITDylibInitializers::SectionExtent(); + jitlink::SectionRange R(*Sec); + if (R.getSize() % G.getPointerSize() != 0) + return make_error<StringError>(SectionName + " section size is not a " + "multiple of the pointer size", + inconvertibleErrorCode()); + return MachOJITDylibInitializers::SectionExtent( + R.getStart(), R.getSize() / G.getPointerSize()); +} + +void MachOPlatform::InitScraperPlugin::modifyPassConfig( + MaterializationResponsibility &MR, const Triple &TT, + jitlink::PassConfiguration &Config) { + + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { + JITLinkSymbolVector InitSectionSymbols; + preserveInitSectionIfPresent(InitSectionSymbols, G, "__mod_init_func"); + preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_selrefs"); + preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_classlist"); + + if (!InitSymbolDeps.empty()) { + std::lock_guard<std::mutex> Lock(InitScraperMutex); + InitSymbolDeps[&MR] = std::move(InitSectionSymbols); + } + + if (auto Err = processObjCImageInfo(G, MR)) + return Err; + + return Error::success(); + }); + + Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib()]( + jitlink::LinkGraph &G) -> Error { + MachOJITDylibInitializers::SectionExtent ModInits, ObjCSelRefs, + ObjCClassList; + + JITTargetAddress ObjCImageInfoAddr = 0; + if (auto *ObjCImageInfoSec = G.findSectionByName("__objc_image_info")) { + if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) { + ObjCImageInfoAddr = Addr; + dbgs() << "Recorded __objc_imageinfo @ " << formatv("{0:x16}", Addr); + } + } + + // Record __mod_init_func. + if (auto ModInitsOrErr = getSectionExtent(G, "__mod_init_func")) + ModInits = std::move(*ModInitsOrErr); + else + return ModInitsOrErr.takeError(); + + // Record __objc_selrefs. + if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__objc_selrefs")) + ObjCSelRefs = std::move(*ObjCSelRefsOrErr); + else + return ObjCSelRefsOrErr.takeError(); + + // Record __objc_classlist. + if (auto ObjCClassListOrErr = getSectionExtent(G, "__objc_classlist")) + ObjCClassList = std::move(*ObjCClassListOrErr); + else + return ObjCClassListOrErr.takeError(); + + // Dump the scraped inits. + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; + dbgs() << " __objc_selrefs: "; + if (ObjCSelRefs.NumPtrs) + dbgs() << ObjCSelRefs.NumPtrs << " pointer(s) at " + << formatv("{0:x16}", ObjCSelRefs.Address) << "\n"; + else + dbgs() << "none\n"; + + dbgs() << " __objc_classlist: "; + if (ObjCClassList.NumPtrs) + dbgs() << ObjCClassList.NumPtrs << " pointer(s) at " + << formatv("{0:x16}", ObjCClassList.Address) << "\n"; + else + dbgs() << "none\n"; + + dbgs() << " __mod_init_func: "; + if (ModInits.NumPtrs) + dbgs() << ModInits.NumPtrs << " pointer(s) at " + << formatv("{0:x16}", ModInits.Address) << "\n"; + else + dbgs() << "none\n"; + }); + + MP.registerInitInfo(JD, ObjCImageInfoAddr, std::move(ModInits), + std::move(ObjCSelRefs), std::move(ObjCClassList)); + + return Error::success(); + }); +} + +ObjectLinkingLayer::Plugin::LocalDependenciesMap +MachOPlatform::InitScraperPlugin::getSyntheticSymbolLocalDependencies( + MaterializationResponsibility &MR) { + std::lock_guard<std::mutex> Lock(InitScraperMutex); + auto I = InitSymbolDeps.find(&MR); + if (I != InitSymbolDeps.end()) { + LocalDependenciesMap Result; + Result[MR.getInitializerSymbol()] = std::move(I->second); + InitSymbolDeps.erase(&MR); + return Result; + } + return LocalDependenciesMap(); +} + +void MachOPlatform::InitScraperPlugin::preserveInitSectionIfPresent( + JITLinkSymbolVector &Symbols, jitlink::LinkGraph &G, + StringRef SectionName) { + if (auto *Sec = G.findSectionByName(SectionName)) { + auto SecBlocks = Sec->blocks(); + if (!llvm::empty(SecBlocks)) + Symbols.push_back( + &G.addAnonymousSymbol(**SecBlocks.begin(), 0, 0, false, true)); + } +} + +Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + + // If there's an ObjC imagine info then either + // (1) It's the first __objc_imageinfo we've seen in this JITDylib. In + // this case we name and record it. + // OR + // (2) We already have a recorded __objc_imageinfo for this JITDylib, + // in which case we just verify it. + auto *ObjCImageInfo = G.findSectionByName("__objc_imageinfo"); + if (!ObjCImageInfo) + return Error::success(); + + auto ObjCImageInfoBlocks = ObjCImageInfo->blocks(); + + // Check that the section is not empty if present. + if (llvm::empty(ObjCImageInfoBlocks)) + return make_error<StringError>("Empty __objc_imageinfo section in " + + G.getName(), + inconvertibleErrorCode()); + + // Check that there's only one block in the section. + if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) + return make_error<StringError>("Multiple blocks in __objc_imageinfo " + "section in " + + G.getName(), + inconvertibleErrorCode()); + + // Check that the __objc_imageinfo section is unreferenced. + // FIXME: We could optimize this check if Symbols had a ref-count. + for (auto &Sec : G.sections()) { + if (&Sec != ObjCImageInfo) + for (auto *B : Sec.blocks()) + for (auto &E : B->edges()) + if (E.getTarget().isDefined() && + &E.getTarget().getBlock().getSection() == ObjCImageInfo) + return make_error<StringError>("__objc_imageinfo is referenced " + "within file " + + G.getName(), + inconvertibleErrorCode()); + } + + auto &ObjCImageInfoBlock = **ObjCImageInfoBlocks.begin(); + auto *ObjCImageInfoData = ObjCImageInfoBlock.getContent().data(); + auto Version = support::endian::read32(ObjCImageInfoData, G.getEndianness()); + auto Flags = + support::endian::read32(ObjCImageInfoData + 4, G.getEndianness()); + + // Lock the mutex while we verify / update the ObjCImageInfos map. + std::lock_guard<std::mutex> Lock(InitScraperMutex); + + auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib()); + if (ObjCImageInfoItr != ObjCImageInfos.end()) { + // We've already registered an __objc_imageinfo section. Verify the + // content of this new section matches, then delete it. + if (ObjCImageInfoItr->second.first != Version) + return make_error<StringError>( + "ObjC version in " + G.getName() + + " does not match first registered version", + inconvertibleErrorCode()); + if (ObjCImageInfoItr->second.second != Flags) + return make_error<StringError>("ObjC flags in " + G.getName() + + " do not match first registered flags", + inconvertibleErrorCode()); + + // __objc_imageinfo is valid. Delete the block. + for (auto *S : ObjCImageInfo->symbols()) + G.removeDefinedSymbol(*S); + G.removeBlock(ObjCImageInfoBlock); + } else { + // We haven't registered an __objc_imageinfo section yet. Register and + // move on. The section should already be marked no-dead-strip. + ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags); + } + + return Error::success(); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/Mangling.cpp b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp new file mode 100644 index 0000000000000..606304741cf75 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Mangling.cpp @@ -0,0 +1,160 @@ +//===----------- Mangling.cpp -- Name Mangling Utilities for ORC ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Mangling.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Mangler.h" +#include "llvm/Object/MachO.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +MangleAndInterner::MangleAndInterner(ExecutionSession &ES, const DataLayout &DL) + : ES(ES), DL(DL) {} + +SymbolStringPtr MangleAndInterner::operator()(StringRef Name) { + std::string MangledName; + { + raw_string_ostream MangledNameStream(MangledName); + Mangler::getNameWithPrefix(MangledNameStream, Name, DL); + } + return ES.intern(MangledName); +} + +void IRSymbolMapper::add(ExecutionSession &ES, const ManglingOptions &MO, + ArrayRef<GlobalValue *> GVs, + SymbolFlagsMap &SymbolFlags, + SymbolNameToDefinitionMap *SymbolToDefinition) { + if (GVs.empty()) + return; + + MangleAndInterner Mangle(ES, GVs[0]->getParent()->getDataLayout()); + for (auto *G : GVs) { + assert(G && "GVs cannot contain null elements"); + if (!G->hasName() || G->isDeclaration() || G->hasLocalLinkage() || + G->hasAvailableExternallyLinkage() || G->hasAppendingLinkage()) + continue; + + if (G->isThreadLocal() && MO.EmulatedTLS) { + auto *GV = cast<GlobalVariable>(G); + + auto Flags = JITSymbolFlags::fromGlobalValue(*GV); + + auto EmuTLSV = Mangle(("__emutls_v." + GV->getName()).str()); + SymbolFlags[EmuTLSV] = Flags; + if (SymbolToDefinition) + (*SymbolToDefinition)[EmuTLSV] = GV; + + // If this GV has a non-zero initializer we'll need to emit an + // __emutls.t symbol too. + if (GV->hasInitializer()) { + const auto *InitVal = GV->getInitializer(); + + // Skip zero-initializers. + if (isa<ConstantAggregateZero>(InitVal)) + continue; + const auto *InitIntValue = dyn_cast<ConstantInt>(InitVal); + if (InitIntValue && InitIntValue->isZero()) + continue; + + auto EmuTLST = Mangle(("__emutls_t." + GV->getName()).str()); + SymbolFlags[EmuTLST] = Flags; + if (SymbolToDefinition) + (*SymbolToDefinition)[EmuTLST] = GV; + } + continue; + } + + // Otherwise we just need a normal linker mangling. + auto MangledName = Mangle(G->getName()); + SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(*G); + if (SymbolToDefinition) + (*SymbolToDefinition)[MangledName] = G; + } +} + +Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>> +getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); + + if (!Obj) + return Obj.takeError(); + + bool IsMachO = isa<object::MachOObjectFile>(Obj->get()); + + SymbolFlagsMap SymbolFlags; + for (auto &Sym : (*Obj)->symbols()) { + Expected<uint32_t> SymFlagsOrErr = Sym.getFlags(); + if (!SymFlagsOrErr) + // TODO: Test this error. + return SymFlagsOrErr.takeError(); + + // Skip symbols not defined in this object file. + if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined) + continue; + + // Skip symbols that are not global. + if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global)) + continue; + + // Skip symbols that have type SF_File. + if (auto SymType = Sym.getType()) { + if (*SymType == object::SymbolRef::ST_File) + continue; + } else + return SymType.takeError(); + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto InternedName = ES.intern(*Name); + auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); + if (!SymFlags) + return SymFlags.takeError(); + + // Strip the 'exported' flag from MachO linker-private symbols. + if (IsMachO && Name->startswith("l")) + *SymFlags &= ~JITSymbolFlags::Exported; + + SymbolFlags[InternedName] = std::move(*SymFlags); + } + + SymbolStringPtr InitSymbol; + + if (IsMachO) { + auto &MachOObj = cast<object::MachOObjectFile>(*Obj->get()); + for (auto &Sec : MachOObj.sections()) { + auto SecType = MachOObj.getSectionType(Sec); + if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) { + size_t Counter = 0; + while (true) { + std::string InitSymString; + raw_string_ostream(InitSymString) + << "$." << ObjBuffer.getBufferIdentifier() << ".__inits." + << Counter++; + InitSymbol = ES.intern(InitSymString); + if (SymbolFlags.count(InitSymbol)) + continue; + SymbolFlags[InitSymbol] = + JITSymbolFlags::MaterializationSideEffectsOnly; + break; + } + break; + } + } + } + + return std::make_pair(std::move(SymbolFlags), std::move(InitSymbol)); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 2572b7f4878d9..02066b458dfce 100644 --- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -50,9 +50,9 @@ public: void lookup(const LookupMap &Symbols, std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override { - JITDylibSearchOrder SearchOrder; - MR.getTargetJITDylib().withSearchOrderDo( - [&](const JITDylibSearchOrder &O) { SearchOrder = O; }); + JITDylibSearchOrder LinkOrder; + MR.getTargetJITDylib().withLinkOrderDo( + [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); auto &ES = Layer.getExecutionSession(); @@ -84,7 +84,13 @@ public: } }; - ES.lookup(LookupKind::Static, SearchOrder, std::move(LookupSet), + for (auto &KV : InternalNamedSymbolDeps) { + SymbolDependenceMap InternalDeps; + InternalDeps[&MR.getTargetJITDylib()] = std::move(KV.second); + MR.addDependencies(KV.first, InternalDeps); + } + + ES.lookup(LookupKind::Static, LinkOrder, std::move(LookupSet), SymbolState::Resolved, std::move(OnResolve), [this](const SymbolDependenceMap &Deps) { registerDependencies(Deps); @@ -138,6 +144,56 @@ public: if (!ExtraSymbolsToClaim.empty()) if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) return notifyFailed(std::move(Err)); + + { + + // Check that InternedResult matches up with MR.getSymbols(). + // This guards against faulty transformations / compilers / object caches. + + // First check that there aren't any missing symbols. + size_t NumMaterializationSideEffectsOnlySymbols = 0; + SymbolNameVector ExtraSymbols; + SymbolNameVector MissingSymbols; + for (auto &KV : MR.getSymbols()) { + + // If this is a materialization-side-effects only symbol then bump + // the counter and make sure it's *not* defined, otherwise make + // sure that it is defined. + if (KV.second.hasMaterializationSideEffectsOnly()) { + ++NumMaterializationSideEffectsOnlySymbols; + if (InternedResult.count(KV.first)) + ExtraSymbols.push_back(KV.first); + continue; + } else if (!InternedResult.count(KV.first)) + MissingSymbols.push_back(KV.first); + } + + // If there were missing symbols then report the error. + if (!MissingSymbols.empty()) { + ES.reportError(make_error<MissingSymbolDefinitions>( + G.getName(), std::move(MissingSymbols))); + MR.failMaterialization(); + return; + } + + // If there are more definitions than expected, add them to the + // ExtraSymbols vector. + if (InternedResult.size() > + MR.getSymbols().size() - NumMaterializationSideEffectsOnlySymbols) { + for (auto &KV : InternedResult) + if (!MR.getSymbols().count(KV.first)) + ExtraSymbols.push_back(KV.first); + } + + // If there were extra definitions then report the error. + if (!ExtraSymbols.empty()) { + ES.reportError(make_error<UnexpectedSymbolDefinitions>( + G.getName(), std::move(ExtraSymbols))); + MR.failMaterialization(); + return; + } + } + if (auto Err = MR.notifyResolved(InternedResult)) { Layer.getExecutionSession().reportError(std::move(Err)); MR.failMaterialization(); @@ -168,16 +224,22 @@ public: // link graph to build the symbol dependence graph. Config.PrePrunePasses.push_back( [this](LinkGraph &G) { return externalizeWeakAndCommonSymbols(G); }); - Config.PostPrunePasses.push_back( - [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); }); Layer.modifyPassConfig(MR, TT, Config); + Config.PostPrunePasses.push_back( + [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); }); + return Error::success(); } private: - using AnonToNamedDependenciesMap = DenseMap<const Symbol *, SymbolNameSet>; + struct LocalSymbolNamedDependencies { + SymbolNameSet Internal, External; + }; + + using LocalSymbolNamedDependenciesMap = + DenseMap<const Symbol *, LocalSymbolNamedDependencies>; Error externalizeWeakAndCommonSymbols(LinkGraph &G) { auto &ES = Layer.getExecutionSession(); @@ -206,30 +268,69 @@ private: Error computeNamedSymbolDependencies(LinkGraph &G) { auto &ES = MR.getTargetJITDylib().getExecutionSession(); - auto AnonDeps = computeAnonDeps(G); + auto LocalDeps = computeLocalDeps(G); + // Compute dependencies for symbols defined in the JITLink graph. for (auto *Sym : G.defined_symbols()) { - // Skip anonymous and non-global atoms: we do not need dependencies for - // these. + // Skip local symbols: we do not track dependencies for these. if (Sym->getScope() == Scope::Local) continue; + assert(Sym->hasName() && + "Defined non-local jitlink::Symbol should have a name"); - auto SymName = ES.intern(Sym->getName()); - SymbolNameSet &SymDeps = NamedSymbolDeps[SymName]; + SymbolNameSet ExternalSymDeps, InternalSymDeps; + // Find internal and external named symbol dependencies. for (auto &E : Sym->getBlock().edges()) { auto &TargetSym = E.getTarget(); - if (TargetSym.getScope() != Scope::Local) - SymDeps.insert(ES.intern(TargetSym.getName())); - else { + if (TargetSym.getScope() != Scope::Local) { + if (TargetSym.isExternal()) + ExternalSymDeps.insert(ES.intern(TargetSym.getName())); + else if (&TargetSym != Sym) + InternalSymDeps.insert(ES.intern(TargetSym.getName())); + } else { assert(TargetSym.isDefined() && - "Anonymous/local symbols must be defined"); - auto I = AnonDeps.find(&TargetSym); - if (I != AnonDeps.end()) - for (auto &S : I->second) - SymDeps.insert(S); + "local symbols must be defined"); + auto I = LocalDeps.find(&TargetSym); + if (I != LocalDeps.end()) { + for (auto &S : I->second.External) + ExternalSymDeps.insert(S); + for (auto &S : I->second.Internal) + InternalSymDeps.insert(S); + } + } + } + + if (ExternalSymDeps.empty() && InternalSymDeps.empty()) + continue; + + auto SymName = ES.intern(Sym->getName()); + if (!ExternalSymDeps.empty()) + ExternalNamedSymbolDeps[SymName] = std::move(ExternalSymDeps); + if (!InternalSymDeps.empty()) + InternalNamedSymbolDeps[SymName] = std::move(InternalSymDeps); + } + + for (auto &P : Layer.Plugins) { + auto SyntheticLocalDeps = P->getSyntheticSymbolLocalDependencies(MR); + if (SyntheticLocalDeps.empty()) + continue; + + for (auto &KV : SyntheticLocalDeps) { + auto &Name = KV.first; + auto &LocalDepsForName = KV.second; + for (auto *Local : LocalDepsForName) { + assert(Local->getScope() == Scope::Local && + "Dependence on non-local symbol"); + auto LocalNamedDepsItr = LocalDeps.find(Local); + if (LocalNamedDepsItr == LocalDeps.end()) + continue; + for (auto &S : LocalNamedDepsItr->second.Internal) + InternalNamedSymbolDeps[Name].insert(S); + for (auto &S : LocalNamedDepsItr->second.External) + ExternalNamedSymbolDeps[Name].insert(S); } } } @@ -237,68 +338,85 @@ private: return Error::success(); } - AnonToNamedDependenciesMap computeAnonDeps(LinkGraph &G) { + LocalSymbolNamedDependenciesMap computeLocalDeps(LinkGraph &G) { + DenseMap<jitlink::Symbol *, DenseSet<jitlink::Symbol *>> DepMap; - auto &ES = MR.getTargetJITDylib().getExecutionSession(); - AnonToNamedDependenciesMap DepMap; - - // For all anonymous symbols: + // For all local symbols: // (1) Add their named dependencies. // (2) Add them to the worklist for further iteration if they have any - // depend on any other anonymous symbols. + // depend on any other local symbols. struct WorklistEntry { - WorklistEntry(Symbol *Sym, DenseSet<Symbol *> SymAnonDeps) - : Sym(Sym), SymAnonDeps(std::move(SymAnonDeps)) {} + WorklistEntry(Symbol *Sym, DenseSet<Symbol *> LocalDeps) + : Sym(Sym), LocalDeps(std::move(LocalDeps)) {} Symbol *Sym = nullptr; - DenseSet<Symbol *> SymAnonDeps; + DenseSet<Symbol *> LocalDeps; }; std::vector<WorklistEntry> Worklist; for (auto *Sym : G.defined_symbols()) - if (!Sym->hasName()) { + if (Sym->getScope() == Scope::Local) { auto &SymNamedDeps = DepMap[Sym]; - DenseSet<Symbol *> SymAnonDeps; + DenseSet<Symbol *> LocalDeps; for (auto &E : Sym->getBlock().edges()) { auto &TargetSym = E.getTarget(); - if (TargetSym.hasName()) - SymNamedDeps.insert(ES.intern(TargetSym.getName())); + if (TargetSym.getScope() != Scope::Local) + SymNamedDeps.insert(&TargetSym); else { assert(TargetSym.isDefined() && - "Anonymous symbols must be defined"); - SymAnonDeps.insert(&TargetSym); + "local symbols must be defined"); + LocalDeps.insert(&TargetSym); } } - if (!SymAnonDeps.empty()) - Worklist.push_back(WorklistEntry(Sym, std::move(SymAnonDeps))); + if (!LocalDeps.empty()) + Worklist.push_back(WorklistEntry(Sym, std::move(LocalDeps))); } - // Loop over all anonymous symbols with anonymous dependencies, propagating - // their respective *named* dependencies. Iterate until we hit a stable + // Loop over all local symbols with local dependencies, propagating + // their respective non-local dependencies. Iterate until we hit a stable // state. bool Changed; do { Changed = false; for (auto &WLEntry : Worklist) { auto *Sym = WLEntry.Sym; - auto &SymNamedDeps = DepMap[Sym]; - auto &SymAnonDeps = WLEntry.SymAnonDeps; + auto &NamedDeps = DepMap[Sym]; + auto &LocalDeps = WLEntry.LocalDeps; - for (auto *TargetSym : SymAnonDeps) { + for (auto *TargetSym : LocalDeps) { auto I = DepMap.find(TargetSym); if (I != DepMap.end()) for (const auto &S : I->second) - Changed |= SymNamedDeps.insert(S).second; + Changed |= NamedDeps.insert(S).second; } } } while (Changed); - return DepMap; + // Intern the results to produce a mapping of jitlink::Symbol* to internal + // and external symbol names. + auto &ES = Layer.getExecutionSession(); + LocalSymbolNamedDependenciesMap Result; + for (auto &KV : DepMap) { + auto *Local = KV.first; + assert(Local->getScope() == Scope::Local && + "DepMap keys should all be local symbols"); + auto &LocalNamedDeps = Result[Local]; + for (auto *Named : KV.second) { + assert(Named->getScope() != Scope::Local && + "DepMap values should all be non-local symbol sets"); + if (Named->isExternal()) + LocalNamedDeps.External.insert(ES.intern(Named->getName())); + else + LocalNamedDeps.Internal.insert(ES.intern(Named->getName())); + } + } + + return Result; } void registerDependencies(const SymbolDependenceMap &QueryDeps) { - for (auto &NamedDepsEntry : NamedSymbolDeps) { + for (auto &NamedDepsEntry : ExternalNamedSymbolDeps) { auto &Name = NamedDepsEntry.first; auto &NameDeps = NamedDepsEntry.second; SymbolDependenceMap SymbolDeps; @@ -323,7 +441,8 @@ private: ObjectLinkingLayer &Layer; MaterializationResponsibility MR; std::unique_ptr<MemoryBuffer> ObjBuffer; - DenseMap<SymbolStringPtr, SymbolNameSet> NamedSymbolDeps; + DenseMap<SymbolStringPtr, SymbolNameSet> ExternalNamedSymbolDeps; + DenseMap<SymbolStringPtr, SymbolNameSet> InternalNamedSymbolDeps; }; ObjectLinkingLayer::Plugin::~Plugin() {} @@ -426,18 +545,21 @@ EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( void EHFrameRegistrationPlugin::modifyPassConfig( MaterializationResponsibility &MR, const Triple &TT, PassConfiguration &PassConfig) { - assert(!InProcessLinks.count(&MR) && "Link for MR already being tracked?"); - PassConfig.PostFixupPasses.push_back( - createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr, - size_t Size) { - if (Addr) - InProcessLinks[&MR] = { Addr, Size }; + PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass( + TT, [this, &MR](JITTargetAddress Addr, size_t Size) { + if (Addr) { + std::lock_guard<std::mutex> Lock(EHFramePluginMutex); + assert(!InProcessLinks.count(&MR) && + "Link for MR already being tracked?"); + InProcessLinks[&MR] = {Addr, Size}; + } })); } Error EHFrameRegistrationPlugin::notifyEmitted( MaterializationResponsibility &MR) { + std::lock_guard<std::mutex> Lock(EHFramePluginMutex); auto EHFrameRangeItr = InProcessLinks.find(&MR); if (EHFrameRangeItr == InProcessLinks.end()) @@ -457,6 +579,8 @@ Error EHFrameRegistrationPlugin::notifyEmitted( } Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) { + std::lock_guard<std::mutex> Lock(EHFramePluginMutex); + auto EHFrameRangeItr = TrackedEHFrameRanges.find(K); if (EHFrameRangeItr == TrackedEHFrameRanges.end()) return Error::success(); @@ -470,6 +594,7 @@ Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) { } Error EHFrameRegistrationPlugin::notifyRemovingAllModules() { + std::lock_guard<std::mutex> Lock(EHFramePluginMutex); std::vector<EHFrameRange> EHFrameRanges = std::move(UntrackedEHFrameRanges); diff --git a/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp index 8ed23de419d1e..18b3c5e12b1c2 100644 --- a/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp +++ b/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -7,13 +7,46 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" +#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Process.h" +#include "llvm/Support/raw_ostream.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; + +template <typename ORCABI> +bool stubAndPointerRangesOk(JITTargetAddress StubBlockAddr, + JITTargetAddress PointerBlockAddr, + unsigned NumStubs) { + constexpr unsigned MaxDisp = ORCABI::StubToPointerMaxDisplacement; + JITTargetAddress FirstStub = StubBlockAddr; + JITTargetAddress LastStub = FirstStub + ((NumStubs - 1) * ORCABI::StubSize); + JITTargetAddress FirstPointer = PointerBlockAddr; + JITTargetAddress LastPointer = + FirstPointer + ((NumStubs - 1) * ORCABI::StubSize); + + if (FirstStub < FirstPointer) { + if (LastStub >= FirstPointer) + return false; // Ranges overlap. + return (FirstPointer - FirstStub <= MaxDisp) && + (LastPointer - LastStub <= MaxDisp); // out-of-range. + } + + if (LastPointer >= FirstStub) + return false; // Ranges overlap. + + return (FirstStub - FirstPointer <= MaxDisp) && + (LastStub - LastPointer <= MaxDisp); +} namespace llvm { namespace orc { -void OrcAArch64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, - void *CallbackMgr) { +void OrcAArch64::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { const uint32_t ResolverCode[] = { // resolver_entry: @@ -48,7 +81,7 @@ void OrcAArch64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, 0xadbf17e4, // 0x070: stp q4, q5, [sp, #-32]! 0xadbf0fe2, // 0x074: stp q2, q3, [sp, #-32]! 0xadbf07e0, // 0x078: stp q0, q1, [sp, #-32]! - 0x580004e0, // 0x07c: ldr x0, Lcallbackmgr + 0x580004e0, // 0x07c: ldr x0, Lreentry_ctx_ptr 0xaa1e03e1, // 0x080: mov x1, x30 0xd1003021, // 0x084: sub x1, x1, #12 0x58000442, // 0x088: ldr x2, Lreentry_fn_ptr @@ -87,43 +120,47 @@ void OrcAArch64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, 0xd65f0220, // 0x10c: ret x17 0x01234567, // 0x110: Lreentry_fn_ptr: 0xdeadbeef, // 0x114: .quad 0 - 0x98765432, // 0x118: Lcallbackmgr: + 0x98765432, // 0x118: Lreentry_ctx_ptr: 0xcafef00d // 0x11c: .quad 0 }; const unsigned ReentryFnAddrOffset = 0x110; - const unsigned CallbackMgrAddrOffset = 0x118; + const unsigned ReentryCtxAddrOffset = 0x118; - memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); - memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); - memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, - sizeof(CallbackMgr)); + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnAddr, + sizeof(uint64_t)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxAddr, + sizeof(uint64_t)); } -void OrcAArch64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, +void OrcAArch64::writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines) { unsigned OffsetToPtr = alignTo(NumTrampolines * TrampolineSize, 8); - memcpy(TrampolineMem + OffsetToPtr, &ResolverAddr, sizeof(void *)); + memcpy(TrampolineBlockWorkingMem + OffsetToPtr, &ResolverAddr, + sizeof(uint64_t)); // OffsetToPtr is actually the offset from the PC for the 2nd instruction, so // subtract 32-bits. OffsetToPtr -= 4; - uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); + uint32_t *Trampolines = + reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) { Trampolines[3 * I + 0] = 0xaa1e03f1; // mov x17, x30 Trampolines[3 * I + 1] = 0x58000010 | (OffsetToPtr << 3); // adr x16, Lptr Trampolines[3 * I + 2] = 0xd63f0200; // blr x16 } - } -Error OrcAArch64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, - unsigned MinStubs, - void *InitialPtrVal) { +void OrcAArch64::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs @@ -144,68 +181,41 @@ Error OrcAArch64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, // // ... - const unsigned StubSize = IndirectStubsInfo::StubSize; - - // Emit at least MinStubs, rounded up to fill the pages allocated. - static const unsigned PageSize = sys::Process::getPageSizeEstimate(); - unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; - unsigned NumStubs = (NumPages * PageSize) / StubSize; - - // Allocate memory for stubs and pointers in one call. - std::error_code EC; - auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( - 2 * NumPages * PageSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); - - if (EC) - return errorCodeToError(EC); - - // Create separate MemoryBlocks representing the stubs and pointers. - sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); - sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + - NumPages * PageSize, - NumPages * PageSize); - - // Populate the stubs page stubs and mark it executable. - uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlock.base()); - uint64_t PtrOffsetField = static_cast<uint64_t>(NumPages * PageSize) - << 3; + static_assert(StubSize == PointerSize, + "Pointer and stub size must match for algorithm below"); + assert(stubAndPointerRangesOk<OrcAArch64>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); + uint64_t PtrDisplacement = + PointersBlockTargetAddress - StubsBlockTargetAddress; + uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); + uint64_t PtrOffsetField = PtrDisplacement << 3; for (unsigned I = 0; I < NumStubs; ++I) Stub[I] = 0xd61f020058000010 | PtrOffsetField; - - if (auto EC = sys::Memory::protectMappedMemory( - StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) - return errorCodeToError(EC); - - // Initialize all pointers to point at FailureAddress. - void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); - for (unsigned I = 0; I < NumStubs; ++I) - Ptr[I] = InitialPtrVal; - - StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); - - return Error::success(); } -void OrcX86_64_Base::writeTrampolines(uint8_t *TrampolineMem, - void *ResolverAddr, - unsigned NumTrampolines) { +void OrcX86_64_Base::writeTrampolines( + char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines) { unsigned OffsetToPtr = NumTrampolines * TrampolineSize; - memcpy(TrampolineMem + OffsetToPtr, &ResolverAddr, sizeof(void *)); + memcpy(TrampolineBlockWorkingMem + OffsetToPtr, &ResolverAddr, + sizeof(uint64_t)); - uint64_t *Trampolines = reinterpret_cast<uint64_t *>(TrampolineMem); + uint64_t *Trampolines = + reinterpret_cast<uint64_t *>(TrampolineBlockWorkingMem); uint64_t CallIndirPCRel = 0xf1c40000000015ff; for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) Trampolines[I] = CallIndirPCRel | ((OffsetToPtr - 6) << 16); } -Error OrcX86_64_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, - unsigned MinStubs, - void *InitialPtrVal) { +void OrcX86_64_Base::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs @@ -226,52 +236,28 @@ Error OrcX86_64_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, // // ... - const unsigned StubSize = IndirectStubsInfo::StubSize; - - // Emit at least MinStubs, rounded up to fill the pages allocated. - static const unsigned PageSize = sys::Process::getPageSizeEstimate(); - unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; - unsigned NumStubs = (NumPages * PageSize) / StubSize; - - // Allocate memory for stubs and pointers in one call. - std::error_code EC; - auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( - 2 * NumPages * PageSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); - - if (EC) - return errorCodeToError(EC); - - // Create separate MemoryBlocks representing the stubs and pointers. - sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); - sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + - NumPages * PageSize, - NumPages * PageSize); - // Populate the stubs page stubs and mark it executable. - uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlock.base()); - uint64_t PtrOffsetField = static_cast<uint64_t>(NumPages * PageSize - 6) - << 16; + static_assert(StubSize == PointerSize, + "Pointer and stub size must match for algorithm below"); + assert(stubAndPointerRangesOk<OrcX86_64_Base>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); + uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); + uint64_t PtrOffsetField = + (PointersBlockTargetAddress - StubsBlockTargetAddress - 6) << 16; for (unsigned I = 0; I < NumStubs; ++I) Stub[I] = 0xF1C40000000025ff | PtrOffsetField; - - if (auto EC = sys::Memory::protectMappedMemory( - StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) - return errorCodeToError(EC); - - // Initialize all pointers to point at FailureAddress. - void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); - for (unsigned I = 0; I < NumStubs; ++I) - Ptr[I] = InitialPtrVal; - - StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); - - return Error::success(); } -void OrcX86_64_SysV::writeResolverCode(uint8_t *ResolverMem, - JITReentryFn ReentryFn, - void *CallbackMgr) { +void OrcX86_64_SysV::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + + LLVM_DEBUG({ + dbgs() << "Writing resolver code to " + << formatv("{0:x16}", ResolverTargetAddress) << "\n"; + }); const uint8_t ResolverCode[] = { // resolver_entry: @@ -295,7 +281,7 @@ void OrcX86_64_SysV::writeResolverCode(uint8_t *ResolverMem, 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) 0x48, 0xbf, // 0x26: movabsq <CBMgr>, %rdi - // 0x28: Callback manager addr. + // 0x28: JIT re-entry ctx addr. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x8b, 0x75, 0x08, // 0x30: movq 8(%rbp), %rsi @@ -325,23 +311,26 @@ void OrcX86_64_SysV::writeResolverCode(uint8_t *ResolverMem, 0x58, // 0x69: popq %rax 0x5d, // 0x6a: popq %rbp 0xc3, // 0x6b: retq - }; + }; const unsigned ReentryFnAddrOffset = 0x3a; - const unsigned CallbackMgrAddrOffset = 0x28; + const unsigned ReentryCtxAddrOffset = 0x28; - memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); - memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); - memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, - sizeof(CallbackMgr)); + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnAddr, + sizeof(uint64_t)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxAddr, + sizeof(uint64_t)); } -void OrcX86_64_Win32::writeResolverCode(uint8_t *ResolverMem, - JITReentryFn ReentryFn, - void *CallbackMgr) { +void OrcX86_64_Win32::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { - // resolverCode is similar to OrcX86_64 with differences specific to windows x64 calling convention: - // arguments go into rcx, rdx and come in reverse order, shadow space allocation on stack + // resolverCode is similar to OrcX86_64 with differences specific to windows + // x64 calling convention: arguments go into rcx, rdx and come in reverse + // order, shadow space allocation on stack const uint8_t ResolverCode[] = { // resolver_entry: 0x55, // 0x00: pushq %rbp @@ -364,7 +353,7 @@ void OrcX86_64_Win32::writeResolverCode(uint8_t *ResolverMem, 0x48, 0x0f, 0xae, 0x04, 0x24, // 0x21: fxsave64 (%rsp) 0x48, 0xb9, // 0x26: movabsq <CBMgr>, %rcx - // 0x28: Callback manager addr. + // 0x28: JIT re-entry ctx addr. 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0x8B, 0x55, 0x08, // 0x30: mov rdx, [rbp+0x8] @@ -402,18 +391,23 @@ void OrcX86_64_Win32::writeResolverCode(uint8_t *ResolverMem, 0xc3, // 0x73: retq }; - const unsigned ReentryFnAddrOffset = 0x3a; - const unsigned CallbackMgrAddrOffset = 0x28; + const unsigned ReentryCtxAddrOffset = 0x28; - memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); - memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); - memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, - sizeof(CallbackMgr)); + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnAddr, + sizeof(uint64_t)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxAddr, + sizeof(uint64_t)); } -void OrcI386::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, - void *CallbackMgr) { +void OrcI386::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + + assert((ReentryFnAddr >> 32) == 0 && "ReentryFnAddr out of range"); + assert((ReentryCtxAddr >> 32) == 0 && "ReentryCtxAddr out of range"); const uint8_t ResolverCode[] = { // resolver_entry: @@ -451,29 +445,37 @@ void OrcI386::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, }; const unsigned ReentryFnAddrOffset = 0x2a; - const unsigned CallbackMgrAddrOffset = 0x25; + const unsigned ReentryCtxAddrOffset = 0x25; - memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); - memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryFn, sizeof(ReentryFn)); - memcpy(ResolverMem + CallbackMgrAddrOffset, &CallbackMgr, - sizeof(CallbackMgr)); + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnAddr, + sizeof(uint32_t)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxAddr, + sizeof(uint32_t)); } -void OrcI386::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, +void OrcI386::writeTrampolines(char *TrampolineWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines) { + assert((ResolverAddr >> 32) == 0 && "ResolverAddr out of range"); uint64_t CallRelImm = 0xF1C4C400000000e8; - uint64_t Resolver = reinterpret_cast<uint64_t>(ResolverAddr); - uint64_t ResolverRel = - Resolver - reinterpret_cast<uint64_t>(TrampolineMem) - 5; + uint64_t ResolverRel = ResolverAddr - TrampolineBlockTargetAddress - 5; - uint64_t *Trampolines = reinterpret_cast<uint64_t *>(TrampolineMem); + uint64_t *Trampolines = reinterpret_cast<uint64_t *>(TrampolineWorkingMem); for (unsigned I = 0; I < NumTrampolines; ++I, ResolverRel -= TrampolineSize) Trampolines[I] = CallRelImm | (ResolverRel << 8); } -Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, - unsigned MinStubs, void *InitialPtrVal) { +void OrcI386::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + assert((StubsBlockTargetAddress >> 32) == 0 && + "StubsBlockTargetAddress is out of range"); + assert((PointersBlockTargetAddress >> 32) == 0 && + "PointersBlockTargetAddress is out of range"); + // Stub format is: // // .section __orc_stubs @@ -494,51 +496,21 @@ Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, // // ... - const unsigned StubSize = IndirectStubsInfo::StubSize; + assert(stubAndPointerRangesOk<OrcI386>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); - // Emit at least MinStubs, rounded up to fill the pages allocated. - static const unsigned PageSize = sys::Process::getPageSizeEstimate(); - unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; - unsigned NumStubs = (NumPages * PageSize) / StubSize; - - // Allocate memory for stubs and pointers in one call. - std::error_code EC; - auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( - 2 * NumPages * PageSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); - - if (EC) - return errorCodeToError(EC); - - // Create separate MemoryBlocks representing the stubs and pointers. - sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); - sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + - NumPages * PageSize, - NumPages * PageSize); - - // Populate the stubs page stubs and mark it executable. - uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlock.base()); - uint64_t PtrAddr = reinterpret_cast<uint64_t>(PtrsBlock.base()); + uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); + uint64_t PtrAddr = PointersBlockTargetAddress; for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 4) Stub[I] = 0xF1C40000000025ff | (PtrAddr << 16); - - if (auto EC = sys::Memory::protectMappedMemory( - StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) - return errorCodeToError(EC); - - // Initialize all pointers to point at FailureAddress. - void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); - for (unsigned I = 0; I < NumStubs; ++I) - Ptr[I] = InitialPtrVal; - - StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); - - return Error::success(); } -void OrcMips32_Base::writeResolverCode(uint8_t *ResolverMem, - JITReentryFn ReentryFn, - void *CallbackMgr, bool isBigEndian) { +void OrcMips32_Base::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr, + bool isBigEndian) { const uint32_t ResolverCode[] = { // resolver_entry: @@ -570,9 +542,9 @@ void OrcMips32_Base::writeResolverCode(uint8_t *ResolverMem, 0xafbe0060, // 0x64: sw $fp,96($sp) 0xafbf0064, // 0x68: sw $ra,100($sp) - // Callback manager addr. - 0x00000000, // 0x6c: lui $a0,callbackmgr - 0x00000000, // 0x70: addiu $a0,$a0,callbackmgr + // JIT re-entry ctx addr. + 0x00000000, // 0x6c: lui $a0,ctx + 0x00000000, // 0x70: addiu $a0,$a0,ctx 0x03e02825, // 0x74: move $a1, $ra 0x24a5ffec, // 0x78: addiu $a1,$a1,-20 @@ -614,50 +586,63 @@ void OrcMips32_Base::writeResolverCode(uint8_t *ResolverMem, }; const unsigned ReentryFnAddrOffset = 0x7c; // JIT re-entry fn addr lui - const unsigned CallbackMgrAddrOffset = 0x6c; // Callback manager addr lui + const unsigned ReentryCtxAddrOffset = 0x6c; // JIT re-entry context addr lui const unsigned Offsett = 0xf8; - memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); // Depending on endian return value will be in v0 or v1. uint32_t MoveVxT9 = isBigEndian ? 0x0060c825 : 0x0040c825; - memcpy(ResolverMem + Offsett, &MoveVxT9, sizeof(MoveVxT9)); - - uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr); - uint32_t CallMgrLUi = 0x3c040000 | (((CallMgrAddr + 0x8000) >> 16) & 0xFFFF); - uint32_t CallMgrADDiu = 0x24840000 | ((CallMgrAddr) & 0xFFFF); - memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, sizeof(CallMgrLUi)); - memcpy(ResolverMem + CallbackMgrAddrOffset + 4, &CallMgrADDiu, - sizeof(CallMgrADDiu)); - - uint64_t ReentryAddr = reinterpret_cast<uint64_t>(ReentryFn); - uint32_t ReentryLUi = 0x3c190000 | (((ReentryAddr + 0x8000) >> 16) & 0xFFFF); - uint32_t ReentryADDiu = 0x27390000 | ((ReentryAddr) & 0xFFFF); - memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, sizeof(ReentryLUi)); - memcpy(ResolverMem + ReentryFnAddrOffset + 4, &ReentryADDiu, - sizeof(ReentryADDiu)); + memcpy(ResolverWorkingMem + Offsett, &MoveVxT9, sizeof(MoveVxT9)); + + uint32_t ReentryCtxLUi = + 0x3c040000 | (((ReentryCtxAddr + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryCtxADDiu = 0x24840000 | ((ReentryCtxAddr)&0xFFFF); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxLUi, + sizeof(ReentryCtxLUi)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset + 4, &ReentryCtxADDiu, + sizeof(ReentryCtxADDiu)); + + uint32_t ReentryFnLUi = + 0x3c190000 | (((ReentryFnAddr + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryFnADDiu = 0x27390000 | ((ReentryFnAddr)&0xFFFF); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnLUi, + sizeof(ReentryFnLUi)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset + 4, &ReentryFnADDiu, + sizeof(ReentryFnADDiu)); } -void OrcMips32_Base::writeTrampolines(uint8_t *TrampolineMem, - void *ResolverAddr, - unsigned NumTrampolines) { +void OrcMips32_Base::writeTrampolines( + char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines) { - uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); - uint64_t ResolveAddr = reinterpret_cast<uint64_t>(ResolverAddr); - uint32_t RHiAddr = ((ResolveAddr + 0x8000) >> 16); + assert((ResolverAddr >> 32) == 0 && "ResolverAddr out of range"); + + uint32_t *Trampolines = + reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); + uint32_t RHiAddr = ((ResolverAddr + 0x8000) >> 16); for (unsigned I = 0; I < NumTrampolines; ++I) { - Trampolines[5 * I + 0] = 0x03e0c025; // move $t8,$ra - Trampolines[5 * I + 1] = 0x3c190000 | (RHiAddr & 0xFFFF); // lui $t9,resolveAddr - Trampolines[5 * I + 2] = 0x27390000 | (ResolveAddr & 0xFFFF); // addiu $t9,$t9,resolveAddr - Trampolines[5 * I + 3] = 0x0320f809; // jalr $t9 - Trampolines[5 * I + 4] = 0x00000000; // nop + // move $t8,$ra + // lui $t9,ResolverAddr + // addiu $t9,$t9,ResolverAddr + // jalr $t9 + // nop + Trampolines[5 * I + 0] = 0x03e0c025; + Trampolines[5 * I + 1] = 0x3c190000 | (RHiAddr & 0xFFFF); + Trampolines[5 * I + 2] = 0x27390000 | (ResolverAddr & 0xFFFF); + Trampolines[5 * I + 3] = 0x0320f809; + Trampolines[5 * I + 4] = 0x00000000; } } -Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, - unsigned MinStubs, - void *InitialPtrVal) { +void OrcMips32_Base::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + assert((StubsBlockTargetAddress >> 32) == 0 && + "InitialPtrVal is out of range"); + // Stub format is: // // .section __orc_stubs @@ -678,33 +663,15 @@ Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, // ptr2: // .word 0x0 // - // ... - - const unsigned StubSize = IndirectStubsInfo::StubSize; - - // Emit at least MinStubs, rounded up to fill the pages allocated. - static const unsigned PageSize = sys::Process::getPageSizeEstimate(); - unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; - unsigned NumStubs = (NumPages * PageSize) / StubSize; + // i.. - // Allocate memory for stubs and pointers in one call. - std::error_code EC; - auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( - 2 * NumPages * PageSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); - - if (EC) - return errorCodeToError(EC); - - // Create separate MemoryBlocks representing the stubs and pointers. - sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); - sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + - NumPages * PageSize, - NumPages * PageSize); + assert(stubAndPointerRangesOk<OrcAArch64>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); // Populate the stubs page stubs and mark it executable. - uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlock.base()); - uint64_t PtrAddr = reinterpret_cast<uint64_t>(Stub) + NumPages * PageSize; + uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlockWorkingMem); + uint64_t PtrAddr = PointersBlockTargetAddress; for (unsigned I = 0; I < NumStubs; ++I) { uint32_t HiAddr = ((PtrAddr + 0x8000) >> 16); @@ -714,26 +681,15 @@ Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, Stub[4 * I + 3] = 0x00000000; // nop PtrAddr += 4; } - - if (auto EC = sys::Memory::protectMappedMemory( - StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) - return errorCodeToError(EC); - - // Initialize all pointers to point at FailureAddress. - void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); - for (unsigned I = 0; I < NumStubs; ++I) - Ptr[I] = InitialPtrVal; - - StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); - - return Error::success(); } -void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, - void *CallbackMgr) { +void OrcMips64::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { const uint32_t ResolverCode[] = { - //resolver_entry: + //resolver_entry: 0x67bdff30, // 0x00: daddiu $sp,$sp,-208 0xffa20000, // 0x04: sd v0,0(sp) 0xffa30008, // 0x08: sd v1,8(sp) @@ -762,13 +718,13 @@ void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, 0xffbe00c0, // 0x64: sd fp,192(sp) 0xffbf00c8, // 0x68: sd ra,200(sp) - // Callback manager addr. - 0x00000000, // 0x6c: lui $a0,heighest(callbackmgr) - 0x00000000, // 0x70: daddiu $a0,$a0,heigher(callbackmgr) + // JIT re-entry ctx addr. + 0x00000000, // 0x6c: lui $a0,heighest(ctx) + 0x00000000, // 0x70: daddiu $a0,$a0,heigher(ctx) 0x00000000, // 0x74: dsll $a0,$a0,16 - 0x00000000, // 0x78: daddiu $a0,$a0,hi(callbackmgr) + 0x00000000, // 0x78: daddiu $a0,$a0,hi(ctx) 0x00000000, // 0x7c: dsll $a0,$a0,16 - 0x00000000, // 0x80: daddiu $a0,$a0,lo(callbackmgr) + 0x00000000, // 0x80: daddiu $a0,$a0,lo(ctx) 0x03e02825, // 0x84: move $a1, $ra 0x64a5ffdc, // 0x88: daddiu $a1,$a1,-36 @@ -814,73 +770,73 @@ void OrcMips64::writeResolverCode(uint8_t *ResolverMem, JITReentryFn ReentryFn, }; const unsigned ReentryFnAddrOffset = 0x8c; // JIT re-entry fn addr lui - const unsigned CallbackMgrAddrOffset = 0x6c; // Callback manager addr lui - - memcpy(ResolverMem, ResolverCode, sizeof(ResolverCode)); - - uint64_t CallMgrAddr = reinterpret_cast<uint64_t>(CallbackMgr); - - uint32_t CallMgrLUi = - 0x3c040000 | (((CallMgrAddr + 0x800080008000) >> 48) & 0xFFFF); - uint32_t CallMgrDADDiu = - 0x64840000 | (((CallMgrAddr + 0x80008000) >> 32) & 0xFFFF); - uint32_t CallMgrDSLL = 0x00042438; - uint32_t CallMgrDADDiu2 = - 0x64840000 | ((((CallMgrAddr + 0x8000) >> 16) & 0xFFFF)); - uint32_t CallMgrDSLL2 = 0x00042438; - uint32_t CallMgrDADDiu3 = 0x64840000 | ((CallMgrAddr)&0xFFFF); - - memcpy(ResolverMem + CallbackMgrAddrOffset, &CallMgrLUi, sizeof(CallMgrLUi)); - memcpy(ResolverMem + (CallbackMgrAddrOffset + 4), &CallMgrDADDiu, - sizeof(CallMgrDADDiu)); - memcpy(ResolverMem + (CallbackMgrAddrOffset + 8), &CallMgrDSLL, - sizeof(CallMgrDSLL)); - memcpy(ResolverMem + (CallbackMgrAddrOffset + 12), &CallMgrDADDiu2, - sizeof(CallMgrDADDiu2)); - memcpy(ResolverMem + (CallbackMgrAddrOffset + 16), &CallMgrDSLL2, - sizeof(CallMgrDSLL2)); - memcpy(ResolverMem + (CallbackMgrAddrOffset + 20), &CallMgrDADDiu3, - sizeof(CallMgrDADDiu3)); - - uint64_t ReentryAddr = reinterpret_cast<uint64_t>(ReentryFn); - - uint32_t ReentryLUi = - 0x3c190000 | (((ReentryAddr + 0x800080008000) >> 48) & 0xFFFF); - - uint32_t ReentryDADDiu = - 0x67390000 | (((ReentryAddr + 0x80008000) >> 32) & 0xFFFF); - - uint32_t ReentryDSLL = 0x0019cc38; - - uint32_t ReentryDADDiu2 = - 0x67390000 | (((ReentryAddr + 0x8000) >> 16) & 0xFFFF); - - uint32_t ReentryDSLL2 = 0x0019cc38; - - uint32_t ReentryDADDiu3 = 0x67390000 | ((ReentryAddr)&0xFFFF); - - memcpy(ResolverMem + ReentryFnAddrOffset, &ReentryLUi, sizeof(ReentryLUi)); - memcpy(ResolverMem + (ReentryFnAddrOffset + 4), &ReentryDADDiu, - sizeof(ReentryDADDiu)); - memcpy(ResolverMem + (ReentryFnAddrOffset + 8), &ReentryDSLL, - sizeof(ReentryDSLL)); - memcpy(ResolverMem + (ReentryFnAddrOffset + 12), &ReentryDADDiu2, - sizeof(ReentryDADDiu2)); - memcpy(ResolverMem + (ReentryFnAddrOffset + 16), &ReentryDSLL2, - sizeof(ReentryDSLL2)); - memcpy(ResolverMem + (ReentryFnAddrOffset + 20), &ReentryDADDiu3, - sizeof(ReentryDADDiu3)); + const unsigned ReentryCtxAddrOffset = 0x6c; // JIT re-entry ctx addr lui + + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + + uint32_t ReentryCtxLUi = + 0x3c040000 | (((ReentryCtxAddr + 0x800080008000) >> 48) & 0xFFFF); + uint32_t ReentryCtxDADDiu = + 0x64840000 | (((ReentryCtxAddr + 0x80008000) >> 32) & 0xFFFF); + uint32_t ReentryCtxDSLL = 0x00042438; + uint32_t ReentryCtxDADDiu2 = + 0x64840000 | ((((ReentryCtxAddr + 0x8000) >> 16) & 0xFFFF)); + uint32_t ReentryCtxDSLL2 = 0x00042438; + uint32_t ReentryCtxDADDiu3 = 0x64840000 | ((ReentryCtxAddr)&0xFFFF); + + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxLUi, + sizeof(ReentryCtxLUi)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 4), &ReentryCtxDADDiu, + sizeof(ReentryCtxDADDiu)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 8), &ReentryCtxDSLL, + sizeof(ReentryCtxDSLL)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 12), &ReentryCtxDADDiu2, + sizeof(ReentryCtxDADDiu2)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 16), &ReentryCtxDSLL2, + sizeof(ReentryCtxDSLL2)); + memcpy(ResolverWorkingMem + (ReentryCtxAddrOffset + 20), &ReentryCtxDADDiu3, + sizeof(ReentryCtxDADDiu3)); + + uint32_t ReentryFnLUi = + 0x3c190000 | (((ReentryFnAddr + 0x800080008000) >> 48) & 0xFFFF); + + uint32_t ReentryFnDADDiu = + 0x67390000 | (((ReentryFnAddr + 0x80008000) >> 32) & 0xFFFF); + + uint32_t ReentryFnDSLL = 0x0019cc38; + + uint32_t ReentryFnDADDiu2 = + 0x67390000 | (((ReentryFnAddr + 0x8000) >> 16) & 0xFFFF); + + uint32_t ReentryFnDSLL2 = 0x0019cc38; + + uint32_t ReentryFnDADDiu3 = 0x67390000 | ((ReentryFnAddr)&0xFFFF); + + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnLUi, + sizeof(ReentryFnLUi)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 4), &ReentryFnDADDiu, + sizeof(ReentryFnDADDiu)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 8), &ReentryFnDSLL, + sizeof(ReentryFnDSLL)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 12), &ReentryFnDADDiu2, + sizeof(ReentryFnDADDiu2)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 16), &ReentryFnDSLL2, + sizeof(ReentryFnDSLL2)); + memcpy(ResolverWorkingMem + (ReentryFnAddrOffset + 20), &ReentryFnDADDiu3, + sizeof(ReentryFnDADDiu3)); } -void OrcMips64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, +void OrcMips64::writeTrampolines(char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines) { - uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineMem); - uint64_t ResolveAddr = reinterpret_cast<uint64_t>(ResolverAddr); + uint32_t *Trampolines = + reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); - uint64_t HeighestAddr = ((ResolveAddr + 0x800080008000) >> 48); - uint64_t HeigherAddr = ((ResolveAddr + 0x80008000) >> 32); - uint64_t HiAddr = ((ResolveAddr + 0x8000) >> 16); + uint64_t HeighestAddr = ((ResolverAddr + 0x800080008000) >> 48); + uint64_t HeigherAddr = ((ResolverAddr + 0x80008000) >> 32); + uint64_t HiAddr = ((ResolverAddr + 0x8000) >> 16); for (unsigned I = 0; I < NumTrampolines; ++I) { Trampolines[10 * I + 0] = 0x03e0c025; // move $t8,$ra @@ -889,16 +845,17 @@ void OrcMips64::writeTrampolines(uint8_t *TrampolineMem, void *ResolverAddr, Trampolines[10 * I + 3] = 0x0019cc38; // dsll $t9,$t9,16 Trampolines[10 * I + 4] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) Trampolines[10 * I + 5] = 0x0019cc38; // dsll $t9,$t9,16 - Trampolines[10 * I + 6] = 0x67390000 | (ResolveAddr & 0xFFFF); // daddiu $t9,$t9,%lo(ptr) + Trampolines[10 * I + 6] = + 0x67390000 | (ResolverAddr & 0xFFFF); // daddiu $t9,$t9,%lo(ptr) Trampolines[10 * I + 7] = 0x0320f809; // jalr $t9 Trampolines[10 * I + 8] = 0x00000000; // nop Trampolines[10 * I + 9] = 0x00000000; // nop } } -Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, - unsigned MinStubs, - void *InitialPtrVal) { +void OrcMips64::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs @@ -926,31 +883,14 @@ Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, // .dword 0x0 // // ... - const unsigned StubSize = IndirectStubsInfo::StubSize; - - // Emit at least MinStubs, rounded up to fill the pages allocated. - static const unsigned PageSize = sys::Process::getPageSizeEstimate(); - unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; - unsigned NumStubs = (NumPages * PageSize) / StubSize; - // Allocate memory for stubs and pointers in one call. - std::error_code EC; - auto StubsMem = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( - 2 * NumPages * PageSize, nullptr, - sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); - - if (EC) - return errorCodeToError(EC); - - // Create separate MemoryBlocks representing the stubs and pointers. - sys::MemoryBlock StubsBlock(StubsMem.base(), NumPages * PageSize); - sys::MemoryBlock PtrsBlock(static_cast<char *>(StubsMem.base()) + - NumPages * PageSize, - NumPages * PageSize); + assert(stubAndPointerRangesOk<OrcAArch64>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); // Populate the stubs page stubs and mark it executable. - uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlock.base()); - uint64_t PtrAddr = reinterpret_cast<uint64_t>(PtrsBlock.base()); + uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlockWorkingMem); + uint64_t PtrAddr = PointersBlockTargetAddress; for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 8) { uint64_t HeighestAddr = ((PtrAddr + 0x800080008000) >> 48); @@ -965,19 +905,6 @@ Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, Stub[8 * I + 6] = 0x03200008; // jr $t9 Stub[8 * I + 7] = 0x00000000; // nop } - - if (auto EC = sys::Memory::protectMappedMemory( - StubsBlock, sys::Memory::MF_READ | sys::Memory::MF_EXEC)) - return errorCodeToError(EC); - - // Initialize all pointers to point at FailureAddress. - void **Ptr = reinterpret_cast<void **>(PtrsBlock.base()); - for (unsigned I = 0; I < NumStubs; ++I) - Ptr[I] = InitialPtrVal; - - StubsInfo = IndirectStubsInfo(NumStubs, std::move(StubsMem)); - - return Error::success(); } } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index e0af3df9d010f..87bb4398765de 100644 --- a/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -133,7 +133,7 @@ private: orc::SymbolNameSet Result; for (auto &S : Symbols) { - if (auto Sym = findSymbol(*S)) { + if (auto Sym = findSymbol(std::string(*S))) { if (!Sym.getFlags().isStrong()) Result.insert(S); } else if (auto Err = Sym.takeError()) { @@ -151,7 +151,7 @@ private: orc::SymbolNameSet UnresolvedSymbols; for (auto &S : Symbols) { - if (auto Sym = findSymbol(*S)) { + if (auto Sym = findSymbol(std::string(*S))) { if (auto Addr = Sym.getAddress()) { Query->notifySymbolMetRequiredState( S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); diff --git a/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 169dc8f1d02b8..139572bd69779 100644 --- a/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/llvm/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -154,7 +154,8 @@ class OrcMCJITReplacement : public ExecutionEngine { M.reportError(std::move(Err)); return SymbolNameSet(); } else { - if (auto Sym2 = M.ClientResolver->findSymbolInLogicalDylib(*S)) { + if (auto Sym2 = + M.ClientResolver->findSymbolInLogicalDylib(std::string(*S))) { if (!Sym2.getFlags().isStrong()) Result.insert(S); } else if (auto Err = Sym2.takeError()) { @@ -187,7 +188,7 @@ class OrcMCJITReplacement : public ExecutionEngine { M.ES.legacyFailQuery(*Query, std::move(Err)); return SymbolNameSet(); } else { - if (auto Sym2 = M.ClientResolver->findSymbol(*S)) { + if (auto Sym2 = M.ClientResolver->findSymbol(std::string(*S))) { if (auto Addr = Sym2.getAddress()) { Query->notifySymbolMetRequiredState( S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); @@ -378,9 +379,9 @@ public: private: JITSymbol findMangledSymbol(StringRef Name) { - if (auto Sym = LazyEmitLayer.findSymbol(Name, false)) + if (auto Sym = LazyEmitLayer.findSymbol(std::string(Name), false)) return Sym; - if (auto Sym = ClientResolver->findSymbol(Name)) + if (auto Sym = ClientResolver->findSymbol(std::string(Name))) return Sym; if (auto Sym = scanArchives(Name)) return Sym; diff --git a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp new file mode 100644 index 0000000000000..5933c2e666d1c --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -0,0 +1,254 @@ +//===--------------- OrcV2CBindings.cpp - C bindings OrcV2 APIs -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm-c/Orc.h" +#include "llvm-c/TargetMachine.h" + +#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" + +using namespace llvm; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class OrcV2CAPIHelper { +public: + using PoolEntry = SymbolStringPtr::PoolEntry; + using PoolEntryPtr = SymbolStringPtr::PoolEntryPtr; + + static PoolEntryPtr releaseSymbolStringPtr(SymbolStringPtr S) { + PoolEntryPtr Result = nullptr; + std::swap(Result, S.S); + return Result; + } + + static PoolEntryPtr getRawPoolEntryPtr(const SymbolStringPtr &S) { + return S.S; + } + + static void releasePoolEntry(PoolEntryPtr P) { + SymbolStringPtr S; + S.S = P; + } +}; + +} // end namespace orc +} // end namespace llvm + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ExecutionSession, LLVMOrcExecutionSessionRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcV2CAPIHelper::PoolEntry, + LLVMOrcSymbolStringPoolEntryRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITDylib, LLVMOrcJITDylibRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITDylib::DefinitionGenerator, + LLVMOrcJITDylibDefinitionGeneratorRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeContext, + LLVMOrcThreadSafeContextRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeModule, LLVMOrcThreadSafeModuleRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITTargetMachineBuilder, + LLVMOrcJITTargetMachineBuilderRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJITBuilder, LLVMOrcLLJITBuilderRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJIT, LLVMOrcLLJITRef) + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) + +LLVMOrcSymbolStringPoolEntryRef +LLVMOrcExecutionSessionIntern(LLVMOrcExecutionSessionRef ES, const char *Name) { + return wrap( + OrcV2CAPIHelper::releaseSymbolStringPtr(unwrap(ES)->intern(Name))); +} + +void LLVMOrcReleaseSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { + OrcV2CAPIHelper::releasePoolEntry(unwrap(S)); +} + +void LLVMOrcDisposeJITDylibDefinitionGenerator( + LLVMOrcJITDylibDefinitionGeneratorRef DG) { + delete unwrap(DG); +} + +void LLVMOrcJITDylibAddGenerator(LLVMOrcJITDylibRef JD, + LLVMOrcJITDylibDefinitionGeneratorRef DG) { + unwrap(JD)->addGenerator( + std::unique_ptr<JITDylib::DefinitionGenerator>(unwrap(DG))); +} + +LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( + LLVMOrcJITDylibDefinitionGeneratorRef *Result, char GlobalPrefix, + LLVMOrcSymbolPredicate Filter, void *FilterCtx) { + assert(Result && "Result can not be null"); + assert((Filter || !FilterCtx) && + "if Filter is null then FilterCtx must also be null"); + + DynamicLibrarySearchGenerator::SymbolPredicate Pred; + if (Filter) + Pred = [=](const SymbolStringPtr &Name) -> bool { + return Filter(wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name)), FilterCtx); + }; + + auto ProcessSymsGenerator = + DynamicLibrarySearchGenerator::GetForCurrentProcess(GlobalPrefix, Pred); + + if (!ProcessSymsGenerator) { + *Result = 0; + return wrap(ProcessSymsGenerator.takeError()); + } + + *Result = wrap(ProcessSymsGenerator->release()); + return LLVMErrorSuccess; +} + +LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void) { + return wrap(new ThreadSafeContext(std::make_unique<LLVMContext>())); +} + +LLVMContextRef +LLVMOrcThreadSafeContextGetContext(LLVMOrcThreadSafeContextRef TSCtx) { + return wrap(unwrap(TSCtx)->getContext()); +} + +void LLVMOrcDisposeThreadSafeContext(LLVMOrcThreadSafeContextRef TSCtx) { + delete unwrap(TSCtx); +} + +LLVMOrcThreadSafeModuleRef +LLVMOrcCreateNewThreadSafeModule(LLVMModuleRef M, + LLVMOrcThreadSafeContextRef TSCtx) { + return wrap( + new ThreadSafeModule(std::unique_ptr<Module>(unwrap(M)), *unwrap(TSCtx))); +} + +void LLVMOrcDisposeThreadSafeModule(LLVMOrcThreadSafeModuleRef TSM) { + delete unwrap(TSM); +} + +LLVMErrorRef LLVMOrcJITTargetMachineBuilderDetectHost( + LLVMOrcJITTargetMachineBuilderRef *Result) { + assert(Result && "Result can not be null"); + + auto JTMB = JITTargetMachineBuilder::detectHost(); + if (!JTMB) { + Result = 0; + return wrap(JTMB.takeError()); + } + + *Result = wrap(new JITTargetMachineBuilder(std::move(*JTMB))); + return LLVMErrorSuccess; +} + +LLVMOrcJITTargetMachineBuilderRef +LLVMOrcJITTargetMachineBuilderFromTargetMachine(LLVMTargetMachineRef TM) { + auto *TemplateTM = unwrap(TM); + + auto JTMB = + std::make_unique<JITTargetMachineBuilder>(TemplateTM->getTargetTriple()); + + (*JTMB) + .setCPU(TemplateTM->getTargetCPU().str()) + .setRelocationModel(TemplateTM->getRelocationModel()) + .setCodeModel(TemplateTM->getCodeModel()) + .setCodeGenOptLevel(TemplateTM->getOptLevel()) + .setFeatures(TemplateTM->getTargetFeatureString()) + .setOptions(TemplateTM->Options); + + LLVMDisposeTargetMachine(TM); + + return wrap(JTMB.release()); +} + +void LLVMOrcDisposeJITTargetMachineBuilder( + LLVMOrcJITTargetMachineBuilderRef JTMB) { + delete unwrap(JTMB); +} + +LLVMOrcLLJITBuilderRef LLVMOrcCreateLLJITBuilder(void) { + return wrap(new LLJITBuilder()); +} + +void LLVMOrcDisposeLLJITBuilder(LLVMOrcLLJITBuilderRef Builder) { + delete unwrap(Builder); +} + +void LLVMOrcLLJITBuilderSetJITTargetMachineBuilder( + LLVMOrcLLJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMB) { + unwrap(Builder)->setJITTargetMachineBuilder(*unwrap(JTMB)); +} + +LLVMErrorRef LLVMOrcCreateLLJIT(LLVMOrcLLJITRef *Result, + LLVMOrcLLJITBuilderRef Builder) { + assert(Result && "Result can not be null"); + + if (!Builder) + Builder = LLVMOrcCreateLLJITBuilder(); + + auto J = unwrap(Builder)->create(); + LLVMOrcDisposeLLJITBuilder(Builder); + + if (!J) { + Result = 0; + return wrap(J.takeError()); + } + + *Result = wrap(J->release()); + return LLVMErrorSuccess; +} + +LLVMErrorRef LLVMOrcDisposeLLJIT(LLVMOrcLLJITRef J) { + delete unwrap(J); + return LLVMErrorSuccess; +} + +LLVMOrcExecutionSessionRef LLVMOrcLLJITGetExecutionSession(LLVMOrcLLJITRef J) { + return wrap(&unwrap(J)->getExecutionSession()); +} + +LLVMOrcJITDylibRef LLVMOrcLLJITGetMainJITDylib(LLVMOrcLLJITRef J) { + return wrap(&unwrap(J)->getMainJITDylib()); +} + +const char *LLVMOrcLLJITGetTripleString(LLVMOrcLLJITRef J) { + return unwrap(J)->getTargetTriple().str().c_str(); +} + +char LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J) { + return unwrap(J)->getDataLayout().getGlobalPrefix(); +} + +LLVMOrcSymbolStringPoolEntryRef +LLVMOrcLLJITMangleAndIntern(LLVMOrcLLJITRef J, const char *UnmangledName) { + return wrap(OrcV2CAPIHelper::releaseSymbolStringPtr( + unwrap(J)->mangleAndIntern(UnmangledName))); +} + +LLVMErrorRef LLVMOrcLLJITAddObjectFile(LLVMOrcLLJITRef J, LLVMOrcJITDylibRef JD, + LLVMMemoryBufferRef ObjBuffer) { + return wrap(unwrap(J)->addObjectFile( + *unwrap(JD), std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); +} + +LLVMErrorRef LLVMOrcLLJITAddLLVMIRModule(LLVMOrcLLJITRef J, + LLVMOrcJITDylibRef JD, + LLVMOrcThreadSafeModuleRef TSM) { + return wrap(unwrap(J)->addIRModule(*unwrap(JD), std::move(*unwrap(TSM)))); +} + +LLVMErrorRef LLVMOrcLLJITLookup(LLVMOrcLLJITRef J, + LLVMOrcJITTargetAddress *Result, + const char *Name) { + assert(Result && "Result can not be null"); + + auto Sym = unwrap(J)->lookup(Name); + if (!Sym) { + *Result = 0; + return wrap(Sym.takeError()); + } + + *Result = Sym->getAddress(); + return LLVMErrorSuccess; +} diff --git a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index a92264c0be14a..21925726072e3 100644 --- a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" +#include "llvm/Object/COFF.h" namespace { @@ -46,10 +47,10 @@ public: MR.addDependenciesForAll(Deps); }; - JITDylibSearchOrder SearchOrder; - MR.getTargetJITDylib().withSearchOrderDo( - [&](const JITDylibSearchOrder &JDs) { SearchOrder = JDs; }); - ES.lookup(LookupKind::Static, SearchOrder, InternedSymbols, + JITDylibSearchOrder LinkOrder; + MR.getTargetJITDylib().withLinkOrderDo( + [&](const JITDylibSearchOrder &LO) { LinkOrder = LO; }); + ES.lookup(LookupKind::Static, LinkOrder, InternedSymbols, SymbolState::Resolved, std::move(OnResolvedWithUnwrap), RegisterDependencies); } @@ -80,8 +81,12 @@ RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( RTDyldObjectLinkingLayer::~RTDyldObjectLinkingLayer() { std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); - for (auto &MemMgr : MemMgrs) + for (auto &MemMgr : MemMgrs) { + for (auto *L : EventListeners) + L->notifyFreeingObject( + static_cast<uint64_t>(reinterpret_cast<uintptr_t>(MemMgr.get()))); MemMgr->deregisterEHFrames(); + } } void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, @@ -96,13 +101,7 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, auto &ES = getExecutionSession(); - // Create a MemoryBufferRef backed MemoryBuffer (i.e. shallow) copy of the - // the underlying buffer to pass into RuntimeDyld. This allows us to hold - // ownership of the real underlying buffer and return it to the user once - // the object has been emitted. - auto ObjBuffer = MemoryBuffer::getMemBuffer(O->getMemBufferRef(), false); - - auto Obj = object::ObjectFile::createObjectFile(*ObjBuffer); + auto Obj = object::ObjectFile::createObjectFile(*O); if (!Obj) { getExecutionSession().reportError(Obj.takeError()); @@ -115,7 +114,27 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, auto InternalSymbols = std::make_shared<std::set<StringRef>>(); { for (auto &Sym : (*Obj)->symbols()) { - if (!(Sym.getFlags() & object::BasicSymbolRef::SF_Global)) { + + // Skip file symbols. + if (auto SymType = Sym.getType()) { + if (*SymType == object::SymbolRef::ST_File) + continue; + } else { + ES.reportError(SymType.takeError()); + R.failMaterialization(); + return; + } + + Expected<uint32_t> SymFlagsOrErr = Sym.getFlags(); + if (!SymFlagsOrErr) { + // TODO: Test this error. + ES.reportError(SymFlagsOrErr.takeError()); + R.failMaterialization(); + return; + } + + // Don't include symbols that aren't global. + if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global)) { if (auto SymName = Sym.getName()) InternalSymbols->insert(*SymName); else { @@ -141,25 +160,79 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, JITDylibSearchOrderResolver Resolver(*SharedR); jitLinkForORC( - **Obj, std::move(O), *MemMgr, Resolver, ProcessAllSections, - [this, K, SharedR, &Obj, InternalSymbols]( + object::OwningBinary<object::ObjectFile>(std::move(*Obj), std::move(O)), + *MemMgr, Resolver, ProcessAllSections, + [this, K, SharedR, MemMgr, InternalSymbols]( + const object::ObjectFile &Obj, std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, std::map<StringRef, JITEvaluatedSymbol> ResolvedSymbols) { - return onObjLoad(K, *SharedR, **Obj, std::move(LoadedObjInfo), + return onObjLoad(K, *SharedR, Obj, MemMgr, std::move(LoadedObjInfo), ResolvedSymbols, *InternalSymbols); }, - [this, K, SharedR, O = std::move(O)](Error Err) mutable { - onObjEmit(K, std::move(O), *SharedR, std::move(Err)); + [this, K, SharedR, MemMgr](object::OwningBinary<object::ObjectFile> Obj, + Error Err) mutable { + onObjEmit(K, *SharedR, std::move(Obj), MemMgr, std::move(Err)); }); } +void RTDyldObjectLinkingLayer::registerJITEventListener(JITEventListener &L) { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + assert(llvm::none_of(EventListeners, + [&](JITEventListener *O) { return O == &L; }) && + "Listener has already been registered"); + EventListeners.push_back(&L); +} + +void RTDyldObjectLinkingLayer::unregisterJITEventListener(JITEventListener &L) { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + auto I = llvm::find(EventListeners, &L); + assert(I != EventListeners.end() && "Listener not registered"); + EventListeners.erase(I); +} + Error RTDyldObjectLinkingLayer::onObjLoad( - VModuleKey K, MaterializationResponsibility &R, object::ObjectFile &Obj, + VModuleKey K, MaterializationResponsibility &R, + const object::ObjectFile &Obj, RuntimeDyld::MemoryManager *MemMgr, std::unique_ptr<RuntimeDyld::LoadedObjectInfo> LoadedObjInfo, std::map<StringRef, JITEvaluatedSymbol> Resolved, std::set<StringRef> &InternalSymbols) { SymbolFlagsMap ExtraSymbolsToClaim; SymbolMap Symbols; + + // Hack to support COFF constant pool comdats introduced during compilation: + // (See http://llvm.org/PR40074) + if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(&Obj)) { + auto &ES = getExecutionSession(); + + // For all resolved symbols that are not already in the responsibilty set: + // check whether the symbol is in a comdat section and if so mark it as + // weak. + for (auto &Sym : COFFObj->symbols()) { + // getFlags() on COFF symbols can't fail. + uint32_t SymFlags = cantFail(Sym.getFlags()); + if (SymFlags & object::BasicSymbolRef::SF_Undefined) + continue; + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto I = Resolved.find(*Name); + + // Skip unresolved symbols, internal symbols, and symbols that are + // already in the responsibility set. + if (I == Resolved.end() || InternalSymbols.count(*Name) || + R.getSymbols().count(ES.intern(*Name))) + continue; + auto Sec = Sym.getSection(); + if (!Sec) + return Sec.takeError(); + if (*Sec == COFFObj->section_end()) + continue; + auto &COFFSec = *COFFObj->getCOFFSection(**Sec); + if (COFFSec.Characteristics & COFF::IMAGE_SCN_LNK_COMDAT) + I->second.setFlags(I->second.getFlags() | JITSymbolFlags::Weak); + } + } + for (auto &KV : Resolved) { // Scan the symbols and add them to the Symbols map for resolution. @@ -184,10 +257,17 @@ Error RTDyldObjectLinkingLayer::onObjLoad( Symbols[InternedName] = JITEvaluatedSymbol(KV.second.getAddress(), Flags); } - if (!ExtraSymbolsToClaim.empty()) + if (!ExtraSymbolsToClaim.empty()) { if (auto Err = R.defineMaterializing(ExtraSymbolsToClaim)) return Err; + // If we claimed responsibility for any weak symbols but were rejected then + // we need to remove them from the resolved set. + for (auto &KV : ExtraSymbolsToClaim) + if (KV.second.isWeak() && !R.getSymbols().count(KV.first)) + Symbols.erase(KV.first); + } + if (auto Err = R.notifyResolved(Symbols)) { R.failMaterialization(); return Err; @@ -196,12 +276,17 @@ Error RTDyldObjectLinkingLayer::onObjLoad( if (NotifyLoaded) NotifyLoaded(K, Obj, *LoadedObjInfo); + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + assert(!LoadedObjInfos.count(MemMgr) && "Duplicate loaded info for MemMgr"); + LoadedObjInfos[MemMgr] = std::move(LoadedObjInfo); + return Error::success(); } void RTDyldObjectLinkingLayer::onObjEmit( - VModuleKey K, std::unique_ptr<MemoryBuffer> ObjBuffer, - MaterializationResponsibility &R, Error Err) { + VModuleKey K, MaterializationResponsibility &R, + object::OwningBinary<object::ObjectFile> O, + RuntimeDyld::MemoryManager *MemMgr, Error Err) { if (Err) { getExecutionSession().reportError(std::move(Err)); R.failMaterialization(); @@ -214,6 +299,22 @@ void RTDyldObjectLinkingLayer::onObjEmit( return; } + std::unique_ptr<object::ObjectFile> Obj; + std::unique_ptr<MemoryBuffer> ObjBuffer; + std::tie(Obj, ObjBuffer) = O.takeBinary(); + + // Run EventListener notifyLoaded callbacks. + { + std::lock_guard<std::mutex> Lock(RTDyldLayerMutex); + auto LOIItr = LoadedObjInfos.find(MemMgr); + assert(LOIItr != LoadedObjInfos.end() && "LoadedObjInfo missing"); + for (auto *L : EventListeners) + L->notifyObjectLoaded( + static_cast<uint64_t>(reinterpret_cast<uintptr_t>(MemMgr)), *Obj, + *LOIItr->second); + LoadedObjInfos.erase(MemMgr); + } + if (NotifyEmitted) NotifyEmitted(K, std::move(ObjBuffer)); } diff --git a/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp b/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp index f22acf50419d4..7240c1ed0ce98 100644 --- a/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp +++ b/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp @@ -209,7 +209,7 @@ void SequenceBBQuery::traverseToExitBlock(const BasicBlock *AtBB, VisitedBlocks.insert(std::make_pair(AtBB, BlockHint)); } - succ_const_iterator PIt = succ_begin(AtBB), EIt = succ_end(AtBB); + const_succ_iterator PIt = succ_begin(AtBB), EIt = succ_end(AtBB); if (PIt == EIt) // No succs. return; diff --git a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp index f29201c147a12..0530b1a97b678 100644 --- a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp @@ -96,7 +96,7 @@ void IRSpeculationLayer::emit(MaterializationResponsibility R, M, LoadValueTy, false, GlobalValue::LinkageTypes::InternalLinkage, ConstantInt::get(LoadValueTy, 0), "__orc_speculate.guard.for." + Fn.getName()); - SpeculatorGuard->setAlignment(Align::None()); + SpeculatorGuard->setAlignment(Align(1)); SpeculatorGuard->setUnnamedAddr(GlobalValue::UnnamedAddr::Local); BasicBlock &ProgramEntry = Fn.getEntryBlock(); |