diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2023-09-02 21:17:18 +0000 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2024-01-07 17:46:17 +0000 |
| commit | fe013be447cd855ccaf6094a1d06aea570450629 (patch) | |
| tree | 9adc1e0a5d25b6280995832bb29d592fb80554a6 /contrib/llvm-project/llvm/lib/ExecutionEngine/Orc | |
| parent | 2f3b605b2e159522ecab77fd518e8139aaf581e9 (diff) | |
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ExecutionEngine/Orc')
34 files changed, 1453 insertions, 866 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp index 40716a7f9b61..7c869bead0b0 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp @@ -10,6 +10,7 @@ #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Object/COFF.h" @@ -54,8 +55,7 @@ public: void materialize(std::unique_ptr<MaterializationResponsibility> R) override { unsigned PointerSize; support::endianness Endianness; - const auto &TT = - CP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + const auto &TT = CP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::x86_64: @@ -125,8 +125,8 @@ private: llvm_unreachable("Unrecognized architecture"); } - auto HeaderContent = G.allocateString( - StringRef(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); + auto HeaderContent = G.allocateContent( + ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, 0); @@ -159,20 +159,36 @@ private: namespace llvm { namespace orc { -Expected<std::unique_ptr<COFFPlatform>> -COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - JITDylib &PlatformJD, const char *OrcRuntimePath, - LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, - const char *VCRuntimePath, - std::optional<SymbolAliasMap> RuntimeAliases) { - auto &EPC = ES.getExecutorProcessControl(); +Expected<std::unique_ptr<COFFPlatform>> COFFPlatform::Create( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer, + LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, + const char *VCRuntimePath, std::optional<SymbolAliasMap> RuntimeAliases) { // If the target is not supported then bail out immediately. - if (!supportedTarget(EPC.getTargetTriple())) + if (!supportedTarget(ES.getTargetTriple())) return make_error<StringError>("Unsupported COFFPlatform triple: " + - EPC.getTargetTriple().str(), + ES.getTargetTriple().str(), inconvertibleErrorCode()); + auto &EPC = ES.getExecutorProcessControl(); + + auto GeneratorArchive = + object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef()); + if (!GeneratorArchive) + return GeneratorArchive.takeError(); + + auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create( + ObjLinkingLayer, nullptr, std::move(*GeneratorArchive)); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + // We need a second instance of the archive (for now) for the Platform. We + // can `cantFail` this call, since if it were going to fail it would have + // failed above. + auto RuntimeArchive = cantFail( + object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef())); + // Create default aliases if the caller didn't supply any. if (!RuntimeAliases) RuntimeAliases = standardPlatformAliases(ES); @@ -184,13 +200,13 @@ COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>"); // Add JIT-dispatch function support symbols. - if (auto Err = HostFuncJD.define(absoluteSymbols( - {{ES.intern("__orc_rt_jit_dispatch"), - {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), - JITSymbolFlags::Exported}}, - {ES.intern("__orc_rt_jit_dispatch_ctx"), - {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), - JITSymbolFlags::Exported}}}))) + if (auto Err = HostFuncJD.define( + absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunction, + JITSymbolFlags::Exported}}, + {ES.intern("__orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContext, + JITSymbolFlags::Exported}}}))) return std::move(Err); PlatformJD.addToLinkOrder(HostFuncJD); @@ -198,13 +214,30 @@ COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, // Create the instance. Error Err = Error::success(); auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform( - ES, ObjLinkingLayer, PlatformJD, OrcRuntimePath, + ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator), + std::move(OrcRuntimeArchiveBuffer), std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err)); if (Err) return std::move(Err); return std::move(P); } +Expected<std::unique_ptr<COFFPlatform>> +COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, + const char *VCRuntimePath, + std::optional<SymbolAliasMap> RuntimeAliases) { + + auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath); + if (!ArchiveBuffer) + return createFileError(OrcRuntimePath, ArchiveBuffer.getError()); + + return Create(ES, ObjLinkingLayer, PlatformJD, std::move(*ArchiveBuffer), + std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, + std::move(RuntimeAliases)); +} + Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() { auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker"); if (!PerJDObj) @@ -348,37 +381,22 @@ bool COFFPlatform::supportedTarget(const Triple &TT) { } } -COFFPlatform::COFFPlatform(ExecutionSession &ES, - ObjectLinkingLayer &ObjLinkingLayer, - JITDylib &PlatformJD, const char *OrcRuntimePath, - LoadDynamicLibrary LoadDynLibrary, - bool StaticVCRuntime, const char *VCRuntimePath, - Error &Err) +COFFPlatform::COFFPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, + std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator, + std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer, + std::unique_ptr<object::Archive> OrcRuntimeArchive, + LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, + const char *VCRuntimePath, Error &Err) : ES(ES), ObjLinkingLayer(ObjLinkingLayer), LoadDynLibrary(std::move(LoadDynLibrary)), + OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)), + OrcRuntimeArchive(std::move(OrcRuntimeArchive)), StaticVCRuntime(StaticVCRuntime), COFFHeaderStartSymbol(ES.intern("__ImageBase")) { ErrorAsOutParameter _(&Err); - // Create a generator for the ORC runtime archive. - auto OrcRuntimeArchiveGenerator = - StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath); - if (!OrcRuntimeArchiveGenerator) { - Err = OrcRuntimeArchiveGenerator.takeError(); - return; - } - - auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath); - if (!ArchiveBuffer) { - Err = createFileError(OrcRuntimePath, ArchiveBuffer.getError()); - return; - } - OrcRuntimeArchiveBuffer = std::move(*ArchiveBuffer); - OrcRuntimeArchive = - std::make_unique<object::Archive>(*OrcRuntimeArchiveBuffer, Err); - if (Err) - return; - Bootstrapping.store(true); ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this)); @@ -391,7 +409,7 @@ COFFPlatform::COFFPlatform(ExecutionSession &ES, } VCRuntimeBootstrap = std::move(*VCRT); - for (auto &Lib : (*OrcRuntimeArchiveGenerator)->getImportedDynamicLibraries()) + for (auto &Lib : OrcRuntimeGenerator->getImportedDynamicLibraries()) DylibsToPreload.insert(Lib); auto ImportedLibs = @@ -405,7 +423,7 @@ COFFPlatform::COFFPlatform(ExecutionSession &ES, for (auto &Lib : *ImportedLibs) DylibsToPreload.insert(Lib); - PlatformJD.addGenerator(std::move(*OrcRuntimeArchiveGenerator)); + PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); // PlatformJD hasn't been set up by the platform yet (since we're creating // the platform now), so set it up. @@ -415,10 +433,10 @@ COFFPlatform::COFFPlatform(ExecutionSession &ES, } for (auto& Lib : DylibsToPreload) - if (auto E2 = LoadDynLibrary(PlatformJD, Lib)) { - Err = std::move(E2); - return; - } + if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) { + Err = std::move(E2); + return; + } if (StaticVCRuntime) if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) { @@ -561,10 +579,9 @@ void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, }); if (!JD) { - SendResult( - make_error<StringError>("No JITDylib with header addr " + - formatv("{0:x}", JDHeaderAddr.getValue()), - inconvertibleErrorCode())); + SendResult(make_error<StringError>("No JITDylib with header addr " + + formatv("{0:x}", JDHeaderAddr), + inconvertibleErrorCode())); return; } @@ -579,10 +596,7 @@ void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, StringRef SymbolName) { - LLVM_DEBUG({ - dbgs() << "COFFPlatform::rt_lookupSymbol(\"" - << formatv("{0:x}", Handle.getValue()) << "\")\n"; - }); + LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"); JITDylib *JD = nullptr; @@ -594,12 +608,9 @@ void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, } if (!JD) { - LLVM_DEBUG({ - dbgs() << " No JITDylib for handle " - << formatv("{0:x}", Handle.getValue()) << "\n"; - }); + LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + - formatv("{0:x}", Handle.getValue()), + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } @@ -612,7 +623,7 @@ void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, void operator()(Expected<SymbolMap> Result) { if (Result) { assert(Result->size() == 1 && "Unexpected result map count"); - SendResult(ExecutorAddr(Result->begin()->second.getAddress())); + SendResult(Result->begin()->second.getAddress()); } else { SendResult(Result.takeError()); } @@ -850,7 +861,7 @@ Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections( jitlink::LinkGraph &G, MaterializationResponsibility &MR) { JITLinkSymbolSet InitSectionSymbols; for (auto &Sec : G.sections()) - if (COFFPlatform::isInitializerSection(Sec.getName())) + if (isCOFFInitializerSection(Sec.getName())) for (auto *B : Sec.blocks()) if (!B->edges_empty()) InitSectionSymbols.insert( @@ -885,14 +896,13 @@ Error COFFPlatform::COFFPlatformPlugin:: // Collect static initializers for (auto &S : G.sections()) - if (COFFPlatform::isInitializerSection(S.getName())) + if (isCOFFInitializerSection(S.getName())) for (auto *B : S.blocks()) { if (B->edges_empty()) continue; for (auto &E : B->edges()) BState.Initializers.push_back(std::make_pair( - S.getName().str(), - ExecutorAddr(E.getTarget().getAddress() + E.getAddend()))); + S.getName().str(), E.getTarget().getAddress() + E.getAddend())); } return Error::success(); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp index d9316fab2de3..94f696fa2086 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp @@ -160,7 +160,7 @@ COFFVCRuntimeBootstrapper::getMSVCToolchainPath() { if (!findVCToolChainViaCommandLine(*VFS, std::nullopt, std::nullopt, std::nullopt, VCToolChainPath, VSLayout) && !findVCToolChainViaEnvironment(*VFS, VCToolChainPath, VSLayout) && - !findVCToolChainViaSetupConfig(*VFS, VCToolChainPath, VSLayout) && + !findVCToolChainViaSetupConfig(*VFS, {}, VCToolChainPath, VSLayout) && !findVCToolChainViaRegistry(VCToolChainPath, VSLayout)) return make_error<StringError>("Couldn't find msvc toolchain.", inconvertibleErrorCode()); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp index 4a9d0d470a8e..0c23f2b25219 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -167,15 +167,16 @@ AsynchronousSymbolQuery::AsynchronousSymbolQuery( OutstandingSymbolsCount = Symbols.size(); for (auto &KV : Symbols) - ResolvedSymbols[KV.first] = nullptr; + ResolvedSymbols[KV.first] = ExecutorSymbolDef(); } void AsynchronousSymbolQuery::notifySymbolMetRequiredState( - const SymbolStringPtr &Name, JITEvaluatedSymbol Sym) { + const SymbolStringPtr &Name, ExecutorSymbolDef Sym) { auto I = ResolvedSymbols.find(Name); assert(I != ResolvedSymbols.end() && "Resolving symbol outside the requested set"); - assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); + assert(I->second == ExecutorSymbolDef() && + "Redundantly resolving symbol Name"); // If this is a materialization-side-effects-only symbol then drop it, // otherwise update its map entry with its resolved address. @@ -447,8 +448,8 @@ void ReExportsMaterializationUnit::materialize( if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly()) continue; - ResolutionMap[KV.first] = JITEvaluatedSymbol( - (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); + ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(), + KV.second.AliasFlags}; } if (auto Err = QueryInfo->R->notifyResolved(ResolutionMap)) { ES.reportError(std::move(Err)); @@ -688,11 +689,15 @@ void JITDylib::removeGenerator(DefinitionGenerator &G) { } Expected<SymbolFlagsMap> -JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { +JITDylib::defineMaterializing(MaterializationResponsibility &FromMR, + SymbolFlagsMap SymbolFlags) { return ES.runSessionLocked([&]() -> Expected<SymbolFlagsMap> { - std::vector<SymbolTable::iterator> AddedSyms; - std::vector<SymbolFlagsMap::iterator> RejectedWeakDefs; + if (FromMR.RT->isDefunct()) + return make_error<ResourceTrackerDefunct>(FromMR.RT); + + std::vector<NonOwningSymbolStringPtr> AddedSyms; + std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs; for (auto SFItr = SymbolFlags.begin(), SFEnd = SymbolFlags.end(); SFItr != SFEnd; ++SFItr) { @@ -708,27 +713,27 @@ JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { // If this is a strong definition then error out. if (!Flags.isWeak()) { // Remove any symbols already added. - for (auto &SI : AddedSyms) - Symbols.erase(SI); + for (auto &S : AddedSyms) + Symbols.erase(Symbols.find_as(S)); // FIXME: Return all duplicates. return make_error<DuplicateDefinition>(std::string(*Name)); } // Otherwise just make a note to discard this symbol after the loop. - RejectedWeakDefs.push_back(SFItr); + RejectedWeakDefs.push_back(NonOwningSymbolStringPtr(Name)); continue; } else EntryItr = Symbols.insert(std::make_pair(Name, SymbolTableEntry(Flags))).first; - AddedSyms.push_back(EntryItr); + AddedSyms.push_back(NonOwningSymbolStringPtr(Name)); EntryItr->second.setState(SymbolState::Materializing); } // Remove any rejected weak definitions from the SymbolFlags map. while (!RejectedWeakDefs.empty()) { - SymbolFlags.erase(RejectedWeakDefs.back()); + SymbolFlags.erase(SymbolFlags.find_as(RejectedWeakDefs.back())); RejectedWeakDefs.pop_back(); } @@ -944,7 +949,7 @@ Error JITDylib::resolve(MaterializationResponsibility &MR, struct WorklistEntry { SymbolTable::iterator SymI; - JITEvaluatedSymbol ResolvedSym; + ExecutorSymbolDef ResolvedSym; }; SymbolNameSet SymbolsInErrorState; @@ -964,7 +969,7 @@ Error JITDylib::resolve(MaterializationResponsibility &MR, "Resolving symbol with materializer attached?"); assert(SymI->second.getState() == SymbolState::Materializing && "Symbol should be materializing"); - assert(SymI->second.getAddress() == 0 && + assert(SymI->second.getAddress() == ExecutorAddr() && "Symbol has already been resolved"); if (SymI->second.getFlags().hasError()) @@ -976,8 +981,7 @@ Error JITDylib::resolve(MaterializationResponsibility &MR, (SymI->second.getFlags() & ~JITSymbolFlags::Common) && "Resolved flags should match the declared flags"); - Worklist.push_back( - {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)}); + Worklist.push_back({SymI, {KV.second.getAddress(), Flags}}); } } @@ -1328,6 +1332,18 @@ void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder, }); } +void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) { + ES.runSessionLocked([&]() { + for (auto &KV : NewLinks) { + // Skip elements of NewLinks that are already in the link order. + if (llvm::find(LinkOrder, KV) != LinkOrder.end()) + continue; + + LinkOrder.push_back(std::move(KV)); + } + }); +} + void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) { ES.runSessionLocked([&]() { LinkOrder.push_back({&JD, JDLookupFlags}); }); } @@ -1437,16 +1453,23 @@ void JITDylib::dump(raw_ostream &OS) { OS << "Link order: " << LinkOrder << "\n" << "Symbol table:\n"; - for (auto &KV : Symbols) { + // Sort symbols so we get a deterministic order and can check them in tests. + std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted; + for (auto &KV : Symbols) + SymbolsSorted.emplace_back(KV.first, &KV.second); + std::sort(SymbolsSorted.begin(), SymbolsSorted.end(), + [](const auto &L, const auto &R) { return *L.first < *R.first; }); + + for (auto &KV : SymbolsSorted) { OS << " \"" << *KV.first << "\": "; - if (auto Addr = KV.second.getAddress()) - OS << format("0x%016" PRIx64, Addr); + if (auto Addr = KV.second->getAddress()) + OS << Addr; else OS << "<not resolved> "; - OS << " " << KV.second.getFlags() << " " << KV.second.getState(); + OS << " " << KV.second->getFlags() << " " << KV.second->getState(); - if (KV.second.hasMaterializerAttached()) { + if (KV.second->hasMaterializerAttached()) { OS << " (Materializer "; auto I = UnmaterializedInfos.find(KV.first); assert(I != UnmaterializedInfos.end() && @@ -1940,6 +1963,7 @@ JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { JITDylib &ExecutionSession::createBareJITDylib(std::string Name) { assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { + assert(SessionOpen && "Cannot create JITDylib after session is closed"); JDs.push_back(new JITDylib(*this, std::move(Name))); return *JDs.back(); }); @@ -2156,7 +2180,7 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, #endif } -Expected<JITEvaluatedSymbol> +Expected<ExecutorSymbolDef> ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, SymbolStringPtr Name, SymbolState RequiredState) { SymbolLookupSet Names({Name}); @@ -2170,13 +2194,13 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, return ResultMap.takeError(); } -Expected<JITEvaluatedSymbol> +Expected<ExecutorSymbolDef> ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name, SymbolState RequiredState) { return lookup(makeJITDylibSearchOrder(SearchOrder), Name, RequiredState); } -Expected<JITEvaluatedSymbol> +Expected<ExecutorSymbolDef> ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name, SymbolState RequiredState) { return lookup(SearchOrder, intern(Name), RequiredState); @@ -2213,9 +2237,9 @@ Error ExecutionSession::registerJITDispatchHandlers( return Error::success(); } -void ExecutionSession::runJITDispatchHandler( - SendResultFunction SendResult, JITTargetAddress HandlerFnTagAddr, - ArrayRef<char> ArgBuffer) { +void ExecutionSession::runJITDispatchHandler(SendResultFunction SendResult, + ExecutorAddr HandlerFnTagAddr, + ArrayRef<char> ArgBuffer) { std::shared_ptr<JITDispatchHandlerFunction> F; { @@ -2666,7 +2690,7 @@ void ExecutionSession::OL_completeLookup( // whether it has a materializer attached, and if so prepare to run // it. if (SymI->second.hasMaterializerAttached()) { - assert(SymI->second.getAddress() == 0 && + assert(SymI->second.getAddress() == ExecutorAddr() && "Symbol not resolved but already has address?"); auto UMII = JD.UnmaterializedInfos.find(Name); assert(UMII != JD.UnmaterializedInfos.end() && @@ -2946,7 +2970,7 @@ Error ExecutionSession::OL_defineMaterializing( << NewSymbolFlags << "\n"; }); if (auto AcceptedDefs = - MR.JD.defineMaterializing(std::move(NewSymbolFlags))) { + MR.JD.defineMaterializing(MR, std::move(NewSymbolFlags))) { // Add all newly accepted symbols to this responsibility object. for (auto &KV : *AcceptedDefs) MR.SymbolFlags.insert(KV); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp index 02c3e617df68..acbf33888ade 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp @@ -60,26 +60,13 @@ public: private: typename ELFT::Shdr *Header; - - bool isTextOrDataSection() const; }; template <typename ELFT> void ELFDebugObjectSection<ELFT>::setTargetMemoryRange(SectionRange Range) { - // Only patch load-addresses for executable and data sections. - if (isTextOrDataSection()) - Header->sh_addr = - static_cast<typename ELFT::uint>(Range.getStart().getValue()); -} - -template <typename ELFT> -bool ELFDebugObjectSection<ELFT>::isTextOrDataSection() const { - switch (Header->sh_type) { - case ELF::SHT_PROGBITS: - case ELF::SHT_X86_64_UNWIND: - return Header->sh_flags & (ELF::SHF_EXECINSTR | ELF::SHF_ALLOC); - } - return false; + // All recorded sections are candidates for load-address patching. + Header->sh_addr = + static_cast<typename ELFT::uint>(Range.getStart().getValue()); } template <typename ELFT> @@ -106,16 +93,19 @@ Error ELFDebugObjectSection<ELFT>::validateInBounds(StringRef Buffer, template <typename ELFT> void ELFDebugObjectSection<ELFT>::dump(raw_ostream &OS, StringRef Name) { - if (auto Addr = static_cast<JITTargetAddress>(Header->sh_addr)) { + if (uint64_t Addr = Header->sh_addr) { OS << formatv(" {0:x16} {1}\n", Addr, Name); } else { OS << formatv(" {0}\n", Name); } } -enum class Requirement { +enum DebugObjectFlags : int { // Request final target memory load-addresses for all sections. - ReportFinalSectionLoadAddresses, + ReportFinalSectionLoadAddresses = 1 << 0, + + // We found sections with debug information when processing the input object. + HasDebugSections = 1 << 1, }; /// The plugin creates a debug object from when JITLink starts processing the @@ -127,10 +117,15 @@ class DebugObject { public: DebugObject(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD, ExecutionSession &ES) - : MemMgr(MemMgr), JD(JD), ES(ES) {} + : MemMgr(MemMgr), JD(JD), ES(ES), Flags(DebugObjectFlags{}) {} - void set(Requirement Req) { Reqs.insert(Req); } - bool has(Requirement Req) const { return Reqs.count(Req) > 0; } + bool hasFlags(DebugObjectFlags F) const { return Flags & F; } + void setFlags(DebugObjectFlags F) { + Flags = static_cast<DebugObjectFlags>(Flags | F); + } + void clearFlags(DebugObjectFlags F) { + Flags = static_cast<DebugObjectFlags>(Flags & ~F); + } using FinalizeContinuation = std::function<void(Expected<ExecutorAddrRange>)>; @@ -159,7 +154,7 @@ protected: private: ExecutionSession &ES; - std::set<Requirement> Reqs; + DebugObjectFlags Flags; FinalizedAlloc Alloc; }; @@ -171,8 +166,7 @@ void DebugObject::finalizeAsync(FinalizeContinuation OnFinalize) { if (auto SimpleSegAlloc = finalizeWorkingMemory()) { auto ROSeg = SimpleSegAlloc->getSegInfo(MemProt::Read); - ExecutorAddrRange DebugObjRange(ExecutorAddr(ROSeg.Addr), - ExecutorAddrDiff(ROSeg.WorkingMem.size())); + ExecutorAddrRange DebugObjRange(ROSeg.Addr, ROSeg.WorkingMem.size()); SimpleSegAlloc->finalize( [this, DebugObjRange, OnFinalize = std::move(OnFinalize)](Expected<FinalizedAlloc> FA) { @@ -222,7 +216,7 @@ private: JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD, ExecutionSession &ES) : DebugObject(MemMgr, JD, ES), Buffer(std::move(Buffer)) { - set(Requirement::ReportFinalSectionLoadAddresses); + setFlags(ReportFinalSectionLoadAddresses); } std::unique_ptr<WritableMemoryBuffer> Buffer; @@ -271,24 +265,23 @@ ELFDebugObject::CreateArchType(MemoryBufferRef Buffer, if (!ObjRef) return ObjRef.takeError(); - // TODO: Add support for other architectures. - uint16_t TargetMachineArch = ObjRef->getHeader().e_machine; - if (TargetMachineArch != ELF::EM_X86_64) - return nullptr; - Expected<ArrayRef<SectionHeader>> Sections = ObjRef->sections(); if (!Sections) return Sections.takeError(); - bool HasDwarfSection = false; for (const SectionHeader &Header : *Sections) { Expected<StringRef> Name = ObjRef->getSectionName(Header); if (!Name) return Name.takeError(); if (Name->empty()) continue; - HasDwarfSection |= isDwarfSection(*Name); + if (isDwarfSection(*Name)) + DebugObj->setFlags(HasDebugSections); + // Only record text and data sections (i.e. no bss, comments, rel, etc.) + if (Header.sh_type != ELF::SHT_PROGBITS && + Header.sh_type != ELF::SHT_X86_64_UNWIND) + continue; if (!(Header.sh_flags & ELF::SHF_ALLOC)) continue; @@ -297,13 +290,6 @@ ELFDebugObject::CreateArchType(MemoryBufferRef Buffer, return std::move(Err); } - if (!HasDwarfSection) { - LLVM_DEBUG(dbgs() << "Aborting debug registration for LinkGraph \"" - << DebugObj->Buffer->getBufferIdentifier() - << "\": input object contains no debug info\n"); - return nullptr; - } - return std::move(DebugObj); } @@ -371,12 +357,11 @@ Error ELFDebugObject::recordSection( StringRef Name, std::unique_ptr<ELFDebugObjectSection<ELFT>> Section) { if (Error Err = Section->validateInBounds(this->getBuffer(), Name.data())) return Err; - auto ItInserted = Sections.try_emplace(Name, std::move(Section)); - if (!ItInserted.second) - return make_error<StringError>("In " + Buffer->getBufferIdentifier() + - ", encountered duplicate section \"" + - Name + "\" while building debug object", - inconvertibleErrorCode()); + bool Inserted = Sections.try_emplace(Name, std::move(Section)).second; + if (!Inserted) + LLVM_DEBUG(dbgs() << "Skipping debug registration for section '" << Name + << "' in object " << Buffer->getBufferIdentifier() + << " (duplicate name)\n"); return Error::success(); } @@ -403,8 +388,15 @@ createDebugObjectFromBuffer(ExecutionSession &ES, LinkGraph &G, } DebugObjectManagerPlugin::DebugObjectManagerPlugin( + ExecutionSession &ES, std::unique_ptr<DebugObjectRegistrar> Target, + bool RequireDebugSections, bool AutoRegisterCode) + : ES(ES), Target(std::move(Target)), + RequireDebugSections(RequireDebugSections), + AutoRegisterCode(AutoRegisterCode) {} + +DebugObjectManagerPlugin::DebugObjectManagerPlugin( ExecutionSession &ES, std::unique_ptr<DebugObjectRegistrar> Target) - : ES(ES), Target(std::move(Target)) {} + : DebugObjectManagerPlugin(ES, std::move(Target), true, true) {} DebugObjectManagerPlugin::~DebugObjectManagerPlugin() = default; @@ -418,8 +410,14 @@ void DebugObjectManagerPlugin::notifyMaterializing( if (auto DebugObj = createDebugObjectFromBuffer(ES, G, Ctx, ObjBuffer)) { // Not all link artifacts allow debugging. - if (*DebugObj != nullptr) - PendingObjs[&MR] = std::move(*DebugObj); + if (*DebugObj == nullptr) + return; + if (RequireDebugSections && !(**DebugObj).hasFlags(HasDebugSections)) { + LLVM_DEBUG(dbgs() << "Skipping debug registration for LinkGraph '" + << G.getName() << "': no debug info\n"); + return; + } + PendingObjs[&MR] = std::move(*DebugObj); } else { ES.reportError(DebugObj.takeError()); } @@ -435,7 +433,7 @@ void DebugObjectManagerPlugin::modifyPassConfig( return; DebugObject &DebugObj = *It->second; - if (DebugObj.has(Requirement::ReportFinalSectionLoadAddresses)) { + if (DebugObj.hasFlags(ReportFinalSectionLoadAddresses)) { PassConfig.PostAllocationPasses.push_back( [&DebugObj](LinkGraph &Graph) -> Error { for (const Section &GraphSection : Graph.sections()) @@ -467,7 +465,8 @@ Error DebugObjectManagerPlugin::notifyEmitted( FinalizePromise.set_value(TargetMem.takeError()); return; } - if (Error Err = Target->registerDebugObject(*TargetMem)) { + if (Error Err = + Target->registerDebugObject(*TargetMem, AutoRegisterCode)) { FinalizePromise.set_value(std::move(Err)); return; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp index 028bd245fb55..aca457642212 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp @@ -172,9 +172,8 @@ raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) { - return OS << format("0x%016" PRIx64, Sym.getAddress()) << " " - << Sym.getFlags(); +raw_ostream &operator<<(raw_ostream &OS, const ExecutorSymbolDef &Sym) { + return OS << Sym.getAddress() << " " << Sym.getFlags(); } raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV) { @@ -299,8 +298,12 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPool &SSP) { std::lock_guard<std::mutex> Lock(SSP.PoolMutex); + SmallVector<std::pair<StringRef, int>, 0> Vec; for (auto &KV : SSP.Pool) - OS << KV.first() << ": " << KV.second << "\n"; + Vec.emplace_back(KV.first(), KV.second); + llvm::sort(Vec, less_first()); + for (auto &[K, V] : Vec) + OS << K << ": " << V << "\n"; return OS; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp index 15e7ffb2f75a..830582bb3649 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp @@ -348,11 +348,12 @@ public: Writer.write(SecCmd); } + static constexpr bool AutoRegisterCode = true; SectionRange R(MachOContainerBlock->getSection()); G.allocActions().push_back( {cantFail(shared::WrapperFunctionCall::Create< - shared::SPSArgList<shared::SPSExecutorAddrRange>>( - RegisterActionAddr, R.getRange())), + shared::SPSArgList<shared::SPSExecutorAddrRange, bool>>( + RegisterActionAddr, R.getRange(), AutoRegisterCode)), {}}); return Error::success(); } @@ -377,11 +378,11 @@ GDBJITDebugInfoRegistrationPlugin::Create(ExecutionSession &ES, ? ES.intern("_llvm_orc_registerJITLoaderGDBAllocAction") : ES.intern("llvm_orc_registerJITLoaderGDBAllocAction"); - if (auto Addr = ES.lookup({&ProcessJD}, RegisterActionAddr)) + if (auto RegisterSym = ES.lookup({&ProcessJD}, RegisterActionAddr)) return std::make_unique<GDBJITDebugInfoRegistrationPlugin>( - ExecutorAddr(Addr->getAddress())); + RegisterSym->getAddress()); else - return Addr.takeError(); + return RegisterSym.takeError(); } Error GDBJITDebugInfoRegistrationPlugin::notifyFailed( diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp index 00032e4dca3f..1bb4ecdff299 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp @@ -14,6 +14,7 @@ #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" #include <optional> @@ -40,8 +41,7 @@ public: unsigned PointerSize; support::endianness Endianness; jitlink::Edge::Kind EdgeKind; - const auto &TT = - ENP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + const auto &TT = ENP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::x86_64: @@ -96,31 +96,24 @@ private: ELFNixPlatform &ENP; }; -StringRef EHFrameSectionName = ".eh_frame"; -StringRef InitArrayFuncSectionName = ".init_array"; - -StringRef ThreadBSSSectionName = ".tbss"; -StringRef ThreadDataSectionName = ".tdata"; - } // end anonymous namespace namespace llvm { namespace orc { -Expected<std::unique_ptr<ELFNixPlatform>> -ELFNixPlatform::Create(ExecutionSession &ES, - ObjectLinkingLayer &ObjLinkingLayer, - JITDylib &PlatformJD, const char *OrcRuntimePath, - std::optional<SymbolAliasMap> RuntimeAliases) { - - auto &EPC = ES.getExecutorProcessControl(); +Expected<std::unique_ptr<ELFNixPlatform>> ELFNixPlatform::Create( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime, + std::optional<SymbolAliasMap> RuntimeAliases) { // If the target is not supported then bail out immediately. - if (!supportedTarget(EPC.getTargetTriple())) + if (!supportedTarget(ES.getTargetTriple())) return make_error<StringError>("Unsupported ELFNixPlatform triple: " + - EPC.getTargetTriple().str(), + ES.getTargetTriple().str(), inconvertibleErrorCode()); + auto &EPC = ES.getExecutorProcessControl(); + // Create default aliases if the caller didn't supply any. if (!RuntimeAliases) { auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD); @@ -134,31 +127,41 @@ ELFNixPlatform::Create(ExecutionSession &ES, return std::move(Err); // Add JIT-dispatch function support symbols. - if (auto Err = PlatformJD.define(absoluteSymbols( - {{ES.intern("__orc_rt_jit_dispatch"), - {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), - JITSymbolFlags::Exported}}, - {ES.intern("__orc_rt_jit_dispatch_ctx"), - {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), - JITSymbolFlags::Exported}}}))) + if (auto Err = PlatformJD.define( + absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunction, + JITSymbolFlags::Exported}}, + {ES.intern("__orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContext, + JITSymbolFlags::Exported}}}))) return std::move(Err); - // Create a generator for the ORC runtime archive. - auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load( - ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple()); - if (!OrcRuntimeArchiveGenerator) - return OrcRuntimeArchiveGenerator.takeError(); - // Create the instance. Error Err = Error::success(); - auto P = std::unique_ptr<ELFNixPlatform>( - new ELFNixPlatform(ES, ObjLinkingLayer, PlatformJD, - std::move(*OrcRuntimeArchiveGenerator), Err)); + auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform( + ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err)); if (Err) return std::move(Err); return std::move(P); } +Expected<std::unique_ptr<ELFNixPlatform>> +ELFNixPlatform::Create(ExecutionSession &ES, + ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + std::optional<SymbolAliasMap> RuntimeAliases) { + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = + StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + return Create(ES, ObjLinkingLayer, PlatformJD, + std::move(*OrcRuntimeArchiveGenerator), + std::move(RuntimeAliases)); +} + Error ELFNixPlatform::setupJITDylib(JITDylib &JD) { return JD.define( std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol)); @@ -204,47 +207,6 @@ ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES, SymbolAliasMap Aliases; addAliases(ES, Aliases, requiredCXXAliases()); addAliases(ES, Aliases, standardRuntimeUtilityAliases()); - - // Determine whether or not the libunwind extended-API function for - // dynamically registering an entire .eh_frame section is available. - // If it is not, we assume that libgcc_s is being used, and alias to - // its __register_frame with the same functionality. - auto RTRegisterFrame = ES.intern("__orc_rt_register_eh_frame_section"); - auto LibUnwindRegisterFrame = ES.intern("__unw_add_dynamic_eh_frame_section"); - auto RTDeregisterFrame = ES.intern("__orc_rt_deregister_eh_frame_section"); - auto LibUnwindDeregisterFrame = - ES.intern("__unw_remove_dynamic_eh_frame_section"); - auto SM = ES.lookup(makeJITDylibSearchOrder(&PlatformJD), - SymbolLookupSet() - .add(LibUnwindRegisterFrame, - SymbolLookupFlags::WeaklyReferencedSymbol) - .add(LibUnwindDeregisterFrame, - SymbolLookupFlags::WeaklyReferencedSymbol)); - if (!SM) { // Weak-ref means no "missing symbol" errors, so this must be - // something more serious that we should report. - return SM.takeError(); - } else if (SM->size() == 2) { - LLVM_DEBUG({ - dbgs() << "Using libunwind " << LibUnwindRegisterFrame - << " for unwind info registration\n"; - }); - Aliases[std::move(RTRegisterFrame)] = {LibUnwindRegisterFrame, - JITSymbolFlags::Exported}; - Aliases[std::move(RTDeregisterFrame)] = {LibUnwindDeregisterFrame, - JITSymbolFlags::Exported}; - } else { - // Since LLVM libunwind is not present, we assume that unwinding - // is provided by libgcc - LLVM_DEBUG({ - dbgs() << "Using libgcc __register_frame" - << " for unwind info registration\n"; - }); - Aliases[std::move(RTRegisterFrame)] = {ES.intern("__register_frame"), - JITSymbolFlags::Exported}; - Aliases[std::move(RTDeregisterFrame)] = {ES.intern("__deregister_frame"), - JITSymbolFlags::Exported}; - } - return Aliases; } @@ -272,13 +234,6 @@ ELFNixPlatform::standardRuntimeUtilityAliases() { StandardRuntimeUtilityAliases); } -bool ELFNixPlatform::isInitializerSection(StringRef SecName) { - if (SecName.consume_front(InitArrayFuncSectionName) && - (SecName.empty() || SecName[0] == '.')) - return true; - return false; -} - bool ELFNixPlatform::supportedTarget(const Triple &TT) { switch (TT.getArch()) { case Triple::x86_64: @@ -433,8 +388,7 @@ void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, void ELFNixPlatform::rt_getDeinitializers( SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) { LLVM_DEBUG({ - dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" - << formatv("{0:x}", Handle.getValue()) << "\")\n"; + dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" << Handle << "\")\n"; }); JITDylib *JD = nullptr; @@ -447,12 +401,9 @@ void ELFNixPlatform::rt_getDeinitializers( } if (!JD) { - LLVM_DEBUG({ - dbgs() << " No JITDylib for handle " - << formatv("{0:x}", Handle.getValue()) << "\n"; - }); + LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + - formatv("{0:x}", Handle.getValue()), + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } @@ -464,8 +415,7 @@ void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, StringRef SymbolName) { LLVM_DEBUG({ - dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" - << formatv("{0:x}", Handle.getValue()) << "\")\n"; + dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"; }); JITDylib *JD = nullptr; @@ -478,12 +428,9 @@ void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, } if (!JD) { - LLVM_DEBUG({ - dbgs() << " No JITDylib for handle " - << formatv("{0:x}", Handle.getValue()) << "\n"; - }); + LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + - formatv("{0:x}", Handle.getValue()), + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } @@ -496,7 +443,7 @@ void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, void operator()(Expected<SymbolMap> Result) { if (Result) { assert(Result->size() == 1 && "Unexpected result map count"); - SendResult(ExecutorAddr(Result->begin()->second.getAddress())); + SendResult(Result->begin()->second.getAddress()); } else { SendResult(Result.takeError()); } @@ -538,7 +485,7 @@ Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) { for (const auto &KV : AddrsToRecord) { auto &Name = KV.first; assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?"); - KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress()); + *KV.second = (*RuntimeSymbolAddrs)[Name].getAddress(); } auto PJDDSOHandle = ES.lookup( @@ -547,7 +494,8 @@ Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) { return PJDDSOHandle.takeError(); if (auto Err = ES.callSPSWrapper<void(uint64_t)>( - orc_rt_elfnix_platform_bootstrap, PJDDSOHandle->getAddress())) + orc_rt_elfnix_platform_bootstrap, + PJDDSOHandle->getAddress().getValue())) return Err; // FIXME: Ordering is fuzzy here. We're probably best off saying @@ -596,8 +544,7 @@ Error ELFNixPlatform::registerInitInfo( for (auto *Sec : InitSections) { // FIXME: Avoid copy here. jitlink::SectionRange R(*Sec); - InitSeq->InitSections[Sec->getName()].push_back( - {ExecutorAddr(R.getStart()), ExecutorAddr(R.getEnd())}); + InitSeq->InitSections[Sec->getName()].push_back(R.getRange()); } return Error::success(); @@ -724,20 +671,19 @@ void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses( Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error { ELFPerObjectSectionsToRegister POSR; - if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { + if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) { jitlink::SectionRange R(*EHFrameSection); if (!R.empty()) - POSR.EHFrameSection = {ExecutorAddr(R.getStart()), - ExecutorAddr(R.getEnd())}; + POSR.EHFrameSection = R.getRange(); } // Get a pointer to the thread data section if there is one. It will be used // below. jitlink::Section *ThreadDataSection = - G.findSectionByName(ThreadDataSectionName); + G.findSectionByName(ELFThreadDataSectionName); // Handle thread BSS section if there is one. - if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) { + if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) { // If there's already a thread data section in this graph then merge the // thread BSS section content into it, otherwise just treat the thread // BSS section as the thread data section. @@ -752,8 +698,7 @@ void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses( if (ThreadDataSection) { jitlink::SectionRange R(*ThreadDataSection); if (!R.empty()) - POSR.ThreadDataSection = {ExecutorAddr(R.getStart()), - ExecutorAddr(R.getEnd())}; + POSR.ThreadDataSection = R.getRange(); } if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) { @@ -781,7 +726,7 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections( JITLinkSymbolSet InitSectionSymbols; for (auto &InitSection : G.sections()) { // Skip non-init sections. - if (!isInitializerSection(InitSection.getName())) + if (!isELFInitializerSection(InitSection.getName())) continue; // Make a pass over live symbols in the section: those blocks are already @@ -816,10 +761,10 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections( SmallVector<jitlink::Section *> InitSections; - LLVM_DEBUG({ dbgs() << "ELFNixPlatform::registerInitSections\n"; }); + LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n"); for (auto &Sec : G.sections()) { - if (isInitializerSection(Sec.getName())) { + if (isELFInitializerSection(Sec.getName())) { InitSections.push_back(&Sec); } } @@ -829,8 +774,7 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections( dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n"; for (auto *Sec : InitSections) { jitlink::SectionRange R(*Sec); - dbgs() << " " << Sec->getName() << ": " - << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n"; + dbgs() << " " << Sec->getName() << ": " << R.getRange() << "\n"; } }); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp index 30d641ee00cf..b8969de54936 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp @@ -45,14 +45,13 @@ Expected<std::unique_ptr<EPCDebugObjectRegistrar>> createJITLoaderGDBRegistrar( assert((*Result)[0].size() == 1 && "Unexpected number of addresses in result"); - return std::make_unique<EPCDebugObjectRegistrar>( - ES, ExecutorAddr((*Result)[0][0])); + return std::make_unique<EPCDebugObjectRegistrar>(ES, (*Result)[0][0]); } -Error EPCDebugObjectRegistrar::registerDebugObject( - ExecutorAddrRange TargetMem) { - return ES.callSPSWrapper<void(shared::SPSExecutorAddrRange)>(RegisterFn, - TargetMem); +Error EPCDebugObjectRegistrar::registerDebugObject(ExecutorAddrRange TargetMem, + bool AutoRegisterCode) { + return ES.callSPSWrapper<void(shared::SPSExecutorAddrRange, bool)>( + RegisterFn, TargetMem, AutoRegisterCode); } } // namespace orc diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp index 1adcc9156957..46e16a55c7e1 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp @@ -53,8 +53,7 @@ Error EPCDynamicLibrarySearchGenerator::tryToGenerate( auto ResultI = Result->front().begin(); for (auto &KV : LookupSymbols) { if (*ResultI) - NewSymbols[KV.first] = - JITEvaluatedSymbol(ResultI->getValue(), JITSymbolFlags::Exported); + NewSymbols[KV.first] = {*ResultI, JITSymbolFlags::Exported}; ++ResultI; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp index 3aa94a7f43e2..56cd982cd5e1 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp @@ -57,9 +57,8 @@ Expected<std::unique_ptr<EPCEHFrameRegistrar>> EPCEHFrameRegistrar::Create( auto RegisterEHFrameWrapperFnAddr = (*Result)[0][0]; auto DeregisterEHFrameWrapperFnAddr = (*Result)[0][1]; - return std::make_unique<EPCEHFrameRegistrar>( - ES, ExecutorAddr(RegisterEHFrameWrapperFnAddr), - ExecutorAddr(DeregisterEHFrameWrapperFnAddr)); + return std::make_unique<EPCEHFrameRegistrar>(ES, RegisterEHFrameWrapperFnAddr, + DeregisterEHFrameWrapperFnAddr); } Error EPCEHFrameRegistrar::registerEHFrames(ExecutorAddrRange EHFrameSection) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp index a3d857c3bfc4..b05f08fd7cdf 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp @@ -158,7 +158,7 @@ void EPCGenericJITLinkMemoryManager::completeAllocation( auto &SegInfo = SegInfos[AG]; SegInfo.ContentSize = Seg.ContentSize; SegInfo.ZeroFillSize = Seg.ZeroFillSize; - SegInfo.Addr = ExecutorAddr(Seg.Addr); + SegInfo.Addr = Seg.Addr; SegInfo.WorkingMem = Seg.WorkingMem; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp index ec82081937e2..fbe25d70c38a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp @@ -235,7 +235,7 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { for (unsigned I = 0; I != 3; ++I) { FR.Segments.push_back({}); auto &Seg = FR.Segments.back(); - Seg.AG = SegMemProts[I]; + Seg.RAG = SegMemProts[I]; Seg.Addr = RemoteAddrs[I]->Start; for (auto &SecAlloc : *SegSections[I]) { Seg.Size = alignTo(Seg.Size, SecAlloc.Align); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp index ddfb30500c7b..833be826f8ae 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp @@ -58,16 +58,16 @@ public: Error deallocateStubs(); - Error createStub(StringRef StubName, JITTargetAddress StubAddr, + Error createStub(StringRef StubName, ExecutorAddr StubAddr, JITSymbolFlags StubFlags) override; Error createStubs(const StubInitsMap &StubInits) override; - JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; + ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override; - JITEvaluatedSymbol findPointer(StringRef Name) override; + ExecutorSymbolDef findPointer(StringRef Name) override; - Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; + Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override; private: using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; @@ -118,12 +118,10 @@ Error EPCTrampolinePool::grow() { unsigned NumTrampolines = TrampolinesPerPage; auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); - EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(), - SegInfo.Addr.getValue(), - ResolverAddress, NumTrampolines); + EPCIU.getABISupport().writeTrampolines( + SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines); for (unsigned I = 0; I < NumTrampolines; ++I) - AvailableTrampolines.push_back(SegInfo.Addr.getValue() + - (I * TrampolineSize)); + AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize)); auto FA = Alloc->finalize(); if (!FA) @@ -135,7 +133,7 @@ Error EPCTrampolinePool::grow() { } Error EPCIndirectStubsManager::createStub(StringRef StubName, - JITTargetAddress StubAddr, + ExecutorAddr StubAddr, JITSymbolFlags StubFlags) { StubInitsMap SIM; SIM[StubName] = std::make_pair(StubAddr, StubFlags); @@ -162,18 +160,16 @@ Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { unsigned ASIdx = 0; std::vector<tpctypes::UInt32Write> PtrUpdates; for (auto &SI : StubInits) - PtrUpdates.push_back( - {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), - static_cast<uint32_t>(SI.second.first)}); + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast<uint32_t>(SI.second.first.getValue())}); return MemAccess.writeUInt32s(PtrUpdates); } case 8: { unsigned ASIdx = 0; std::vector<tpctypes::UInt64Write> PtrUpdates; for (auto &SI : StubInits) - PtrUpdates.push_back( - {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), - static_cast<uint64_t>(SI.second.first)}); + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast<uint64_t>(SI.second.first.getValue())}); return MemAccess.writeUInt64s(PtrUpdates); } default: @@ -182,27 +178,27 @@ Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { } } -JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, - bool ExportedStubsOnly) { +ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name, + bool ExportedStubsOnly) { std::lock_guard<std::mutex> Lock(ISMMutex); auto I = StubInfos.find(Name); if (I == StubInfos.end()) - return nullptr; + return ExecutorSymbolDef(); return {I->second.first.StubAddress, I->second.second}; } -JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { +ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) { std::lock_guard<std::mutex> Lock(ISMMutex); auto I = StubInfos.find(Name); if (I == StubInfos.end()) - return nullptr; + return ExecutorSymbolDef(); return {I->second.first.PointerAddress, I->second.second}; } Error EPCIndirectStubsManager::updatePointer(StringRef Name, - JITTargetAddress NewAddr) { + ExecutorAddr NewAddr) { - JITTargetAddress PtrAddr = 0; + ExecutorAddr PtrAddr; { std::lock_guard<std::mutex> Lock(ISMMutex); auto I = StubInfos.find(Name); @@ -215,11 +211,11 @@ Error EPCIndirectStubsManager::updatePointer(StringRef Name, auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); switch (EPCIU.getABISupport().getPointerSize()) { case 4: { - tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); + tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue()); return MemAccess.writeUInt32s(PUpdate); } case 8: { - tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); + tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue()); return MemAccess.writeUInt64s(PUpdate); } default: @@ -290,9 +286,9 @@ Error EPCIndirectionUtils::cleanup() { return Err; } -Expected<JITTargetAddress> -EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { +Expected<ExecutorAddr> +EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { using namespace jitlink; assert(ABI && "ABI can not be null"); @@ -307,7 +303,7 @@ EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, return Alloc.takeError(); auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); - ResolverBlockAddr = SegInfo.Addr.getValue(); + ResolverBlockAddr = SegInfo.Addr; ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr, ReentryFnAddr, ReentryCtxAddr); @@ -331,7 +327,7 @@ TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { } LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( - ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { + ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) { assert(!LCTM && "createLazyCallThroughManager can not have been called before"); LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, @@ -377,9 +373,8 @@ EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { auto StubSeg = Alloc->getSegInfo(StubProt); auto PtrSeg = Alloc->getSegInfo(PtrProt); - ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), - StubSeg.Addr.getValue(), - PtrSeg.Addr.getValue(), NumStubsToAllocate); + ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr, + PtrSeg.Addr, NumStubsToAllocate); auto FA = Alloc->finalize(); if (!FA) @@ -390,8 +385,8 @@ EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { auto StubExecutorAddr = StubSeg.Addr; auto PtrExecutorAddr = PtrSeg.Addr; for (unsigned I = 0; I != NumStubsToAllocate; ++I) { - AvailableIndirectStubs.push_back(IndirectStubInfo( - StubExecutorAddr.getValue(), PtrExecutorAddr.getValue())); + AvailableIndirectStubs.push_back( + IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr)); StubExecutorAddr += ABI->getStubSize(); PtrExecutorAddr += ABI->getPointerSize(); } @@ -412,19 +407,19 @@ EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { static JITTargetAddress reentry(JITTargetAddress LCTMAddr, JITTargetAddress TrampolineAddr) { auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); - std::promise<JITTargetAddress> LandingAddrP; + std::promise<ExecutorAddr> LandingAddrP; auto LandingAddrF = LandingAddrP.get_future(); LCTM.resolveTrampolineLandingAddress( - TrampolineAddr, - [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); - return LandingAddrF.get(); + ExecutorAddr(TrampolineAddr), + [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); }); + return LandingAddrF.get().getValue(); } Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { auto &LCTM = EPCIU.getLazyCallThroughManager(); return EPCIU - .writeResolverBlock(pointerToJITTargetAddress(&reentry), - pointerToJITTargetAddress(&LCTM)) + .writeResolverBlock(ExecutorAddr::fromPtr(&reentry), + ExecutorAddr::fromPtr(&LCTM)) .takeError(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 377a59993eb0..fb685e6c3727 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -156,8 +156,7 @@ Error CtorDtorRunner::run() { for (auto &KV : CtorDtorsByPriority) { for (auto &Name : KV.second) { assert(CtorDtorMap->count(Name) && "No entry for Name"); - auto CtorDtor = reinterpret_cast<CtorDtorTy>( - static_cast<uintptr_t>((*CtorDtorMap)[Name].getAddress())); + auto CtorDtor = (*CtorDtorMap)[Name].getAddress().toPtr<CtorDtorTy>(); CtorDtor(); } } @@ -186,12 +185,10 @@ int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor, Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, MangleAndInterner &Mangle) { SymbolMap RuntimeInterposes; - RuntimeInterposes[Mangle("__dso_handle")] = - JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride), - JITSymbolFlags::Exported); - RuntimeInterposes[Mangle("__cxa_atexit")] = - JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride), - JITSymbolFlags::Exported); + RuntimeInterposes[Mangle("__dso_handle")] = { + ExecutorAddr::fromPtr(&DSOHandleOverride), JITSymbolFlags::Exported}; + RuntimeInterposes[Mangle("__cxa_atexit")] = { + ExecutorAddr::fromPtr(&CXAAtExitOverride), JITSymbolFlags::Exported}; return JD.define(absoluteSymbols(std::move(RuntimeInterposes))); } @@ -257,11 +254,8 @@ Error DynamicLibrarySearchGenerator::tryToGenerate( std::string Tmp((*Name).data() + HasGlobalPrefix, (*Name).size() - HasGlobalPrefix); - if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { - NewSymbols[Name] = JITEvaluatedSymbol( - static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Addr)), - JITSymbolFlags::Exported); - } + if (void *P = Dylib.getAddressOfSymbol(Tmp.c_str())) + NewSymbols[Name] = {ExecutorAddr::fromPtr(P), JITSymbolFlags::Exported}; } if (NewSymbols.empty()) @@ -274,57 +268,41 @@ Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> StaticLibraryDefinitionGenerator::Load( ObjectLayer &L, const char *FileName, GetObjectFileInterface GetObjFileInterface) { - auto ArchiveBuffer = MemoryBuffer::getFile(FileName); - - if (!ArchiveBuffer) - return createFileError(FileName, ArchiveBuffer.getError()); - - return Create(L, std::move(*ArchiveBuffer), std::move(GetObjFileInterface)); -} - -Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> -StaticLibraryDefinitionGenerator::Load( - ObjectLayer &L, const char *FileName, const Triple &TT, - GetObjectFileInterface GetObjFileInterface) { auto B = object::createBinary(FileName); if (!B) return createFileError(FileName, B.takeError()); // If this is a regular archive then create an instance from it. - if (isa<object::Archive>(B->getBinary())) - return Create(L, std::move(B->takeBinary().second), + if (isa<object::Archive>(B->getBinary())) { + auto [Archive, ArchiveBuffer] = B->takeBinary(); + return Create(L, std::move(ArchiveBuffer), + std::unique_ptr<object::Archive>( + static_cast<object::Archive *>(Archive.release())), std::move(GetObjFileInterface)); + } // If this is a universal binary then search for a slice matching the given // Triple. if (auto *UB = cast<object::MachOUniversalBinary>(B->getBinary())) { - for (const auto &Obj : UB->objects()) { - auto ObjTT = Obj.getTriple(); - if (ObjTT.getArch() == TT.getArch() && - ObjTT.getSubArch() == TT.getSubArch() && - (TT.getVendor() == Triple::UnknownVendor || - ObjTT.getVendor() == TT.getVendor())) { - // We found a match. Create an instance from a buffer covering this - // slice. - auto SliceBuffer = MemoryBuffer::getFileSlice(FileName, Obj.getSize(), - Obj.getOffset()); - if (!SliceBuffer) - return make_error<StringError>( - Twine("Could not create buffer for ") + TT.str() + " slice of " + - FileName + ": [ " + formatv("{0:x}", Obj.getOffset()) + - " .. " + formatv("{0:x}", Obj.getOffset() + Obj.getSize()) + - ": " + SliceBuffer.getError().message(), - SliceBuffer.getError()); - return Create(L, std::move(*SliceBuffer), - std::move(GetObjFileInterface)); - } - } - return make_error<StringError>(Twine("Universal binary ") + FileName + - " does not contain a slice for " + - TT.str(), - inconvertibleErrorCode()); + const auto &TT = L.getExecutionSession().getTargetTriple(); + + auto SliceRange = getSliceRangeForArch(*UB, TT); + if (!SliceRange) + return SliceRange.takeError(); + + auto SliceBuffer = MemoryBuffer::getFileSlice(FileName, SliceRange->second, + SliceRange->first); + if (!SliceBuffer) + return make_error<StringError>( + Twine("Could not create buffer for ") + TT.str() + " slice of " + + FileName + ": [ " + formatv("{0:x}", SliceRange->first) + " .. " + + formatv("{0:x}", SliceRange->first + SliceRange->second) + ": " + + SliceBuffer.getError().message(), + SliceBuffer.getError()); + + return Create(L, std::move(*SliceBuffer), std::move(GetObjFileInterface)); } return make_error<StringError>(Twine("Unrecognized file type for ") + @@ -335,12 +313,15 @@ StaticLibraryDefinitionGenerator::Load( Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> StaticLibraryDefinitionGenerator::Create( ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, + std::unique_ptr<object::Archive> Archive, GetObjectFileInterface GetObjFileInterface) { + Error Err = Error::success(); std::unique_ptr<StaticLibraryDefinitionGenerator> ADG( new StaticLibraryDefinitionGenerator( - L, std::move(ArchiveBuffer), std::move(GetObjFileInterface), Err)); + L, std::move(ArchiveBuffer), std::move(Archive), + std::move(GetObjFileInterface), Err)); if (Err) return std::move(Err); @@ -348,6 +329,50 @@ StaticLibraryDefinitionGenerator::Create( return std::move(ADG); } +Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> +StaticLibraryDefinitionGenerator::Create( + ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, + GetObjectFileInterface GetObjFileInterface) { + + auto B = object::createBinary(ArchiveBuffer->getMemBufferRef()); + if (!B) + return B.takeError(); + + // If this is a regular archive then create an instance from it. + if (isa<object::Archive>(*B)) + return Create(L, std::move(ArchiveBuffer), + std::unique_ptr<object::Archive>( + static_cast<object::Archive *>(B->release())), + std::move(GetObjFileInterface)); + + // If this is a universal binary then search for a slice matching the given + // Triple. + if (auto *UB = cast<object::MachOUniversalBinary>(B->get())) { + + const auto &TT = L.getExecutionSession().getTargetTriple(); + + auto SliceRange = getSliceRangeForArch(*UB, TT); + if (!SliceRange) + return SliceRange.takeError(); + + MemoryBufferRef SliceRef( + StringRef(ArchiveBuffer->getBufferStart() + SliceRange->first, + SliceRange->second), + ArchiveBuffer->getBufferIdentifier()); + + auto Archive = object::Archive::create(SliceRef); + if (!Archive) + return Archive.takeError(); + + return Create(L, std::move(ArchiveBuffer), std::move(*Archive), + std::move(GetObjFileInterface)); + } + + return make_error<StringError>(Twine("Unrecognized file type for ") + + ArchiveBuffer->getBufferIdentifier(), + inconvertibleErrorCode()); +} + Error StaticLibraryDefinitionGenerator::tryToGenerate( LookupState &LS, LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { @@ -417,12 +442,33 @@ Error StaticLibraryDefinitionGenerator::buildObjectFilesMap() { return Error::success(); } +Expected<std::pair<size_t, size_t>> +StaticLibraryDefinitionGenerator::getSliceRangeForArch( + object::MachOUniversalBinary &UB, const Triple &TT) { + + for (const auto &Obj : UB.objects()) { + auto ObjTT = Obj.getTriple(); + if (ObjTT.getArch() == TT.getArch() && + ObjTT.getSubArch() == TT.getSubArch() && + (TT.getVendor() == Triple::UnknownVendor || + ObjTT.getVendor() == TT.getVendor())) { + // We found a match. Return the range for the slice. + return std::make_pair(Obj.getOffset(), Obj.getSize()); + } + } + + return make_error<StringError>(Twine("Universal binary ") + UB.getFileName() + + " does not contain a slice for " + + TT.str(), + inconvertibleErrorCode()); +} + StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator( ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, + std::unique_ptr<object::Archive> Archive, GetObjectFileInterface GetObjFileInterface, Error &Err) : L(L), GetObjFileInterface(std::move(GetObjFileInterface)), - ArchiveBuffer(std::move(ArchiveBuffer)), - Archive(std::make_unique<object::Archive>(*this->ArchiveBuffer, Err)) { + ArchiveBuffer(std::move(ArchiveBuffer)), Archive(std::move(Archive)) { ErrorAsOutParameter _(&Err); if (!this->GetObjFileInterface) this->GetObjFileInterface = getObjectFileInterface; @@ -506,7 +552,7 @@ DLLImportDefinitionGenerator::getTargetEndianness(const Triple &TT) { Expected<std::unique_ptr<jitlink::LinkGraph>> DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) { - Triple TT = ES.getExecutorProcessControl().getTargetTriple(); + Triple TT = ES.getTargetTriple(); auto PointerSize = getTargetEndianness(TT); if (!PointerSize) return PointerSize.takeError(); @@ -522,13 +568,13 @@ DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) { for (auto &KV : Resolved) { jitlink::Symbol &Target = G->addAbsoluteSymbol( - *KV.first, ExecutorAddr(KV.second.getAddress()), *PointerSize, + *KV.first, KV.second.getAddress(), *PointerSize, jitlink::Linkage::Strong, jitlink::Scope::Local, false); // Create __imp_ symbol jitlink::Symbol &Ptr = jitlink::x86_64::createAnonymousPointer(*G, Sec, &Target); - auto NameCopy = G->allocateString(Twine(getImpPrefix()) + *KV.first); + auto NameCopy = G->allocateContent(Twine(getImpPrefix()) + *KV.first); StringRef NameCopyRef = StringRef(NameCopy.data(), NameCopy.size()); Ptr.setName(NameCopyRef); Ptr.setLinkage(jitlink::Linkage::Strong); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index 361fcd4a2e9c..b8b013f8a7a9 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -11,8 +11,8 @@ #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/Host.h" #include "llvm/Support/Process.h" +#include "llvm/TargetParser/Host.h" #define DEBUG_TYPE "orc" @@ -192,7 +192,7 @@ SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager( shared::WrapperFunctionResult Result) mutable { ResultP.set_value(std::move(Result)); }, - pointerToJITTargetAddress(FnTag), {Data, Size}); + ExecutorAddr::fromPtr(FnTag), {Data, Size}); return ResultF.get().release(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 989bb094cc25..a0d81cdf2086 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -8,13 +8,13 @@ #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" #include "llvm/IR/IRBuilder.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" #include "llvm/MC/MCInstrAnalysis.h" #include "llvm/Support/Format.h" +#include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/Cloning.h" #include <sstream> @@ -40,7 +40,7 @@ public: private: void materialize(std::unique_ptr<MaterializationResponsibility> R) override { SymbolMap Result; - Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); + Result[Name] = {Compile(), JITSymbolFlags::Exported}; // No dependencies, so these calls cannot fail. cantFail(R->notifyResolved(Result)); cantFail(R->notifyEmitted()); @@ -62,7 +62,7 @@ namespace orc { TrampolinePool::~TrampolinePool() = default; void IndirectStubsManager::anchor() {} -Expected<JITTargetAddress> +Expected<ExecutorAddr> JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { if (auto TrampolineAddr = TP->getTrampoline()) { auto CallbackName = @@ -78,8 +78,8 @@ JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { return TrampolineAddr.takeError(); } -JITTargetAddress JITCompileCallbackManager::executeCompileCallback( - JITTargetAddress TrampolineAddr) { +ExecutorAddr +JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) { SymbolStringPtr Name; { @@ -91,14 +91,10 @@ JITTargetAddress JITCompileCallbackManager::executeCompileCallback( // callee. if (I == AddrToSymbol.end()) { Lock.unlock(); - std::string ErrMsg; - { - raw_string_ostream ErrMsgStream(ErrMsg); - ErrMsgStream << "No compile callback for trampoline at " - << format("0x%016" PRIx64, TrampolineAddr); - } ES.reportError( - make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); + make_error<StringError>("No compile callback for trampoline at " + + formatv("{0:x}", TrampolineAddr), + inconvertibleErrorCode())); return ErrorHandlerAddress; } else Name = I->second; @@ -120,7 +116,7 @@ JITTargetAddress JITCompileCallbackManager::executeCompileCallback( Expected<std::unique_ptr<JITCompileCallbackManager>> createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, - JITTargetAddress ErrorHandlerAddress) { + ExecutorAddr ErrorHandlerAddress) { switch (T.getArch()) { default: return make_error<StringError>( @@ -244,9 +240,9 @@ createLocalIndirectStubsManagerBuilder(const Triple &T) { } } -Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) { +Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) { Constant *AddrIntVal = - ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); + ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue()); Constant *AddrPtrVal = ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, PointerType::get(&FT, 0)); @@ -329,26 +325,6 @@ Function* cloneFunctionDecl(Module &Dst, const Function &F, return NewF; } -void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, - ValueMaterializer *Materializer, - Function *NewF) { - assert(!OrigF.isDeclaration() && "Nothing to move"); - if (!NewF) - NewF = cast<Function>(VMap[&OrigF]); - else - assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap."); - assert(NewF && "Function mapping missing from VMap."); - assert(NewF->getParent() != OrigF.getParent() && - "moveFunctionBody should only be used to move bodies between " - "modules."); - - SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned. - CloneFunctionInto(NewF, &OrigF, VMap, - CloneFunctionChangeType::DifferentModule, Returns, "", - nullptr, nullptr, Materializer); - OrigF.deleteBody(); -} - GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, ValueToValueMapTy *VMap) { GlobalVariable *NewGV = new GlobalVariable( @@ -361,24 +337,6 @@ GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, return NewGV; } -void moveGlobalVariableInitializer(GlobalVariable &OrigGV, - ValueToValueMapTy &VMap, - ValueMaterializer *Materializer, - GlobalVariable *NewGV) { - assert(OrigGV.hasInitializer() && "Nothing to move"); - if (!NewGV) - NewGV = cast<GlobalVariable>(VMap[&OrigGV]); - else - assert(VMap[&OrigGV] == NewGV && - "Incorrect global variable mapping in VMap."); - assert(NewGV->getParent() != OrigGV.getParent() && - "moveGlobalVariableInitializer should only be used to move " - "initializers between modules"); - - NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, - nullptr, Materializer)); -} - GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, ValueToValueMapTy &VMap) { assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); @@ -390,15 +348,6 @@ GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, return NewA; } -void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, - ValueToValueMapTy &VMap) { - auto *MFs = Src.getModuleFlagsMetadata(); - if (!MFs) - return; - for (auto *MF : MFs->operands()) - Dst.addModuleFlag(MapMetadata(MF, VMap)); -} - Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, jitlink::LinkGraph &G, MCDisassembler &Disassembler, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index 70a3c404d836..b66f52f1ec5d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -9,8 +9,8 @@ #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/MC/TargetRegistry.h" -#include "llvm/Support/Host.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/Host.h" namespace llvm { namespace orc { @@ -18,13 +18,10 @@ namespace orc { JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT) : TT(std::move(TT)) { Options.EmulatedTLS = true; - Options.ExplicitEmulatedTLS = true; Options.UseInitArray = true; } Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { - // FIXME: getProcessTriple is bogus. It returns the host LLVM was compiled on, - // rather than a valid triple for the current process. JITTargetMachineBuilder TMBuilder((Triple(sys::getProcessTriple()))); // Retrieve host CPU name and sub-target features and add them to builder. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index bc84988e3254..7c7c2f000368 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -9,6 +9,11 @@ #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/COFFPlatform.h" +#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" +#include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h" +#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" +#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" @@ -98,10 +103,16 @@ public: ORC_RT_RTLD_GLOBAL = 0x8 }; - if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlopen_wrapper")) { - return J.getExecutionSession().callSPSWrapper<SPSDLOpenSig>( - *WrapperAddr, DSOHandles[&JD], JD.getName(), - int32_t(ORC_RT_RTLD_LAZY)); + auto &ES = J.getExecutionSession(); + auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo( + [](const JITDylibSearchOrder &SO) { return SO; }); + + if (auto WrapperAddr = + ES.lookup(MainSearchOrder, + J.mangleAndIntern("__orc_rt_jit_dlopen_wrapper"))) { + return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(), + DSOHandles[&JD], JD.getName(), + int32_t(ORC_RT_RTLD_LAZY)); } else return WrapperAddr.takeError(); } @@ -110,10 +121,16 @@ public: using llvm::orc::shared::SPSExecutorAddr; using SPSDLCloseSig = int32_t(SPSExecutorAddr); - if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlclose_wrapper")) { + auto &ES = J.getExecutionSession(); + auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo( + [](const JITDylibSearchOrder &SO) { return SO; }); + + if (auto WrapperAddr = + ES.lookup(MainSearchOrder, + J.mangleAndIntern("__orc_rt_jit_dlclose_wrapper"))) { int32_t result; auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>( - *WrapperAddr, result, DSOHandles[&JD]); + WrapperAddr->getAddress(), result, DSOHandles[&JD]); if (E) return E; else if (result) @@ -176,7 +193,7 @@ private: /// some runtime API, including __cxa_atexit, dlopen, and dlclose. class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { public: - GenericLLVMIRPlatformSupport(LLJIT &J) + GenericLLVMIRPlatformSupport(LLJIT &J, JITDylib &PlatformJD) : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")), DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) { @@ -188,17 +205,14 @@ public: SymbolMap StdInterposes; - StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(this), - JITSymbolFlags::Exported); - StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(registerCxaAtExitHelper), - JITSymbolFlags()); - - cantFail( - J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes)))); - cantFail(setupJITDylib(J.getMainJITDylib())); - cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule())); + StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = { + ExecutorAddr::fromPtr(this), JITSymbolFlags::Exported}; + StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = { + ExecutorAddr::fromPtr(registerCxaAtExitHelper), JITSymbolFlags()}; + + cantFail(PlatformJD.define(absoluteSymbols(std::move(StdInterposes)))); + cantFail(setupJITDylib(PlatformJD)); + cantFail(J.addIRModule(PlatformJD, createPlatformRuntimeModule())); } ExecutionSession &getExecutionSession() { return J.getExecutionSession(); } @@ -208,12 +222,10 @@ public: // Add per-jitdylib standard interposes. SymbolMap PerJDInterposes; - PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper), - JITSymbolFlags()); - PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper), - JITSymbolFlags()); + PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = { + ExecutorAddr::fromPtr(runAtExitsHelper), JITSymbolFlags()}; + PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] = { + ExecutorAddr::fromPtr(registerAtExitHelper), JITSymbolFlags()}; cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes)))); auto Ctx = std::make_unique<LLVMContext>(); @@ -227,7 +239,7 @@ public: "__dso_handle"); DSOHandle->setVisibility(GlobalValue::DefaultVisibility); DSOHandle->setInitializer( - ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD))); + ConstantInt::get(Int64Ty, ExecutorAddr::fromPtr(&JD).getValue())); auto *GenericIRPlatformSupportTy = StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); @@ -287,7 +299,7 @@ public: dbgs() << " Running init " << formatv("{0:x16}", InitFnAddr) << "...\n"; }); - auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr); + auto *InitFn = InitFnAddr.toPtr<void (*)()>(); InitFn(); } } else @@ -308,7 +320,7 @@ public: dbgs() << " Running deinit " << formatv("{0:x16}", DeinitFnAddr) << "...\n"; }); - auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr); + auto *DeinitFn = DeinitFnAddr.toPtr<void (*)()>(); DeinitFn(); } } else @@ -329,8 +341,7 @@ public: } private: - - Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) { + Expected<std::vector<ExecutorAddr>> getInitializers(JITDylib &JD) { if (auto Err = issueInitLookups(JD)) return std::move(Err); @@ -370,7 +381,7 @@ private: if (!LookupResult) return LookupResult.takeError(); - std::vector<JITTargetAddress> Initializers; + std::vector<ExecutorAddr> Initializers; while (!DFSLinkOrder.empty()) { auto &NextJD = *DFSLinkOrder.back(); DFSLinkOrder.pop_back(); @@ -384,7 +395,7 @@ private: return Initializers; } - Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) { + Expected<std::vector<ExecutorAddr>> getDeinitializers(JITDylib &JD) { auto &ES = getExecutionSession(); auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits"); @@ -427,7 +438,7 @@ private: if (!LookupResult) return LookupResult.takeError(); - std::vector<JITTargetAddress> DeInitializers; + std::vector<ExecutorAddr> DeInitializers; for (auto &NextJD : DFSLinkOrder) { auto DeInitsItr = LookupResult->find(NextJD.get()); assert(DeInitsItr != LookupResult->end() && @@ -695,6 +706,14 @@ Error LLJITBuilderState::prepareForConstruction() { dbgs() << "\n"; }); + // Create DL if not specified. + if (!DL) { + if (auto DLOrErr = JTMB->getDefaultDataLayoutForTarget()) + DL = std::move(*DLOrErr); + else + return DLOrErr.takeError(); + } + // If neither ES nor EPC has been set then create an EPC instance. if (!ES && !EPC) { LLVM_DEBUG({ @@ -705,21 +724,38 @@ Error LLJITBuilderState::prepareForConstruction() { EPC = std::move(*EPCOrErr); else return EPCOrErr.takeError(); - } else + } else if (EPC) { LLVM_DEBUG({ dbgs() << "Using explicitly specified ExecutorProcessControl instance " << EPC.get() << "\n"; }); + } else { + LLVM_DEBUG({ + dbgs() << "Using explicitly specified ExecutionSession instance " + << ES.get() << "\n"; + }); + } // If the client didn't configure any linker options then auto-configure the // JIT linker. if (!CreateObjectLinkingLayer) { auto &TT = JTMB->getTargetTriple(); - if (TT.getArch() == Triple::riscv64 || - TT.getArch() == Triple::loongarch64 || - (TT.isOSBinFormatMachO() && - (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64))) { - + bool UseJITLink = false; + switch (TT.getArch()) { + case Triple::riscv64: + case Triple::loongarch64: + UseJITLink = true; + break; + case Triple::aarch64: + UseJITLink = !TT.isOSBinFormatCOFF(); + break; + case Triple::x86_64: + UseJITLink = !TT.isOSBinFormatCOFF(); + break; + default: + break; + } + if (UseJITLink) { JTMB->setRelocationModel(Reloc::PIC_); JTMB->setCodeModel(CodeModel::Small); CreateObjectLinkingLayer = @@ -737,6 +773,30 @@ Error LLJITBuilderState::prepareForConstruction() { } } + // If we need a process JITDylib but no setup function has been given then + // create a default one. + if (!SetupProcessSymbolsJITDylib && + (LinkProcessSymbolsByDefault || EnableDebuggerSupport)) { + + LLVM_DEBUG({ + dbgs() << "Creating default Process JD setup function (neeeded for"; + if (LinkProcessSymbolsByDefault) + dbgs() << " <link-process-syms-by-default>"; + if (EnableDebuggerSupport) + dbgs() << " <debugger-support>"; + dbgs() << ")\n"; + }); + + SetupProcessSymbolsJITDylib = [this](JITDylib &JD) -> Error { + auto G = orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( + DL->getGlobalPrefix()); + if (!G) + return G.takeError(); + JD.addGenerator(std::move(*G)); + return Error::success(); + }; + } + return Error::success(); } @@ -747,6 +807,54 @@ LLJIT::~LLJIT() { ES->reportError(std::move(Err)); } +JITDylibSP LLJIT::getProcessSymbolsJITDylib() { return ProcessSymbols; } + +JITDylibSP LLJIT::getPlatformJITDylib() { return Platform; } + +Expected<JITDylib &> LLJIT::createJITDylib(std::string Name) { + auto JD = ES->createJITDylib(std::move(Name)); + if (!JD) + return JD.takeError(); + + JD->addToLinkOrder(DefaultLinks); + return JD; +} + +Expected<JITDylib &> LLJIT::loadPlatformDynamicLibrary(const char *Path) { + auto G = EPCDynamicLibrarySearchGenerator::Load(*ES, Path); + if (!G) + return G.takeError(); + + if (auto *ExistingJD = ES->getJITDylibByName(Path)) + return *ExistingJD; + + auto &JD = ES->createBareJITDylib(Path); + JD.addGenerator(std::move(*G)); + return JD; +} + +Error LLJIT::linkStaticLibraryInto(JITDylib &JD, + std::unique_ptr<MemoryBuffer> LibBuffer) { + auto G = StaticLibraryDefinitionGenerator::Create(*ObjLinkingLayer, + std::move(LibBuffer)); + if (!G) + return G.takeError(); + + JD.addGenerator(std::move(*G)); + + return Error::success(); +} + +Error LLJIT::linkStaticLibraryInto(JITDylib &JD, const char *Path) { + auto G = StaticLibraryDefinitionGenerator::Load(*ObjLinkingLayer, Path); + if (!G) + return G.takeError(); + + JD.addGenerator(std::move(*G)); + + return Error::success(); +} + Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) { assert(TSM && "Can not add null module"); @@ -777,7 +885,7 @@ Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD, if (auto Sym = ES->lookup( makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name)) - return ExecutorAddr(Sym->getAddress()); + return Sym->getAddress(); else return Sym.takeError(); } @@ -832,7 +940,7 @@ LLJIT::createCompileFunction(LLJITBuilderState &S, } LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) - : DL(""), TT(S.JTMB->getTargetTriple()) { + : DL(std::move(*S.DL)), TT(S.JTMB->getTargetTriple()) { ErrorAsOutParameter _(&Err); @@ -851,22 +959,6 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) } } - if (auto MainOrErr = this->ES->createJITDylib("main")) - Main = &*MainOrErr; - else { - Err = MainOrErr.takeError(); - return; - } - - if (S.DL) - DL = std::move(*S.DL); - else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) - DL = std::move(*DLOrErr); - else { - Err = DLOrErr.takeError(); - return; - } - auto ObjLayer = createObjectLinkingLayer(S, *ES); if (!ObjLayer) { Err = ObjLayer.takeError(); @@ -905,10 +997,77 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) }); } - if (S.SetUpPlatform) - Err = S.SetUpPlatform(*this); - else - setUpGenericLLVMIRPlatform(*this); + if (S.SetupProcessSymbolsJITDylib) { + ProcessSymbols = &ES->createBareJITDylib("<Process Symbols>"); + if (auto Err2 = S.SetupProcessSymbolsJITDylib(*ProcessSymbols)) { + Err = std::move(Err2); + return; + } + } + + if (S.EnableDebuggerSupport) { + if (auto *OLL = dyn_cast<ObjectLinkingLayer>(ObjLinkingLayer.get())) { + switch (TT.getObjectFormat()) { + case Triple::ELF: { + auto Registrar = createJITLoaderGDBRegistrar(*ES); + if (!Registrar) { + Err = Registrar.takeError(); + return; + } + OLL->addPlugin(std::make_unique<DebugObjectManagerPlugin>( + *ES, std::move(*Registrar), true, true)); + break; + } + case Triple::MachO: { + assert(ProcessSymbols && "ProcessSymbols JD should be available when " + "EnableDebuggerSupport is set"); + auto DS = + GDBJITDebugInfoRegistrationPlugin::Create(*ES, *ProcessSymbols, TT); + if (!DS) { + Err = DS.takeError(); + return; + } + OLL->addPlugin(std::move(*DS)); + break; + } + default: + LLVM_DEBUG({ + dbgs() << "Cannot enable LLJIT debugger support: " + << Triple::getObjectFormatTypeName(TT.getObjectFormat()) + << " not supported.\n"; + }); + } + } else { + LLVM_DEBUG({ + dbgs() << "Cannot enable LLJIT debugger support: " + " debugger support is only available when using JITLink.\n"; + }); + } + } + + if (!S.SetUpPlatform) + S.SetUpPlatform = setUpGenericLLVMIRPlatform; + + if (auto PlatformJDOrErr = S.SetUpPlatform(*this)) { + Platform = PlatformJDOrErr->get(); + if (Platform) + DefaultLinks.push_back( + {Platform, JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } else { + Err = PlatformJDOrErr.takeError(); + return; + } + + if (S.LinkProcessSymbolsByDefault) + DefaultLinks.push_back( + {ProcessSymbols, JITDylibLookupFlags::MatchExportedSymbolsOnly}); + + if (auto MainOrErr = createJITDylib("main")) + Main = &*MainOrErr; + else { + Err = MainOrErr.takeError(); + return; + } } std::string LLJIT::mangle(StringRef UnmangledName) const { @@ -934,24 +1093,136 @@ Error LLJIT::applyDataLayout(Module &M) { return Error::success(); } -Error setUpOrcPlatform(LLJIT& J) { - LLVM_DEBUG( - { dbgs() << "Setting up orc platform support for LLJIT\n"; }); - J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J)); +Error setUpOrcPlatformManually(LLJIT &J) { + LLVM_DEBUG({ dbgs() << "Setting up orc platform support for LLJIT\n"; }); + J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J)); + return Error::success(); +} + +class LoadAndLinkDynLibrary { +public: + LoadAndLinkDynLibrary(LLJIT &J) : J(J) {} + Error operator()(JITDylib &JD, StringRef DLLName) { + if (!DLLName.ends_with_insensitive(".dll")) + return make_error<StringError>("DLLName not ending with .dll", + inconvertibleErrorCode()); + auto DLLNameStr = DLLName.str(); // Guarantees null-termination. + auto DLLJD = J.loadPlatformDynamicLibrary(DLLNameStr.c_str()); + if (!DLLJD) + return DLLJD.takeError(); + JD.addToLinkOrder(*DLLJD); return Error::success(); + } + +private: + LLJIT &J; +}; + +Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) { + auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib(); + if (!ProcessSymbolsJD) + return make_error<StringError>( + "Native platforms require a process symbols JITDylib", + inconvertibleErrorCode()); + + const Triple &TT = J.getTargetTriple(); + ObjectLinkingLayer *ObjLinkingLayer = + dyn_cast<ObjectLinkingLayer>(&J.getObjLinkingLayer()); + + if (!ObjLinkingLayer) + return make_error<StringError>( + "SetUpTargetPlatform requires ObjectLinkingLayer", + inconvertibleErrorCode()); + + std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer; + if (OrcRuntime.index() == 0) { + auto A = errorOrToExpected(MemoryBuffer::getFile(std::get<0>(OrcRuntime))); + if (!A) + return A.takeError(); + RuntimeArchiveBuffer = std::move(*A); + } else + RuntimeArchiveBuffer = std::move(std::get<1>(OrcRuntime)); + + auto &ES = J.getExecutionSession(); + auto &PlatformJD = ES.createBareJITDylib("<Platform>"); + PlatformJD.addToLinkOrder(*ProcessSymbolsJD); + + J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J)); + + switch (TT.getObjectFormat()) { + case Triple::COFF: { + const char *VCRuntimePath = nullptr; + bool StaticVCRuntime = false; + if (VCRuntime) { + VCRuntimePath = VCRuntime->first.c_str(); + StaticVCRuntime = VCRuntime->second; + } + if (auto P = COFFPlatform::Create( + ES, *ObjLinkingLayer, PlatformJD, std::move(RuntimeArchiveBuffer), + LoadAndLinkDynLibrary(J), StaticVCRuntime, VCRuntimePath)) + J.getExecutionSession().setPlatform(std::move(*P)); + else + return P.takeError(); + break; + } + case Triple::ELF: { + auto G = StaticLibraryDefinitionGenerator::Create( + *ObjLinkingLayer, std::move(RuntimeArchiveBuffer)); + if (!G) + return G.takeError(); + + if (auto P = ELFNixPlatform::Create(ES, *ObjLinkingLayer, PlatformJD, + std::move(*G))) + J.getExecutionSession().setPlatform(std::move(*P)); + else + return P.takeError(); + break; + } + case Triple::MachO: { + auto G = StaticLibraryDefinitionGenerator::Create( + *ObjLinkingLayer, std::move(RuntimeArchiveBuffer)); + if (!G) + return G.takeError(); + + if (auto P = MachOPlatform::Create(ES, *ObjLinkingLayer, PlatformJD, + std::move(*G))) + ES.setPlatform(std::move(*P)); + else + return P.takeError(); + break; + } + default: + return make_error<StringError>("Unsupported object format in triple " + + TT.str(), + inconvertibleErrorCode()); + } + + return &PlatformJD; } -void setUpGenericLLVMIRPlatform(LLJIT &J) { +Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) { LLVM_DEBUG( { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; }); - J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J)); + auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib(); + if (!ProcessSymbolsJD) + return make_error<StringError>( + "Native platforms require a process symbols JITDylib", + inconvertibleErrorCode()); + + auto &PlatformJD = J.getExecutionSession().createBareJITDylib("<Platform>"); + PlatformJD.addToLinkOrder(*ProcessSymbolsJD); + + J.setPlatformSupport( + std::make_unique<GenericLLVMIRPlatformSupport>(J, PlatformJD)); + + return &PlatformJD; } -Error setUpInactivePlatform(LLJIT &J) { +Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J) { LLVM_DEBUG( { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; }); J.setPlatformSupport(std::make_unique<InactivePlatformSupport>()); - return Error::success(); + return nullptr; } Error LLLazyJITBuilderState::prepareForConstruction() { @@ -984,7 +1255,7 @@ LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { LCTMgr = std::move(S.LCTMgr); else { if (auto LCTMgrOrErr = createLocalLazyCallThroughManager( - S.TT, *ES, S.LazyCompileFailureAddr.getValue())) + S.TT, *ES, S.LazyCompileFailureAddr)) LCTMgr = std::move(*LCTMgrOrErr); else { Err = LCTMgrOrErr.takeError(); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp index c0a740d42dbd..d95a642934f1 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -8,19 +8,20 @@ #include "llvm/ExecutionEngine/Orc/LazyReexports.h" -#include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" +#include "llvm/TargetParser/Triple.h" #define DEBUG_TYPE "orc" namespace llvm { namespace orc { -LazyCallThroughManager::LazyCallThroughManager( - ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr, TrampolinePool *TP) +LazyCallThroughManager::LazyCallThroughManager(ExecutionSession &ES, + ExecutorAddr ErrorHandlerAddr, + TrampolinePool *TP) : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(TP) {} -Expected<JITTargetAddress> LazyCallThroughManager::getCallThroughTrampoline( +Expected<ExecutorAddr> LazyCallThroughManager::getCallThroughTrampoline( JITDylib &SourceJD, SymbolStringPtr SymbolName, NotifyResolvedFunction NotifyResolved) { assert(TP && "TrampolinePool not set"); @@ -36,24 +37,24 @@ Expected<JITTargetAddress> LazyCallThroughManager::getCallThroughTrampoline( return *Trampoline; } -JITTargetAddress LazyCallThroughManager::reportCallThroughError(Error Err) { +ExecutorAddr LazyCallThroughManager::reportCallThroughError(Error Err) { ES.reportError(std::move(Err)); return ErrorHandlerAddr; } Expected<LazyCallThroughManager::ReexportsEntry> -LazyCallThroughManager::findReexport(JITTargetAddress TrampolineAddr) { +LazyCallThroughManager::findReexport(ExecutorAddr TrampolineAddr) { std::lock_guard<std::mutex> Lock(LCTMMutex); auto I = Reexports.find(TrampolineAddr); if (I == Reexports.end()) return createStringError(inconvertibleErrorCode(), - "Missing reexport for trampoline address %p", - TrampolineAddr); + "Missing reexport for trampoline address %p" + + formatv("{0:x}", TrampolineAddr)); return I->second; } -Error LazyCallThroughManager::notifyResolved(JITTargetAddress TrampolineAddr, - JITTargetAddress ResolvedAddr) { +Error LazyCallThroughManager::notifyResolved(ExecutorAddr TrampolineAddr, + ExecutorAddr ResolvedAddr) { NotifyResolvedFunction NotifyResolved; { std::lock_guard<std::mutex> Lock(LCTMMutex); @@ -68,7 +69,7 @@ Error LazyCallThroughManager::notifyResolved(JITTargetAddress TrampolineAddr, } void LazyCallThroughManager::resolveTrampolineLandingAddress( - JITTargetAddress TrampolineAddr, + ExecutorAddr TrampolineAddr, NotifyLandingResolvedFunction NotifyLandingResolved) { auto Entry = findReexport(TrampolineAddr); @@ -84,7 +85,7 @@ void LazyCallThroughManager::resolveTrampolineLandingAddress( if (Result) { assert(Result->size() == 1 && "Unexpected result size"); assert(Result->count(SymbolName) && "Unexpected result value"); - JITTargetAddress LandingAddr = (*Result)[SymbolName].getAddress(); + ExecutorAddr LandingAddr = (*Result)[SymbolName].getAddress(); if (auto Err = notifyResolved(TrampolineAddr, LandingAddr)) NotifyLandingResolved(reportCallThroughError(std::move(Err))); @@ -104,7 +105,7 @@ void LazyCallThroughManager::resolveTrampolineLandingAddress( Expected<std::unique_ptr<LazyCallThroughManager>> createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, - JITTargetAddress ErrorHandlerAddr) { + ExecutorAddr ErrorHandlerAddr) { switch (T.getArch()) { default: return make_error<StringError>( @@ -187,7 +188,7 @@ void LazyReexportsMaterializationUnit::materialize( auto CallThroughTrampoline = LCTManager.getCallThroughTrampoline( SourceJD, Alias.second.Aliasee, [&ISManager = this->ISManager, - StubSym = Alias.first](JITTargetAddress ResolvedAddr) -> Error { + StubSym = Alias.first](ExecutorAddr ResolvedAddr) -> Error { return ISManager.updatePointer(*StubSym, ResolvedAddr); }); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp index 59c63d38458b..75075c5c2a22 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp @@ -31,8 +31,8 @@ void lookupAndRecordAddrs( return OnRec(Result.takeError()); for (auto &KV : Pairs) { auto I = Result->find(KV.first); - KV.second->setValue((I != Result->end()) ? I->second.getAddress() - : 0); + *KV.second = + I != Result->end() ? I->second.getAddress() : orc::ExecutorAddr(); } OnRec(Error::success()); }, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 914a1b5afc71..a3a766d602c1 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -9,10 +9,13 @@ #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/MachO.h" +#include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" #include <optional> @@ -62,8 +65,7 @@ std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP, std::string Name) { unsigned PointerSize; support::endianness Endianness; - const auto &TT = - MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + const auto &TT = MOP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::aarch64: @@ -147,8 +149,8 @@ private: if (G.getEndianness() != support::endian::system_endianness()) MachO::swapStruct(Hdr); - auto HeaderContent = G.allocateString( - StringRef(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); + auto HeaderContent = G.allocateContent( + ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, 0); @@ -246,24 +248,23 @@ private: ExecutorAddr MachOHeaderAddr; }; -StringRef DataCommonSectionName = "__DATA,__common"; -StringRef DataDataSectionName = "__DATA,__data"; -StringRef EHFrameSectionName = "__TEXT,__eh_frame"; -StringRef CompactUnwindInfoSectionName = "__TEXT,__unwind_info"; -StringRef ModInitFuncSectionName = "__DATA,__mod_init_func"; -StringRef ObjCClassListSectionName = "__DATA,__objc_classlist"; -StringRef ObjCImageInfoSectionName = "__DATA,__objc_image_info"; -StringRef ObjCSelRefsSectionName = "__DATA,__objc_selrefs"; -StringRef Swift5ProtoSectionName = "__TEXT,__swift5_proto"; -StringRef Swift5ProtosSectionName = "__TEXT,__swift5_protos"; -StringRef Swift5TypesSectionName = "__TEXT,__swift5_types"; -StringRef ThreadBSSSectionName = "__DATA,__thread_bss"; -StringRef ThreadDataSectionName = "__DATA,__thread_data"; -StringRef ThreadVarsSectionName = "__DATA,__thread_vars"; - -StringRef InitSectionNames[] = { - ModInitFuncSectionName, ObjCSelRefsSectionName, ObjCClassListSectionName, - Swift5ProtosSectionName, Swift5ProtoSectionName, Swift5TypesSectionName}; +static StringRef ObjCRuntimeObjectSectionsData[] = { + MachOObjCCatListSectionName, MachOObjCClassListSectionName, + MachOObjCClassRefsSectionName, MachOObjCConstSectionName, + MachOObjCDataSectionName, MachOObjCSelRefsSectionName}; + +static StringRef ObjCRuntimeObjectSectionsText[] = { + MachOObjCClassNameSectionName, MachOObjCMethNameSectionName, + MachOObjCMethTypeSectionName, MachOSwift5TypesSectionName, + MachOSwift5TypeRefSectionName, MachOSwift5FieldMetadataSectionName, + MachOSwift5EntrySectionName, MachOSwift5ProtoSectionName, + MachOSwift5ProtosSectionName}; + +static StringRef ObjCRuntimeObjectSectionName = + "__llvm_jitlink_ObjCRuntimeRegistrationObject"; + +static StringRef ObjCImageInfoSymbolName = + "__llvm_jitlink_macho_objc_imageinfo"; } // end anonymous namespace @@ -272,17 +273,18 @@ namespace orc { Expected<std::unique_ptr<MachOPlatform>> MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - JITDylib &PlatformJD, const char *OrcRuntimePath, + JITDylib &PlatformJD, + std::unique_ptr<DefinitionGenerator> OrcRuntime, std::optional<SymbolAliasMap> RuntimeAliases) { - auto &EPC = ES.getExecutorProcessControl(); - // If the target is not supported then bail out immediately. - if (!supportedTarget(EPC.getTargetTriple())) + if (!supportedTarget(ES.getTargetTriple())) return make_error<StringError>("Unsupported MachOPlatform triple: " + - EPC.getTargetTriple().str(), + ES.getTargetTriple().str(), inconvertibleErrorCode()); + auto &EPC = ES.getExecutorProcessControl(); + // Create default aliases if the caller didn't supply any. if (!RuntimeAliases) RuntimeAliases = standardPlatformAliases(ES); @@ -292,31 +294,40 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, return std::move(Err); // Add JIT-dispatch function support symbols. - if (auto Err = PlatformJD.define(absoluteSymbols( - {{ES.intern("___orc_rt_jit_dispatch"), - {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), - JITSymbolFlags::Exported}}, - {ES.intern("___orc_rt_jit_dispatch_ctx"), - {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), - JITSymbolFlags::Exported}}}))) + if (auto Err = PlatformJD.define( + absoluteSymbols({{ES.intern("___orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunction, + JITSymbolFlags::Exported}}, + {ES.intern("___orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContext, + JITSymbolFlags::Exported}}}))) return std::move(Err); - // Create a generator for the ORC runtime archive. - auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load( - ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple()); - if (!OrcRuntimeArchiveGenerator) - return OrcRuntimeArchiveGenerator.takeError(); - // Create the instance. Error Err = Error::success(); - auto P = std::unique_ptr<MachOPlatform>( - new MachOPlatform(ES, ObjLinkingLayer, PlatformJD, - std::move(*OrcRuntimeArchiveGenerator), Err)); + auto P = std::unique_ptr<MachOPlatform>(new MachOPlatform( + ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err)); if (Err) return std::move(Err); return std::move(P); } +Expected<std::unique_ptr<MachOPlatform>> +MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + std::optional<SymbolAliasMap> RuntimeAliases) { + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = + StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + return Create(ES, ObjLinkingLayer, PlatformJD, + std::move(*OrcRuntimeArchiveGenerator), + std::move(RuntimeAliases)); +} + Error MachOPlatform::setupJITDylib(JITDylib &JD) { if (auto Err = JD.define(std::make_unique<MachOHeaderMaterializationUnit>( *this, MachOHeaderStartSymbol))) @@ -398,15 +409,6 @@ MachOPlatform::standardRuntimeUtilityAliases() { StandardRuntimeUtilityAliases); } -bool MachOPlatform::isInitializerSection(StringRef SegName, - StringRef SectName) { - for (auto &Name : InitSectionNames) { - if (Name.startswith(SegName) && Name.substr(7) == SectName) - return true; - } - return false; -} - bool MachOPlatform::supportedTarget(const Triple &TT) { switch (TT.getArch()) { case Triple::aarch64: @@ -654,10 +656,9 @@ void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, }); if (!JD) { - SendResult( - make_error<StringError>("No JITDylib with header addr " + - formatv("{0:x}", JDHeaderAddr.getValue()), - inconvertibleErrorCode())); + SendResult(make_error<StringError>("No JITDylib with header addr " + + formatv("{0:x}", JDHeaderAddr), + inconvertibleErrorCode())); return; } @@ -667,8 +668,7 @@ void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, StringRef SymbolName) { LLVM_DEBUG({ - dbgs() << "MachOPlatform::rt_lookupSymbol(\"" - << formatv("{0:x}", Handle.getValue()) << "\")\n"; + dbgs() << "MachOPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"; }); JITDylib *JD = nullptr; @@ -681,12 +681,9 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, } if (!JD) { - LLVM_DEBUG({ - dbgs() << " No JITDylib for handle " - << formatv("{0:x}", Handle.getValue()) << "\n"; - }); + LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + - formatv("{0:x}", Handle.getValue()), + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } @@ -699,7 +696,7 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, void operator()(Expected<SymbolMap> Result) { if (Result) { assert(Result->size() == 1 && "Unexpected result map count"); - SendResult(ExecutorAddr(Result->begin()->second.getAddress())); + SendResult(Result->begin()->second.getAddress()); } else { SendResult(Result.takeError()); } @@ -766,10 +763,14 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( // then add passes to preserve, process and register the init // sections/symbols. Config.PrePrunePasses.push_back([this, &MR](LinkGraph &G) { - if (auto Err = preserveInitSections(G, MR)) + if (auto Err = preserveImportantSections(G, MR)) return Err; return processObjCImageInfo(G, MR); }); + Config.PostPrunePasses.push_back( + [this](LinkGraph &G) { return createObjCRuntimeObject(G); }); + Config.PostAllocationPasses.push_back( + [this, &MR](LinkGraph &G) { return populateObjCRuntimeObject(G, MR); }); } // Insert TLV lowering at the start of the PostPrunePasses, since we want @@ -829,7 +830,10 @@ Error MachOPlatform::MachOPlatformPlugin:: &MP.RegisterObjectPlatformSections.Addr}, {*MP.DeregisterObjectPlatformSections.Name, &MP.DeregisterObjectPlatformSections.Addr}, - {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}}; + {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}, + {*MP.RegisterObjCRuntimeObject.Name, &MP.RegisterObjCRuntimeObject.Addr}, + {*MP.DeregisterObjCRuntimeObject.Name, + &MP.DeregisterObjCRuntimeObject.Addr}}; bool RegisterMachOHeader = false; @@ -898,11 +902,40 @@ Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol( return Error::success(); } -Error MachOPlatform::MachOPlatformPlugin::preserveInitSections( +Error MachOPlatform::MachOPlatformPlugin::preserveImportantSections( jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + // __objc_imageinfo is "important": we want to preserve it and record its + // address in the first graph that it appears in, then verify and discard it + // in all subsequent graphs. In this pass we preserve unconditionally -- we'll + // manually throw it away in the processObjCImageInfo pass. + if (auto *ObjCImageInfoSec = + G.findSectionByName(MachOObjCImageInfoSectionName)) { + if (ObjCImageInfoSec->blocks_size() != 1) + return make_error<StringError>( + "In " + G.getName() + + "__DATA,__objc_imageinfo contains multiple blocks", + inconvertibleErrorCode()); + G.addAnonymousSymbol(**ObjCImageInfoSec->blocks().begin(), 0, 0, false, + true); + + for (auto *B : ObjCImageInfoSec->blocks()) + if (!B->edges_empty()) + return make_error<StringError>("In " + G.getName() + ", " + + MachOObjCImageInfoSectionName + + " contains references to symbols", + inconvertibleErrorCode()); + } + // Init sections are important: We need to preserve them and so that their + // addresses can be captured and reported to the ORC runtime in + // registerObjectPlatformSections. JITLinkSymbolSet InitSectionSymbols; - for (auto &InitSectionName : InitSectionNames) { + for (auto &InitSectionName : MachOInitSectionNames) { + // Skip ObjCImageInfo -- this shouldn't have any dependencies, and we may + // remove it later. + if (InitSectionName == MachOObjCImageInfoSectionName) + continue; + // Skip non-init sections. auto *InitSection = G.findSectionByName(InitSectionName); if (!InitSection) @@ -944,7 +977,7 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( // OR // (2) We already have a recorded __objc_imageinfo for this JITDylib, // in which case we just verify it. - auto *ObjCImageInfo = G.findSectionByName(ObjCImageInfoSectionName); + auto *ObjCImageInfo = G.findSectionByName(MachOObjCImageInfoSectionName); if (!ObjCImageInfo) return Error::success(); @@ -952,14 +985,14 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( // Check that the section is not empty if present. if (ObjCImageInfoBlocks.empty()) - return make_error<StringError>("Empty " + ObjCImageInfoSectionName + + return make_error<StringError>("Empty " + MachOObjCImageInfoSectionName + " section in " + G.getName(), inconvertibleErrorCode()); // Check that there's only one block in the section. if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) return make_error<StringError>("Multiple blocks in " + - ObjCImageInfoSectionName + + MachOObjCImageInfoSectionName + " section in " + G.getName(), inconvertibleErrorCode()); @@ -971,7 +1004,7 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( for (auto &E : B->edges()) if (E.getTarget().isDefined() && &E.getTarget().getBlock().getSection() == ObjCImageInfo) - return make_error<StringError>(ObjCImageInfoSectionName + + return make_error<StringError>(MachOObjCImageInfoSectionName + " is referenced within file " + G.getName(), inconvertibleErrorCode()); @@ -990,12 +1023,12 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( if (ObjCImageInfoItr != ObjCImageInfos.end()) { // We've already registered an __objc_imageinfo section. Verify the // content of this new section matches, then delete it. - if (ObjCImageInfoItr->second.first != Version) + if (ObjCImageInfoItr->second.Version != Version) return make_error<StringError>( "ObjC version in " + G.getName() + " does not match first registered version", inconvertibleErrorCode()); - if (ObjCImageInfoItr->second.second != Flags) + if (ObjCImageInfoItr->second.Flags != Flags) return make_error<StringError>("ObjC flags in " + G.getName() + " do not match first registered flags", inconvertibleErrorCode()); @@ -1007,7 +1040,14 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( } else { // We haven't registered an __objc_imageinfo section yet. Register and // move on. The section should already be marked no-dead-strip. - ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags); + G.addDefinedSymbol(ObjCImageInfoBlock, 0, ObjCImageInfoSymbolName, + ObjCImageInfoBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Hidden, false, true); + if (auto Err = MR.defineMaterializing( + {{MR.getExecutionSession().intern(ObjCImageInfoSymbolName), + JITSymbolFlags()}})) + return Err; + ObjCImageInfos[&MR.getTargetJITDylib()] = {Version, Flags}; } return Error::success(); @@ -1024,7 +1064,7 @@ Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( } // Store key in __thread_vars struct fields. - if (auto *ThreadDataSec = G.findSectionByName(ThreadVarsSectionName)) { + if (auto *ThreadDataSec = G.findSectionByName(MachOThreadVarsSectionName)) { std::optional<uint64_t> Key; { std::lock_guard<std::mutex> Lock(MP.PlatformMutex); @@ -1098,10 +1138,11 @@ MachOPlatform::MachOPlatformPlugin::findUnwindSectionInfo( } }; - if (Section *EHFrameSec = G.findSectionByName(EHFrameSectionName)) + if (Section *EHFrameSec = G.findSectionByName(MachOEHFrameSectionName)) ScanUnwindInfoSection(*EHFrameSec, US.DwarfSection); - if (Section *CUInfoSec = G.findSectionByName(CompactUnwindInfoSectionName)) + if (Section *CUInfoSec = + G.findSectionByName(MachOCompactUnwindInfoSectionName)) ScanUnwindInfoSection(*CUInfoSec, US.CompactUnwindSection); // If we didn't find any pointed-to code-blocks then there's no need to @@ -1150,10 +1191,10 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( // Get a pointer to the thread data section if there is one. It will be used // below. jitlink::Section *ThreadDataSection = - G.findSectionByName(ThreadDataSectionName); + G.findSectionByName(MachOThreadDataSectionName); // Handle thread BSS section if there is one. - if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) { + if (auto *ThreadBSSSection = G.findSectionByName(MachOThreadBSSSectionName)) { // If there's already a thread data section in this graph then merge the // thread BSS section content into it, otherwise just treat the thread // BSS section as the thread data section. @@ -1166,8 +1207,9 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( SmallVector<std::pair<StringRef, ExecutorAddrRange>, 8> MachOPlatformSecs; // Collect data sections to register. - StringRef DataSections[] = {DataDataSectionName, DataCommonSectionName, - EHFrameSectionName}; + StringRef DataSections[] = {MachODataDataSectionName, + MachODataCommonSectionName, + MachOEHFrameSectionName}; for (auto &SecName : DataSections) { if (auto *Sec = G.findSectionByName(SecName)) { jitlink::SectionRange R(*Sec); @@ -1181,17 +1223,13 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( if (ThreadDataSection) { jitlink::SectionRange R(*ThreadDataSection); if (!R.empty()) - MachOPlatformSecs.push_back({ThreadDataSectionName, R.getRange()}); + MachOPlatformSecs.push_back({MachOThreadDataSectionName, R.getRange()}); } // If any platform sections were found then add an allocation action to call // the registration function. - StringRef PlatformSections[] = { - ModInitFuncSectionName, ObjCClassListSectionName, - ObjCImageInfoSectionName, ObjCSelRefsSectionName, - Swift5ProtoSectionName, Swift5ProtosSectionName, - Swift5TypesSectionName, - }; + StringRef PlatformSections[] = {MachOModInitFuncSectionName, + ObjCRuntimeObjectSectionName}; for (auto &SecName : PlatformSections) { auto *Sec = G.findSectionByName(SecName); @@ -1252,5 +1290,207 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::createObjCRuntimeObject( + jitlink::LinkGraph &G) { + + bool NeedTextSegment = false; + size_t NumRuntimeSections = 0; + + for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData) + if (G.findSectionByName(ObjCRuntimeSectionName)) + ++NumRuntimeSections; + + for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) { + if (G.findSectionByName(ObjCRuntimeSectionName)) { + ++NumRuntimeSections; + NeedTextSegment = true; + } + } + + // Early out for no runtime sections. + if (NumRuntimeSections == 0) + return Error::success(); + + // If there were any runtime sections then we need to add an __objc_imageinfo + // section. + ++NumRuntimeSections; + + size_t MachOSize = sizeof(MachO::mach_header_64) + + (NeedTextSegment + 1) * sizeof(MachO::segment_command_64) + + NumRuntimeSections * sizeof(MachO::section_64); + + auto &Sec = G.createSection(ObjCRuntimeObjectSectionName, + MemProt::Read | MemProt::Write); + G.createMutableContentBlock(Sec, MachOSize, ExecutorAddr(), 16, 0, true); + + return Error::success(); +} + +Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + + auto *ObjCRuntimeObjectSec = + G.findSectionByName(ObjCRuntimeObjectSectionName); + + if (!ObjCRuntimeObjectSec) + return Error::success(); + + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + case Triple::x86_64: + // Supported. + break; + default: + return make_error<StringError>("Unrecognized MachO arch in triple " + + G.getTargetTriple().str(), + inconvertibleErrorCode()); + } + + auto &SecBlock = **ObjCRuntimeObjectSec->blocks().begin(); + + struct SecDesc { + MachO::section_64 Sec; + unique_function<void(size_t RecordOffset)> AddFixups; + }; + + std::vector<SecDesc> TextSections, DataSections; + auto AddSection = [&](SecDesc &SD, jitlink::Section &GraphSec) { + jitlink::SectionRange SR(GraphSec); + StringRef FQName = GraphSec.getName(); + memset(&SD.Sec, 0, sizeof(MachO::section_64)); + memcpy(SD.Sec.sectname, FQName.drop_front(7).data(), FQName.size() - 7); + memcpy(SD.Sec.segname, FQName.data(), 6); + SD.Sec.addr = SR.getStart() - SecBlock.getAddress(); + SD.Sec.size = SR.getSize(); + SD.Sec.flags = MachO::S_REGULAR; + }; + + // Add the __objc_imageinfo section. + { + DataSections.push_back({}); + auto &SD = DataSections.back(); + memset(&SD.Sec, 0, sizeof(SD.Sec)); + memcpy(SD.Sec.sectname, "__objc_imageinfo", 16); + strcpy(SD.Sec.segname, "__DATA"); + SD.Sec.size = 8; + SD.AddFixups = [&](size_t RecordOffset) { + jitlink::Edge::Kind PointerEdge = jitlink::Edge::Invalid; + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + PointerEdge = jitlink::aarch64::Pointer64; + break; + case Triple::x86_64: + PointerEdge = jitlink::x86_64::Pointer64; + break; + default: + llvm_unreachable("Unsupported architecture"); + } + + // Look for an existing __objc_imageinfo symbol. + jitlink::Symbol *ObjCImageInfoSym = nullptr; + for (auto *Sym : G.external_symbols()) + if (Sym->getName() == ObjCImageInfoSymbolName) { + ObjCImageInfoSym = Sym; + break; + } + if (!ObjCImageInfoSym) + for (auto *Sym : G.absolute_symbols()) + if (Sym->getName() == ObjCImageInfoSymbolName) { + ObjCImageInfoSym = Sym; + break; + } + if (!ObjCImageInfoSym) + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && Sym->getName() == ObjCImageInfoSymbolName) { + ObjCImageInfoSym = Sym; + break; + } + if (!ObjCImageInfoSym) + ObjCImageInfoSym = + &G.addExternalSymbol(ObjCImageInfoSymbolName, 8, false); + + SecBlock.addEdge(PointerEdge, + RecordOffset + ((char *)&SD.Sec.addr - (char *)&SD.Sec), + *ObjCImageInfoSym, -SecBlock.getAddress().getValue()); + }; + } + + for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData) { + if (auto *GraphSec = G.findSectionByName(ObjCRuntimeSectionName)) { + DataSections.push_back({}); + AddSection(DataSections.back(), *GraphSec); + } + } + + for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) { + if (auto *GraphSec = G.findSectionByName(ObjCRuntimeSectionName)) { + TextSections.push_back({}); + AddSection(TextSections.back(), *GraphSec); + } + } + + assert(ObjCRuntimeObjectSec->blocks_size() == 1 && + "Unexpected number of blocks in runtime sections object"); + + // Build the header struct up-front. This also gives us a chance to check + // that the triple is supported, which we'll assume below. + MachO::mach_header_64 Hdr; + Hdr.magic = MachO::MH_MAGIC_64; + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + Hdr.cputype = MachO::CPU_TYPE_ARM64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; + break; + case Triple::x86_64: + Hdr.cputype = MachO::CPU_TYPE_X86_64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; + break; + default: + llvm_unreachable("Unsupported architecture"); + } + + Hdr.filetype = MachO::MH_DYLIB; + Hdr.ncmds = 1 + !TextSections.empty(); + Hdr.sizeofcmds = + Hdr.ncmds * sizeof(MachO::segment_command_64) + + (TextSections.size() + DataSections.size()) * sizeof(MachO::section_64); + Hdr.flags = 0; + Hdr.reserved = 0; + + auto SecContent = SecBlock.getAlreadyMutableContent(); + char *P = SecContent.data(); + auto WriteMachOStruct = [&](auto S) { + if (G.getEndianness() != support::endian::system_endianness()) + MachO::swapStruct(S); + memcpy(P, &S, sizeof(S)); + P += sizeof(S); + }; + + auto WriteSegment = [&](StringRef Name, std::vector<SecDesc> &Secs) { + MachO::segment_command_64 SegLC; + memset(&SegLC, 0, sizeof(SegLC)); + memcpy(SegLC.segname, Name.data(), Name.size()); + SegLC.cmd = MachO::LC_SEGMENT_64; + SegLC.cmdsize = sizeof(MachO::segment_command_64) + + Secs.size() * sizeof(MachO::section_64); + SegLC.nsects = Secs.size(); + WriteMachOStruct(SegLC); + for (auto &SD : Secs) { + if (SD.AddFixups) + SD.AddFixups(P - SecContent.data()); + WriteMachOStruct(SD.Sec); + } + }; + + WriteMachOStruct(Hdr); + if (!TextSections.empty()) + WriteSegment("__TEXT", TextSections); + if (!DataSections.empty()) + WriteSegment("__DATA", DataSections); + + assert(P == SecContent.end() && "Underflow writing ObjC runtime object"); + return Error::success(); +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp index b457c7297bed..ca4950077ffe 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -322,7 +322,8 @@ void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); tpctypes::SharedMemorySegFinalizeRequest SegReq; - SegReq.AG = Segment.AG; + SegReq.RAG = {Segment.AG.getMemProt(), Segment.AG.getMemLifetimePolicy() == + MemLifetimePolicy::Finalize}; SegReq.Addr = AI.MappingBase + Segment.Offset; SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp index 0c3beba43a35..7c8fa63477d0 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp @@ -10,6 +10,7 @@ #include "llvm/ExecutionEngine/Orc/COFFPlatform.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Object/COFF.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/MachO.h" @@ -85,7 +86,7 @@ getMachOObjectFileSymbolInfo(ExecutionSession &ES, } auto SegName = Obj.getSectionFinalSegmentName(Sec.getRawDataRefImpl()); auto SecName = cantFail(Obj.getSectionName(Sec.getRawDataRefImpl())); - if (MachOPlatform::isInitializerSection(SegName, SecName)) { + if (isMachOInitializerSection(SegName, SecName)) { addInitSymbol(I, ES, Obj.getFileName()); break; } @@ -138,7 +139,7 @@ getELFObjectFileSymbolInfo(ExecutionSession &ES, SymbolStringPtr InitSymbol; for (auto &Sec : Obj.sections()) { if (auto SecName = Sec.getName()) { - if (ELFNixPlatform::isInitializerSection(*SecName)) { + if (isELFInitializerSection(*SecName)) { addInitSymbol(I, ES, Obj.getFileName()); break; } @@ -219,7 +220,7 @@ getCOFFObjectFileSymbolInfo(ExecutionSession &ES, SymbolStringPtr InitSymbol; for (auto &Sec : Obj.sections()) { if (auto SecName = Sec.getName()) { - if (COFFPlatform::isInitializerSection(*SecName)) { + if (isCOFFInitializerSection(*SecName)) { addInitSymbol(I, ES, Obj.getFileName()); break; } @@ -287,22 +288,5 @@ getObjectFileInterface(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { return getGenericObjectFileSymbolInfo(ES, **Obj); } -bool hasInitializerSection(jitlink::LinkGraph &G) { - bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); - bool IsElf = G.getTargetTriple().isOSBinFormatELF(); - if (!IsMachO && !IsElf) - return false; - - for (auto &Sec : G.sections()) { - if (IsMachO && std::apply(MachOPlatform::isInitializerSection, - Sec.getName().split(","))) - return true; - if (IsElf && ELFNixPlatform::isInitializerSection(Sec.getName())) - return true; - } - - return false; -} - } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 2b11c472e812..a29f3d1c3aec 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -8,8 +8,10 @@ #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" +#include "llvm/ExecutionEngine/JITLink/aarch32.h" #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Support/MemoryBuffer.h" #include <string> #include <vector> @@ -22,6 +24,55 @@ using namespace llvm::orc; namespace { +bool hasInitializerSection(jitlink::LinkGraph &G) { + bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); + bool IsElf = G.getTargetTriple().isOSBinFormatELF(); + if (!IsMachO && !IsElf) + return false; + + for (auto &Sec : G.sections()) { + if (IsMachO && isMachOInitializerSection(Sec.getName())) + return true; + if (IsElf && isELFInitializerSection(Sec.getName())) + return true; + } + + return false; +} + +ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) { + switch (TT.getArch()) { + case Triple::arm: + case Triple::armeb: + case Triple::thumb: + case Triple::thumbeb: + if (Sym.hasTargetFlags(aarch32::ThumbSymbol)) { + // Set LSB to indicate thumb target + assert(Sym.isCallable() && "Only callable symbols can have thumb flag"); + assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear"); + return Sym.getAddress() + 0x01; + } + return Sym.getAddress(); + default: + return Sym.getAddress(); + } +} + +JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) { + JITSymbolFlags Flags; + + if (Sym.getLinkage() == Linkage::Weak) + Flags |= JITSymbolFlags::Weak; + + if (Sym.getScope() == Scope::Default) + Flags |= JITSymbolFlags::Exported; + + if (Sym.isCallable()) + Flags |= JITSymbolFlags::Callable; + + return Flags; +} + class LinkGraphMaterializationUnit : public MaterializationUnit { public: static std::unique_ptr<LinkGraphMaterializationUnit> @@ -48,14 +99,8 @@ private: continue; assert(Sym->hasName() && "Anonymous non-local symbol?"); - JITSymbolFlags Flags; - if (Sym->getScope() == Scope::Default) - Flags |= JITSymbolFlags::Exported; - - if (Sym->isCallable()) - Flags |= JITSymbolFlags::Callable; - - LGI.SymbolFlags[ES.intern(Sym->getName())] = Flags; + LGI.SymbolFlags[ES.intern(Sym->getName())] = + getJITSymbolFlagsForSymbol(*Sym); } if (hasInitializerSection(G)) @@ -189,17 +234,9 @@ public: for (auto *Sym : G.defined_symbols()) if (Sym->hasName() && Sym->getScope() != Scope::Local) { auto InternedName = ES.intern(Sym->getName()); - JITSymbolFlags Flags; - - if (Sym->isCallable()) - Flags |= JITSymbolFlags::Callable; - if (Sym->getScope() == Scope::Default) - Flags |= JITSymbolFlags::Exported; - if (Sym->getLinkage() == Linkage::Weak) - Flags |= JITSymbolFlags::Weak; - - InternedResult[InternedName] = - JITEvaluatedSymbol(Sym->getAddress().getValue(), Flags); + auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); + auto Flags = getJITSymbolFlagsForSymbol(*Sym); + InternedResult[InternedName] = {Ptr, Flags}; if (AutoClaim && !MR->getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); @@ -210,15 +247,9 @@ public: for (auto *Sym : G.absolute_symbols()) if (Sym->hasName() && Sym->getScope() != Scope::Local) { auto InternedName = ES.intern(Sym->getName()); - JITSymbolFlags Flags; - if (Sym->isCallable()) - Flags |= JITSymbolFlags::Callable; - if (Sym->getScope() == Scope::Default) - Flags |= JITSymbolFlags::Exported; - if (Sym->getLinkage() == Linkage::Weak) - Flags |= JITSymbolFlags::Weak; - InternedResult[InternedName] = - JITEvaluatedSymbol(Sym->getAddress().getValue(), Flags); + auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); + auto Flags = getJITSymbolFlagsForSymbol(*Sym); + InternedResult[InternedName] = {Ptr, Flags}; if (AutoClaim && !MR->getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); @@ -407,10 +438,8 @@ private: Sym->getScope() != Scope::Local) { auto Name = ES.intern(Sym->getName()); if (!MR->getSymbols().count(ES.intern(Sym->getName()))) { - JITSymbolFlags SF = JITSymbolFlags::Weak; - if (Sym->getScope() == Scope::Default) - SF |= JITSymbolFlags::Exported; - NewSymbolsToClaim[Name] = SF; + NewSymbolsToClaim[Name] = + getJITSymbolFlagsForSymbol(*Sym) | JITSymbolFlags::Weak; NameToSym.push_back(std::make_pair(std::move(Name), Sym)); } } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp index 48dd0df80415..6d568199378a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -14,17 +14,17 @@ #define DEBUG_TYPE "orc" using namespace llvm; +using namespace llvm::orc; template <typename ORCABI> -bool stubAndPointerRangesOk(JITTargetAddress StubBlockAddr, - JITTargetAddress PointerBlockAddr, - unsigned NumStubs) { +static bool stubAndPointerRangesOk(ExecutorAddr StubBlockAddr, + ExecutorAddr PointerBlockAddr, + unsigned NumStubs) { constexpr unsigned MaxDisp = ORCABI::StubToPointerMaxDisplacement; - JITTargetAddress FirstStub = StubBlockAddr; - JITTargetAddress LastStub = FirstStub + ((NumStubs - 1) * ORCABI::StubSize); - JITTargetAddress FirstPointer = PointerBlockAddr; - JITTargetAddress LastPointer = - FirstPointer + ((NumStubs - 1) * ORCABI::StubSize); + ExecutorAddr FirstStub = StubBlockAddr; + ExecutorAddr LastStub = FirstStub + ((NumStubs - 1) * ORCABI::StubSize); + ExecutorAddr FirstPointer = PointerBlockAddr; + ExecutorAddr LastPointer = FirstPointer + ((NumStubs - 1) * ORCABI::StubSize); if (FirstStub < FirstPointer) { if (LastStub >= FirstPointer) @@ -44,9 +44,9 @@ namespace llvm { namespace orc { void OrcAArch64::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { const uint32_t ResolverCode[] = { // resolver_entry: @@ -135,8 +135,8 @@ void OrcAArch64::writeResolverCode(char *ResolverWorkingMem, } void OrcAArch64::writeTrampolines(char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, unsigned NumTrampolines) { unsigned OffsetToPtr = alignTo(NumTrampolines * TrampolineSize, 8); @@ -159,17 +159,17 @@ void OrcAArch64::writeTrampolines(char *TrampolineBlockWorkingMem, } void OrcAArch64::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs // stub1: - // ldr x0, ptr1 ; PC-rel load of ptr1 - // br x0 ; Jump to resolver + // ldr x16, ptr1 ; PC-rel load of ptr1 + // br x16 ; Jump to resolver // stub2: - // ldr x0, ptr2 ; PC-rel load of ptr2 - // br x0 ; Jump to resolver + // ldr x16, ptr2 ; PC-rel load of ptr2 + // br x16 ; Jump to resolver // // ... // @@ -188,17 +188,19 @@ void OrcAArch64::writeIndirectStubsBlock( "PointersBlock is out of range"); uint64_t PtrDisplacement = PointersBlockTargetAddress - StubsBlockTargetAddress; + assert((PtrDisplacement % 8 == 0) && + "Displacement to pointer is not a multiple of 8"); uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); - uint64_t PtrOffsetField = PtrDisplacement << 3; + uint64_t PtrOffsetField = ((PtrDisplacement >> 2) & 0x7ffff) << 5; for (unsigned I = 0; I < NumStubs; ++I) Stub[I] = 0xd61f020058000010 | PtrOffsetField; } -void OrcX86_64_Base::writeTrampolines( - char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, unsigned NumTrampolines) { +void OrcX86_64_Base::writeTrampolines(char *TrampolineBlockWorkingMem, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, + unsigned NumTrampolines) { unsigned OffsetToPtr = NumTrampolines * TrampolineSize; @@ -214,8 +216,8 @@ void OrcX86_64_Base::writeTrampolines( } void OrcX86_64_Base::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs @@ -250,9 +252,9 @@ void OrcX86_64_Base::writeIndirectStubsBlock( } void OrcX86_64_SysV::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { LLVM_DEBUG({ dbgs() << "Writing resolver code to " @@ -324,9 +326,9 @@ void OrcX86_64_SysV::writeResolverCode(char *ResolverWorkingMem, } void OrcX86_64_Win32::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { // resolverCode is similar to OrcX86_64 with differences specific to windows // x64 calling convention: arguments go into rcx, rdx and come in reverse @@ -402,12 +404,13 @@ void OrcX86_64_Win32::writeResolverCode(char *ResolverWorkingMem, } void OrcI386::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { - assert((ReentryFnAddr >> 32) == 0 && "ReentryFnAddr out of range"); - assert((ReentryCtxAddr >> 32) == 0 && "ReentryCtxAddr out of range"); + assert((ReentryFnAddr.getValue() >> 32) == 0 && "ReentryFnAddr out of range"); + assert((ReentryCtxAddr.getValue() >> 32) == 0 && + "ReentryCtxAddr out of range"); const uint8_t ResolverCode[] = { // resolver_entry: @@ -455,10 +458,10 @@ void OrcI386::writeResolverCode(char *ResolverWorkingMem, } void OrcI386::writeTrampolines(char *TrampolineWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, unsigned NumTrampolines) { - assert((ResolverAddr >> 32) == 0 && "ResolverAddr out of range"); + assert((ResolverAddr.getValue() >> 32) == 0 && "ResolverAddr out of range"); uint64_t CallRelImm = 0xF1C4C400000000e8; uint64_t ResolverRel = ResolverAddr - TrampolineBlockTargetAddress - 5; @@ -468,12 +471,13 @@ void OrcI386::writeTrampolines(char *TrampolineWorkingMem, Trampolines[I] = CallRelImm | (ResolverRel << 8); } -void OrcI386::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { - assert((StubsBlockTargetAddress >> 32) == 0 && +void OrcI386::writeIndirectStubsBlock(char *StubsBlockWorkingMem, + ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, + unsigned NumStubs) { + assert((StubsBlockTargetAddress.getValue() >> 32) == 0 && "StubsBlockTargetAddress is out of range"); - assert((PointersBlockTargetAddress >> 32) == 0 && + assert((PointersBlockTargetAddress.getValue() >> 32) == 0 && "PointersBlockTargetAddress is out of range"); // Stub format is: @@ -501,15 +505,15 @@ void OrcI386::writeIndirectStubsBlock( "PointersBlock is out of range"); uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); - uint64_t PtrAddr = PointersBlockTargetAddress; + uint64_t PtrAddr = PointersBlockTargetAddress.getValue(); for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 4) Stub[I] = 0xF1C40000000025ff | (PtrAddr << 16); } void OrcMips32_Base::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr, + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr, bool isBigEndian) { const uint32_t ResolverCode[] = { @@ -596,32 +600,32 @@ void OrcMips32_Base::writeResolverCode(char *ResolverWorkingMem, memcpy(ResolverWorkingMem + Offsett, &MoveVxT9, sizeof(MoveVxT9)); uint32_t ReentryCtxLUi = - 0x3c040000 | (((ReentryCtxAddr + 0x8000) >> 16) & 0xFFFF); - uint32_t ReentryCtxADDiu = 0x24840000 | ((ReentryCtxAddr)&0xFFFF); + 0x3c040000 | (((ReentryCtxAddr.getValue() + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryCtxADDiu = 0x24840000 | (ReentryCtxAddr.getValue() & 0xFFFF); memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxLUi, sizeof(ReentryCtxLUi)); memcpy(ResolverWorkingMem + ReentryCtxAddrOffset + 4, &ReentryCtxADDiu, sizeof(ReentryCtxADDiu)); uint32_t ReentryFnLUi = - 0x3c190000 | (((ReentryFnAddr + 0x8000) >> 16) & 0xFFFF); - uint32_t ReentryFnADDiu = 0x27390000 | ((ReentryFnAddr)&0xFFFF); + 0x3c190000 | (((ReentryFnAddr.getValue() + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryFnADDiu = 0x27390000 | (ReentryFnAddr.getValue() & 0xFFFF); memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnLUi, sizeof(ReentryFnLUi)); memcpy(ResolverWorkingMem + ReentryFnAddrOffset + 4, &ReentryFnADDiu, sizeof(ReentryFnADDiu)); } -void OrcMips32_Base::writeTrampolines( - char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, unsigned NumTrampolines) { +void OrcMips32_Base::writeTrampolines(char *TrampolineBlockWorkingMem, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, + unsigned NumTrampolines) { - assert((ResolverAddr >> 32) == 0 && "ResolverAddr out of range"); + assert((ResolverAddr.getValue() >> 32) == 0 && "ResolverAddr out of range"); uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); - uint32_t RHiAddr = ((ResolverAddr + 0x8000) >> 16); + uint32_t RHiAddr = ((ResolverAddr.getValue() + 0x8000) >> 16); for (unsigned I = 0; I < NumTrampolines; ++I) { // move $t8,$ra @@ -631,16 +635,16 @@ void OrcMips32_Base::writeTrampolines( // nop Trampolines[5 * I + 0] = 0x03e0c025; Trampolines[5 * I + 1] = 0x3c190000 | (RHiAddr & 0xFFFF); - Trampolines[5 * I + 2] = 0x27390000 | (ResolverAddr & 0xFFFF); + Trampolines[5 * I + 2] = 0x27390000 | (ResolverAddr.getValue() & 0xFFFF); Trampolines[5 * I + 3] = 0x0320f809; Trampolines[5 * I + 4] = 0x00000000; } } void OrcMips32_Base::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { - assert((StubsBlockTargetAddress >> 32) == 0 && + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { + assert((StubsBlockTargetAddress.getValue() >> 32) == 0 && "InitialPtrVal is out of range"); // Stub format is: @@ -671,7 +675,7 @@ void OrcMips32_Base::writeIndirectStubsBlock( // Populate the stubs page stubs and mark it executable. uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlockWorkingMem); - uint64_t PtrAddr = PointersBlockTargetAddress; + uint64_t PtrAddr = PointersBlockTargetAddress.getValue(); for (unsigned I = 0; I < NumStubs; ++I) { uint32_t HiAddr = ((PtrAddr + 0x8000) >> 16); @@ -684,9 +688,9 @@ void OrcMips32_Base::writeIndirectStubsBlock( } void OrcMips64::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { const uint32_t ResolverCode[] = { //resolver_entry: @@ -775,14 +779,16 @@ void OrcMips64::writeResolverCode(char *ResolverWorkingMem, memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); uint32_t ReentryCtxLUi = - 0x3c040000 | (((ReentryCtxAddr + 0x800080008000) >> 48) & 0xFFFF); + 0x3c040000 | + (((ReentryCtxAddr.getValue() + 0x800080008000) >> 48) & 0xFFFF); uint32_t ReentryCtxDADDiu = - 0x64840000 | (((ReentryCtxAddr + 0x80008000) >> 32) & 0xFFFF); + 0x64840000 | (((ReentryCtxAddr.getValue() + 0x80008000) >> 32) & 0xFFFF); uint32_t ReentryCtxDSLL = 0x00042438; uint32_t ReentryCtxDADDiu2 = - 0x64840000 | ((((ReentryCtxAddr + 0x8000) >> 16) & 0xFFFF)); + 0x64840000 | ((((ReentryCtxAddr.getValue() + 0x8000) >> 16) & 0xFFFF)); uint32_t ReentryCtxDSLL2 = 0x00042438; - uint32_t ReentryCtxDADDiu3 = 0x64840000 | ((ReentryCtxAddr)&0xFFFF); + uint32_t ReentryCtxDADDiu3 = + 0x64840000 | (ReentryCtxAddr.getValue() & 0xFFFF); memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxLUi, sizeof(ReentryCtxLUi)); @@ -798,19 +804,20 @@ void OrcMips64::writeResolverCode(char *ResolverWorkingMem, sizeof(ReentryCtxDADDiu3)); uint32_t ReentryFnLUi = - 0x3c190000 | (((ReentryFnAddr + 0x800080008000) >> 48) & 0xFFFF); + 0x3c190000 | + (((ReentryFnAddr.getValue() + 0x800080008000) >> 48) & 0xFFFF); uint32_t ReentryFnDADDiu = - 0x67390000 | (((ReentryFnAddr + 0x80008000) >> 32) & 0xFFFF); + 0x67390000 | (((ReentryFnAddr.getValue() + 0x80008000) >> 32) & 0xFFFF); uint32_t ReentryFnDSLL = 0x0019cc38; uint32_t ReentryFnDADDiu2 = - 0x67390000 | (((ReentryFnAddr + 0x8000) >> 16) & 0xFFFF); + 0x67390000 | (((ReentryFnAddr.getValue() + 0x8000) >> 16) & 0xFFFF); uint32_t ReentryFnDSLL2 = 0x0019cc38; - uint32_t ReentryFnDADDiu3 = 0x67390000 | ((ReentryFnAddr)&0xFFFF); + uint32_t ReentryFnDADDiu3 = 0x67390000 | (ReentryFnAddr.getValue() & 0xFFFF); memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnLUi, sizeof(ReentryFnLUi)); @@ -827,16 +834,16 @@ void OrcMips64::writeResolverCode(char *ResolverWorkingMem, } void OrcMips64::writeTrampolines(char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, unsigned NumTrampolines) { uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); - uint64_t HeighestAddr = ((ResolverAddr + 0x800080008000) >> 48); - uint64_t HeigherAddr = ((ResolverAddr + 0x80008000) >> 32); - uint64_t HiAddr = ((ResolverAddr + 0x8000) >> 16); + uint64_t HeighestAddr = ((ResolverAddr.getValue() + 0x800080008000) >> 48); + uint64_t HeigherAddr = ((ResolverAddr.getValue() + 0x80008000) >> 32); + uint64_t HiAddr = ((ResolverAddr.getValue() + 0x8000) >> 16); for (unsigned I = 0; I < NumTrampolines; ++I) { Trampolines[10 * I + 0] = 0x03e0c025; // move $t8,$ra @@ -845,17 +852,18 @@ void OrcMips64::writeTrampolines(char *TrampolineBlockWorkingMem, Trampolines[10 * I + 3] = 0x0019cc38; // dsll $t9,$t9,16 Trampolines[10 * I + 4] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) Trampolines[10 * I + 5] = 0x0019cc38; // dsll $t9,$t9,16 - Trampolines[10 * I + 6] = - 0x67390000 | (ResolverAddr & 0xFFFF); // daddiu $t9,$t9,%lo(ptr) + Trampolines[10 * I + 6] = 0x67390000 | (ResolverAddr.getValue() & + 0xFFFF); // daddiu $t9,$t9,%lo(ptr) Trampolines[10 * I + 7] = 0x0320f809; // jalr $t9 Trampolines[10 * I + 8] = 0x00000000; // nop Trampolines[10 * I + 9] = 0x00000000; // nop } } -void OrcMips64::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { +void OrcMips64::writeIndirectStubsBlock(char *StubsBlockWorkingMem, + ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, + unsigned NumStubs) { // Stub format is: // // .section __orc_stubs @@ -890,7 +898,7 @@ void OrcMips64::writeIndirectStubsBlock( // Populate the stubs page stubs and mark it executable. uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlockWorkingMem); - uint64_t PtrAddr = PointersBlockTargetAddress; + uint64_t PtrAddr = PointersBlockTargetAddress.getValue(); for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 8) { uint64_t HeighestAddr = ((PtrAddr + 0x800080008000) >> 48); @@ -908,9 +916,9 @@ void OrcMips64::writeIndirectStubsBlock( } void OrcRiscv64::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { const uint32_t ResolverCode[] = { 0xef810113, // 0x00: addi sp,sp,-264 @@ -1008,8 +1016,8 @@ void OrcRiscv64::writeResolverCode(char *ResolverWorkingMem, } void OrcRiscv64::writeTrampolines(char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, unsigned NumTrampolines) { unsigned OffsetToPtr = alignTo(NumTrampolines * TrampolineSize, 8); @@ -1031,8 +1039,8 @@ void OrcRiscv64::writeTrampolines(char *TrampolineBlockWorkingMem, } void OrcRiscv64::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs @@ -1078,9 +1086,9 @@ void OrcRiscv64::writeIndirectStubsBlock( } void OrcLoongArch64::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { LLVM_DEBUG({ dbgs() << "Writing resolver code to " @@ -1150,10 +1158,10 @@ void OrcLoongArch64::writeResolverCode(char *ResolverWorkingMem, sizeof(uint64_t)); } -void OrcLoongArch64::writeTrampolines( - char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, unsigned NumTrampolines) { +void OrcLoongArch64::writeTrampolines(char *TrampolineBlockWorkingMem, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, + unsigned NumTrampolines) { LLVM_DEBUG({ dbgs() << "Writing trampoline code to " @@ -1181,8 +1189,8 @@ void OrcLoongArch64::writeTrampolines( } void OrcLoongArch64::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp index b823197b404f..a73aec6d98c6 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -184,8 +184,8 @@ static SymbolMap toSymbolMap(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs) { SymbolMap SM; for (size_t I = 0; I != NumPairs; ++I) { JITSymbolFlags Flags = toJITSymbolFlags(Syms[I].Sym.Flags); - SM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = - JITEvaluatedSymbol(Syms[I].Sym.Address, Flags); + SM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = { + ExecutorAddr(Syms[I].Sym.Address), Flags}; } return SM; } @@ -269,8 +269,8 @@ static LLVMOrcSymbolLookupFlags fromSymbolLookupFlags(SymbolLookupFlags SLF) { } static LLVMJITEvaluatedSymbol -fromJITEvaluatedSymbol(const JITEvaluatedSymbol &S) { - return {S.getAddress(), fromJITSymbolFlags(S.getFlags())}; +fromExecutorSymbolDef(const ExecutorSymbolDef &S) { + return {S.getAddress().getValue(), fromJITSymbolFlags(S.getFlags())}; } } // end anonymous namespace @@ -385,7 +385,7 @@ void LLVMOrcExecutionSessionLookup( for (auto &KV : *Result) CResult.push_back(LLVMOrcCSymbolMapPair{ wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(KV.first)), - fromJITEvaluatedSymbol(KV.second)}); + fromExecutorSymbolDef(KV.second)}); HandleResult(LLVMErrorSuccess, CResult.data(), CResult.size(), Ctx); } else HandleResult(wrap(Result.takeError()), nullptr, 0, Ctx); @@ -741,31 +741,19 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForPath( LLVMErrorRef LLVMOrcCreateStaticLibrarySearchGeneratorForPath( LLVMOrcDefinitionGeneratorRef *Result, LLVMOrcObjectLayerRef ObjLayer, - const char *FileName, const char *TargetTriple) { + const char *FileName) { assert(Result && "Result can not be null"); assert(FileName && "Filename can not be null"); assert(ObjLayer && "ObjectLayer can not be null"); - if (TargetTriple) { - auto TT = Triple(TargetTriple); - auto LibrarySymsGenerator = - StaticLibraryDefinitionGenerator::Load(*unwrap(ObjLayer), FileName, TT); - if (!LibrarySymsGenerator) { - *Result = nullptr; - return wrap(LibrarySymsGenerator.takeError()); - } - *Result = wrap(LibrarySymsGenerator->release()); - return LLVMErrorSuccess; - } else { - auto LibrarySymsGenerator = - StaticLibraryDefinitionGenerator::Load(*unwrap(ObjLayer), FileName); - if (!LibrarySymsGenerator) { - *Result = nullptr; - return wrap(LibrarySymsGenerator.takeError()); - } - *Result = wrap(LibrarySymsGenerator->release()); - return LLVMErrorSuccess; + auto LibrarySymsGenerator = + StaticLibraryDefinitionGenerator::Load(*unwrap(ObjLayer), FileName); + if (!LibrarySymsGenerator) { + *Result = nullptr; + return wrap(LibrarySymsGenerator.takeError()); } + *Result = wrap(LibrarySymsGenerator->release()); + return LLVMErrorSuccess; } LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void) { @@ -859,9 +847,9 @@ LLVMErrorRef LLVMOrcObjectLayerAddObjectFile(LLVMOrcObjectLayerRef ObjLayer, *unwrap(JD), std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); } -LLVMErrorRef LLVMOrcLLJITAddObjectFileWithRT(LLVMOrcObjectLayerRef ObjLayer, - LLVMOrcResourceTrackerRef RT, - LLVMMemoryBufferRef ObjBuffer) { +LLVMErrorRef LLVMOrcObjectLayerAddObjectFileWithRT(LLVMOrcObjectLayerRef ObjLayer, + LLVMOrcResourceTrackerRef RT, + LLVMMemoryBufferRef ObjBuffer) { return wrap( unwrap(ObjLayer)->add(ResourceTrackerSP(unwrap(RT)), std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); @@ -1210,8 +1198,8 @@ LLVMErrorRef LLVMOrcCreateLocalLazyCallThroughManager( const char *TargetTriple, LLVMOrcExecutionSessionRef ES, LLVMOrcJITTargetAddress ErrorHandlerAddr, LLVMOrcLazyCallThroughManagerRef *Result) { - auto LCTM = createLocalLazyCallThroughManager(Triple(TargetTriple), - *unwrap(ES), ErrorHandlerAddr); + auto LCTM = createLocalLazyCallThroughManager( + Triple(TargetTriple), *unwrap(ES), ExecutorAddr(ErrorHandlerAddr)); if (!LCTM) return wrap(LCTM.takeError()); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 07b19b2e54f1..9ef333222028 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -38,7 +38,8 @@ public: LookupResult Result; for (auto &KV : *InternedResult) - Result[*KV.first] = std::move(KV.second); + Result[*KV.first] = {KV.second.getAddress().getValue(), + KV.second.getFlags()}; OnResolved(Result); }; @@ -326,7 +327,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( } else if (AutoClaimObjectSymbols) ExtraSymbolsToClaim[InternedName] = Flags; - Symbols[InternedName] = JITEvaluatedSymbol(KV.second.getAddress(), Flags); + Symbols[InternedName] = {ExecutorAddr(KV.second.getAddress()), Flags}; } if (!ExtraSymbolsToClaim.empty()) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp new file mode 100644 index 000000000000..ecf5e2915773 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp @@ -0,0 +1,94 @@ +//===---------- ObjectFormats.cpp - Object format details for ORC ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ORC-specific object format details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" + +namespace llvm { +namespace orc { + +StringRef MachODataCommonSectionName = "__DATA,__common"; +StringRef MachODataDataSectionName = "__DATA,__data"; +StringRef MachOEHFrameSectionName = "__TEXT,__eh_frame"; +StringRef MachOCompactUnwindInfoSectionName = "__TEXT,__unwind_info"; +StringRef MachOModInitFuncSectionName = "__DATA,__mod_init_func"; +StringRef MachOObjCCatListSectionName = "__DATA,__objc_catlist"; +StringRef MachOObjCCatList2SectionName = "__DATA,__objc_catlist2"; +StringRef MachOObjCClassListSectionName = "__DATA,__objc_classlist"; +StringRef MachOObjCClassNameSectionName = "__TEXT,__objc_classname"; +StringRef MachOObjCClassRefsSectionName = "__DATA,__objc_classrefs"; +StringRef MachOObjCConstSectionName = "__DATA,__objc_const"; +StringRef MachOObjCDataSectionName = "__DATA,__objc_data"; +StringRef MachOObjCImageInfoSectionName = "__DATA,__objc_imageinfo"; +StringRef MachOObjCMethNameSectionName = "__TEXT,__objc_methname"; +StringRef MachOObjCMethTypeSectionName = "__TEXT,__objc_methtype"; +StringRef MachOObjCNLCatListSectionName = "__DATA,__objc_nlcatlist"; +StringRef MachOObjCSelRefsSectionName = "__DATA,__objc_selrefs"; +StringRef MachOSwift5ProtoSectionName = "__TEXT,__swift5_proto"; +StringRef MachOSwift5ProtosSectionName = "__TEXT,__swift5_protos"; +StringRef MachOSwift5TypesSectionName = "__TEXT,__swift5_types"; +StringRef MachOSwift5TypeRefSectionName = "__TEXT,__swift5_typeref"; +StringRef MachOSwift5FieldMetadataSectionName = "__TEXT,__swift5_fieldmd"; +StringRef MachOSwift5EntrySectionName = "__TEXT,__swift5_entry"; +StringRef MachOThreadBSSSectionName = "__DATA,__thread_bss"; +StringRef MachOThreadDataSectionName = "__DATA,__thread_data"; +StringRef MachOThreadVarsSectionName = "__DATA,__thread_vars"; + +StringRef MachOInitSectionNames[19] = { + MachOModInitFuncSectionName, MachOObjCCatListSectionName, + MachOObjCCatList2SectionName, MachOObjCClassListSectionName, + MachOObjCClassNameSectionName, MachOObjCClassRefsSectionName, + MachOObjCConstSectionName, MachOObjCDataSectionName, + MachOObjCImageInfoSectionName, MachOObjCMethNameSectionName, + MachOObjCMethTypeSectionName, MachOObjCNLCatListSectionName, + MachOObjCSelRefsSectionName, MachOSwift5ProtoSectionName, + MachOSwift5ProtosSectionName, MachOSwift5TypesSectionName, + MachOSwift5TypeRefSectionName, MachOSwift5FieldMetadataSectionName, + MachOSwift5EntrySectionName, +}; + +StringRef ELFEHFrameSectionName = ".eh_frame"; +StringRef ELFInitArrayFuncSectionName = ".init_array"; + +StringRef ELFThreadBSSSectionName = ".tbss"; +StringRef ELFThreadDataSectionName = ".tdata"; + +bool isMachOInitializerSection(StringRef SegName, StringRef SecName) { + for (auto &InitSection : MachOInitSectionNames) { + // Loop below assumes all MachO init sectios have a length-6 + // segment name. + assert(InitSection[6] == ',' && "Init section seg name has length != 6"); + if (InitSection.starts_with(SegName) && InitSection.substr(7) == SecName) + return true; + } + return false; +} + +bool isMachOInitializerSection(StringRef QualifiedName) { + for (auto &InitSection : MachOInitSectionNames) + if (InitSection == QualifiedName) + return true; + return false; +} + +bool isELFInitializerSection(StringRef SecName) { + if (SecName.consume_front(ELFInitArrayFuncSectionName) && + (SecName.empty() || SecName[0] == '.')) + return true; + return false; +} + +bool isCOFFInitializerSection(StringRef SecName) { + return SecName.startswith(".CRT"); +} + +} // namespace orc +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp index 1bd10c9c6c0e..3d3ca891d881 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp @@ -49,7 +49,7 @@ Expected<int32_t> SimpleRemoteEPC::runAsMain(ExecutorAddr MainFnAddr, ArrayRef<std::string> Args) { int64_t Result = 0; if (auto Err = callSPSWrapper<rt::SPSRunAsMainSignature>( - RunAsMainAddr, Result, ExecutorAddr(MainFnAddr), Args)) + RunAsMainAddr, Result, MainFnAddr, Args)) return std::move(Err); return Result; } @@ -57,7 +57,7 @@ Expected<int32_t> SimpleRemoteEPC::runAsMain(ExecutorAddr MainFnAddr, Expected<int32_t> SimpleRemoteEPC::runAsVoidFunction(ExecutorAddr VoidFnAddr) { int32_t Result = 0; if (auto Err = callSPSWrapper<rt::SPSRunAsVoidFunctionSignature>( - RunAsVoidFunctionAddr, Result, ExecutorAddr(VoidFnAddr))) + RunAsVoidFunctionAddr, Result, VoidFnAddr)) return std::move(Err); return Result; } @@ -66,7 +66,7 @@ Expected<int32_t> SimpleRemoteEPC::runAsIntFunction(ExecutorAddr IntFnAddr, int Arg) { int32_t Result = 0; if (auto Err = callSPSWrapper<rt::SPSRunAsIntFunctionSignature>( - RunAsIntFunctionAddr, Result, ExecutorAddr(IntFnAddr), Arg)) + RunAsIntFunctionAddr, Result, IntFnAddr, Arg)) return std::move(Err); return Result; } @@ -126,23 +126,22 @@ SimpleRemoteEPC::handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, case SimpleRemoteEPCOpcode::Setup: dbgs() << "Setup"; assert(SeqNo == 0 && "Non-zero SeqNo for Setup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Setup?"); + assert(!TagAddr && "Non-zero TagAddr for Setup?"); break; case SimpleRemoteEPCOpcode::Hangup: dbgs() << "Hangup"; assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + assert(!TagAddr && "Non-zero TagAddr for Hangup?"); break; case SimpleRemoteEPCOpcode::Result: dbgs() << "Result"; - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + assert(!TagAddr && "Non-zero TagAddr for Result?"); break; case SimpleRemoteEPCOpcode::CallWrapper: dbgs() << "CallWrapper"; break; } - dbgs() << ", seqno = " << SeqNo - << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + dbgs() << ", seqno = " << SeqNo << ", tag-addr = " << TagAddr << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) << " bytes\n"; }); @@ -227,11 +226,11 @@ Error SimpleRemoteEPC::sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, case SimpleRemoteEPCOpcode::Hangup: dbgs() << "Hangup"; assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + assert(!TagAddr && "Non-zero TagAddr for Hangup?"); break; case SimpleRemoteEPCOpcode::Result: dbgs() << "Result"; - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + assert(!TagAddr && "Non-zero TagAddr for Result?"); break; case SimpleRemoteEPCOpcode::CallWrapper: dbgs() << "CallWrapper"; @@ -239,8 +238,7 @@ Error SimpleRemoteEPC::sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, default: llvm_unreachable("Invalid opcode"); } - dbgs() << ", seqno = " << SeqNo - << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + dbgs() << ", seqno = " << SeqNo << ", tag-addr = " << TagAddr << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) << " bytes\n"; }); @@ -317,13 +315,19 @@ Error SimpleRemoteEPC::setup(Setup S) { dbgs() << "SimpleRemoteEPC received setup message:\n" << " Triple: " << EI->TargetTriple << "\n" << " Page size: " << EI->PageSize << "\n" - << " Bootstrap symbols:\n"; + << " Bootstrap map" << (EI->BootstrapMap.empty() ? " empty" : ":") + << "\n"; + for (const auto &KV : EI->BootstrapMap) + dbgs() << " " << KV.first() << ": " << KV.second.size() + << "-byte SPS encoded buffer\n"; + dbgs() << " Bootstrap symbols" + << (EI->BootstrapSymbols.empty() ? " empty" : ":") << "\n"; for (const auto &KV : EI->BootstrapSymbols) - dbgs() << " " << KV.first() << ": " - << formatv("{0:x16}", KV.second.getValue()) << "\n"; + dbgs() << " " << KV.first() << ": " << KV.second << "\n"; }); TargetTriple = Triple(EI->TargetTriple); PageSize = EI->PageSize; + BootstrapMap = std::move(EI->BootstrapMap); BootstrapSymbols = std::move(EI->BootstrapSymbols); if (auto Err = getBootstrapSymbols( @@ -402,7 +406,7 @@ void SimpleRemoteEPC::handleCallWrapper( ExecutorAddr(), {WFR.data(), WFR.size()})) getExecutionSession().reportError(std::move(Err)); }, - TagAddr.getValue(), ArgBytes); + TagAddr, ArgBytes); }, "callWrapper task")); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp index b52d01318c0d..d4cbd1970d8f 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp @@ -36,16 +36,15 @@ void ImplSymbolMap::trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD) { // Trigger Speculative Compiles. void Speculator::speculateForEntryPoint(Speculator *Ptr, uint64_t StubId) { assert(Ptr && " Null Address Received in orc_speculate_for "); - Ptr->speculateFor(StubId); + Ptr->speculateFor(ExecutorAddr(StubId)); } Error Speculator::addSpeculationRuntime(JITDylib &JD, MangleAndInterner &Mangle) { - JITEvaluatedSymbol ThisPtr(pointerToJITTargetAddress(this), - JITSymbolFlags::Exported); - JITEvaluatedSymbol SpeculateForEntryPtr( - pointerToJITTargetAddress(&speculateForEntryPoint), - JITSymbolFlags::Exported); + ExecutorSymbolDef ThisPtr(ExecutorAddr::fromPtr(this), + JITSymbolFlags::Exported); + ExecutorSymbolDef SpeculateForEntryPtr( + ExecutorAddr::fromPtr(&speculateForEntryPoint), JITSymbolFlags::Exported); return JD.define(absoluteSymbols({ {Mangle("__orc_speculator"), ThisPtr}, // Data Symbol {Mangle("__orc_speculate_for"), SpeculateForEntryPtr} // Callable Symbol diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp index 147f915f61d6..3f70dbf60437 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp @@ -132,11 +132,11 @@ Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( #if defined(LLVM_ON_UNIX) int NativeProt = 0; - if ((Segment.AG.getMemProt() & MemProt::Read) == MemProt::Read) + if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read) NativeProt |= PROT_READ; - if ((Segment.AG.getMemProt() & MemProt::Write) == MemProt::Write) + if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write) NativeProt |= PROT_WRITE; - if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) + if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec) NativeProt |= PROT_EXEC; if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt)) @@ -144,8 +144,7 @@ Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( #elif defined(_WIN32) - DWORD NativeProt = - getWindowsProtectionFlags(Segment.AG.getMemProt()); + DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot); if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt, &NativeProt)) @@ -153,7 +152,7 @@ Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( #endif - if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) + if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec) sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(), Segment.Size); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp index 8296b03398a0..8eca874c48b8 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp @@ -67,9 +67,9 @@ using namespace llvm; using namespace llvm::orc; // Register debug object, return error message or null for success. -static void registerJITLoaderGDBImpl(const char *ObjAddr, size_t Size) { +static void appendJITDebugDescriptor(const char *ObjAddr, size_t Size) { LLVM_DEBUG({ - dbgs() << "Registering debug object with GDB JIT interface " + dbgs() << "Adding debug object to GDB JIT interface " << formatv("([{0:x16} -- {1:x16}])", reinterpret_cast<uintptr_t>(ObjAddr), reinterpret_cast<uintptr_t>(ObjAddr + Size)) @@ -94,20 +94,20 @@ static void registerJITLoaderGDBImpl(const char *ObjAddr, size_t Size) { __jit_debug_descriptor.first_entry = E; __jit_debug_descriptor.relevant_entry = E; - - // Run into the rendezvous breakpoint. __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; - __jit_debug_register_code(); } extern "C" orc::shared::CWrapperFunctionResult llvm_orc_registerJITLoaderGDBAllocAction(const char *Data, size_t Size) { using namespace orc::shared; - return WrapperFunction<SPSError(SPSExecutorAddrRange)>::handle( + return WrapperFunction<SPSError(SPSExecutorAddrRange, bool)>::handle( Data, Size, - [](ExecutorAddrRange R) { - registerJITLoaderGDBImpl(R.Start.toPtr<const char *>(), + [](ExecutorAddrRange R, bool AutoRegisterCode) { + appendJITDebugDescriptor(R.Start.toPtr<const char *>(), R.size()); + // Run into the rendezvous breakpoint. + if (AutoRegisterCode) + __jit_debug_register_code(); return Error::success(); }) .release(); @@ -116,11 +116,14 @@ llvm_orc_registerJITLoaderGDBAllocAction(const char *Data, size_t Size) { extern "C" orc::shared::CWrapperFunctionResult llvm_orc_registerJITLoaderGDBWrapper(const char *Data, uint64_t Size) { using namespace orc::shared; - return WrapperFunction<SPSError(SPSExecutorAddrRange)>::handle( + return WrapperFunction<SPSError(SPSExecutorAddrRange, bool)>::handle( Data, Size, - [](ExecutorAddrRange R) { - registerJITLoaderGDBImpl(R.Start.toPtr<const char *>(), + [](ExecutorAddrRange R, bool AutoRegisterCode) { + appendJITDebugDescriptor(R.Start.toPtr<const char *>(), R.size()); + // Run into the rendezvous breakpoint. + if (AutoRegisterCode) + __jit_debug_register_code(); return Error::success(); }) .release(); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp index ce94bf1e039a..4da031716e32 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp @@ -132,9 +132,9 @@ Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) { assert(Seg.Size <= std::numeric_limits<size_t>::max()); if (auto EC = sys::Memory::protectMappedMemory( {Mem, static_cast<size_t>(Seg.Size)}, - toSysMemoryProtectionFlags(Seg.AG.getMemProt()))) + toSysMemoryProtectionFlags(Seg.RAG.Prot))) return BailOut(errorCodeToError(EC)); - if ((Seg.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) + if ((Seg.RAG.Prot & MemProt::Exec) == MemProt::Exec) sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp index 8ab0af3eab6e..67bc379f9821 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp @@ -10,8 +10,8 @@ #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/Host.h" #include "llvm/Support/Process.h" +#include "llvm/TargetParser/Host.h" #include "OrcRTBootstrap.h" @@ -68,23 +68,22 @@ SimpleRemoteEPCServer::handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, case SimpleRemoteEPCOpcode::Setup: dbgs() << "Setup"; assert(SeqNo == 0 && "Non-zero SeqNo for Setup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Setup?"); + assert(!TagAddr && "Non-zero TagAddr for Setup?"); break; case SimpleRemoteEPCOpcode::Hangup: dbgs() << "Hangup"; assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + assert(!TagAddr && "Non-zero TagAddr for Hangup?"); break; case SimpleRemoteEPCOpcode::Result: dbgs() << "Result"; - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + assert(!TagAddr && "Non-zero TagAddr for Result?"); break; case SimpleRemoteEPCOpcode::CallWrapper: dbgs() << "CallWrapper"; break; } - dbgs() << ", seqno = " << SeqNo - << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + dbgs() << ", seqno = " << SeqNo << ", tag-addr = " << TagAddr << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) << " bytes\n"; }); @@ -158,23 +157,22 @@ Error SimpleRemoteEPCServer::sendMessage(SimpleRemoteEPCOpcode OpC, case SimpleRemoteEPCOpcode::Setup: dbgs() << "Setup"; assert(SeqNo == 0 && "Non-zero SeqNo for Setup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Setup?"); + assert(!TagAddr && "Non-zero TagAddr for Setup?"); break; case SimpleRemoteEPCOpcode::Hangup: dbgs() << "Hangup"; assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + assert(!TagAddr && "Non-zero TagAddr for Hangup?"); break; case SimpleRemoteEPCOpcode::Result: dbgs() << "Result"; - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + assert(!TagAddr && "Non-zero TagAddr for Result?"); break; case SimpleRemoteEPCOpcode::CallWrapper: dbgs() << "CallWrapper"; break; } - dbgs() << ", seqno = " << SeqNo - << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + dbgs() << ", seqno = " << SeqNo << ", tag-addr = " << TagAddr << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) << " bytes\n"; }); @@ -187,6 +185,7 @@ Error SimpleRemoteEPCServer::sendMessage(SimpleRemoteEPCOpcode OpC, } Error SimpleRemoteEPCServer::sendSetupMessage( + StringMap<std::vector<char>> BootstrapMap, StringMap<ExecutorAddr> BootstrapSymbols) { using namespace SimpleRemoteEPCDefaultBootstrapSymbolNames; @@ -198,6 +197,7 @@ Error SimpleRemoteEPCServer::sendSetupMessage( EI.PageSize = *PageSize; else return PageSize.takeError(); + EI.BootstrapMap = std::move(BootstrapMap); EI.BootstrapSymbols = std::move(BootstrapSymbols); assert(!EI.BootstrapSymbols.count(ExecutorSessionObjectName) && |
