diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ExecutionEngine/Orc')
33 files changed, 4397 insertions, 572 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index 5b73c0e2fbc8..9ff6cec8c6c5 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -184,6 +184,8 @@ void CompileOnDemandLayer::emit( CompileOnDemandLayer::PerDylibResources & CompileOnDemandLayer::getPerDylibResources(JITDylib &TargetD) { + std::lock_guard<std::mutex> Lock(CODLayerMutex); + auto I = DylibResources.find(&TargetD); if (I == DylibResources.end()) { auto &ImplD = diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp index 12a501f7f98c..64e5090e4c53 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -29,7 +29,6 @@ char SymbolsNotFound::ID = 0; char SymbolsCouldNotBeRemoved::ID = 0; char MissingSymbolDefinitions::ID = 0; char UnexpectedSymbolDefinitions::ID = 0; -char Task::ID = 0; char MaterializationTask::ID = 0; RegisterDependenciesFunction NoDependenciesToRegister = @@ -90,14 +89,17 @@ void FailedToMaterialize::log(raw_ostream &OS) const { OS << "Failed to materialize symbols: " << *Symbols; } -SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols) { +SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP, + SymbolNameSet Symbols) + : SSP(std::move(SSP)) { for (auto &Sym : Symbols) this->Symbols.push_back(Sym); assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); } -SymbolsNotFound::SymbolsNotFound(SymbolNameVector Symbols) - : Symbols(std::move(Symbols)) { +SymbolsNotFound::SymbolsNotFound(std::shared_ptr<SymbolStringPool> SSP, + SymbolNameVector Symbols) + : SSP(std::move(SSP)), Symbols(std::move(Symbols)) { assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); } @@ -109,8 +111,9 @@ void SymbolsNotFound::log(raw_ostream &OS) const { OS << "Symbols not found: " << Symbols; } -SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved(SymbolNameSet Symbols) - : Symbols(std::move(Symbols)) { +SymbolsCouldNotBeRemoved::SymbolsCouldNotBeRemoved( + std::shared_ptr<SymbolStringPool> SSP, SymbolNameSet Symbols) + : SSP(std::move(SSP)), Symbols(std::move(Symbols)) { assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); } @@ -1333,11 +1336,13 @@ Error JITDylib::remove(const SymbolNameSet &Names) { // If any of the symbols are not defined, return an error. if (!Missing.empty()) - return make_error<SymbolsNotFound>(std::move(Missing)); + return make_error<SymbolsNotFound>(ES.getSymbolStringPool(), + std::move(Missing)); // If any of the symbols are currently materializing, return an error. if (!Materializing.empty()) - return make_error<SymbolsCouldNotBeRemoved>(std::move(Materializing)); + return make_error<SymbolsCouldNotBeRemoved>(ES.getSymbolStringPool(), + std::move(Materializing)); // Remove the symbols. for (auto &SymbolMaterializerItrPair : SymbolsToRemove) { @@ -1793,8 +1798,6 @@ void Platform::lookupInitSymbolsAsync( } } -void Task::anchor() {} - void MaterializationTask::printDescription(raw_ostream &OS) { OS << "Materialization task: " << MU->getName() << " in " << MR->getTargetJITDylib().getName(); @@ -2086,8 +2089,8 @@ Error ExecutionSession::registerJITDispatchHandlers( } void ExecutionSession::runJITDispatchHandler( - ExecutorProcessControl::SendResultFunction SendResult, - JITTargetAddress HandlerFnTagAddr, ArrayRef<char> ArgBuffer) { + SendResultFunction SendResult, JITTargetAddress HandlerFnTagAddr, + ArrayRef<char> ArgBuffer) { std::shared_ptr<JITDispatchHandlerFunction> F; { @@ -2234,7 +2237,8 @@ Error ExecutionSession::IL_updateCandidatesFor( // weakly referenced" specific error here to reduce confusion. if (SymI->second.getFlags().hasMaterializationSideEffectsOnly() && SymLookupFlags != SymbolLookupFlags::WeaklyReferencedSymbol) - return make_error<SymbolsNotFound>(SymbolNameVector({Name})); + return make_error<SymbolsNotFound>(getSymbolStringPool(), + SymbolNameVector({Name})); // If we matched against this symbol but it is in the error state // then bail out and treat it as a failure to materialize. @@ -2422,7 +2426,7 @@ void ExecutionSession::OL_applyQueryPhase1( } else { LLVM_DEBUG(dbgs() << "Phase 1 failed with unresolved symbols.\n"); IPLS->fail(make_error<SymbolsNotFound>( - IPLS->DefGeneratorCandidates.getSymbolNames())); + getSymbolStringPool(), IPLS->DefGeneratorCandidates.getSymbolNames())); } } @@ -2492,7 +2496,8 @@ void ExecutionSession::OL_completeLookup( dbgs() << "error: " "required, but symbol is has-side-effects-only\n"; }); - return make_error<SymbolsNotFound>(SymbolNameVector({Name})); + return make_error<SymbolsNotFound>(getSymbolStringPool(), + SymbolNameVector({Name})); } // If we matched against this symbol but it is in the error state @@ -2594,7 +2599,7 @@ void ExecutionSession::OL_completeLookup( } } - LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-refererced symbols\n"); + LLVM_DEBUG(dbgs() << "Stripping unmatched weakly-referenced symbols\n"); IPLS->LookupSet.forEachWithRemoval( [&](const SymbolStringPtr &Name, SymbolLookupFlags SymLookupFlags) { if (SymLookupFlags == SymbolLookupFlags::WeaklyReferencedSymbol) { @@ -2606,7 +2611,8 @@ void ExecutionSession::OL_completeLookup( if (!IPLS->LookupSet.empty()) { LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n"); - return make_error<SymbolsNotFound>(IPLS->LookupSet.getSymbolNames()); + return make_error<SymbolsNotFound>(getSymbolStringPool(), + IPLS->LookupSet.getSymbolNames()); } // Record whether the query completed. @@ -2733,7 +2739,8 @@ void ExecutionSession::OL_completeLookupFlags( if (!IPLS->LookupSet.empty()) { LLVM_DEBUG(dbgs() << "Failing due to unresolved symbols\n"); - return make_error<SymbolsNotFound>(IPLS->LookupSet.getSymbolNames()); + return make_error<SymbolsNotFound>(getSymbolStringPool(), + IPLS->LookupSet.getSymbolNames()); } LLVM_DEBUG(dbgs() << "Succeded, result = " << Result << "\n"); @@ -2911,6 +2918,7 @@ void ExecutionSession::dumpDispatchInfo(Task &T) { runSessionLocked([&]() { dbgs() << "Dispatching: "; T.printDescription(dbgs()); + dbgs() << "\n"; }); } #endif // NDEBUG diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp index 36efc744bf30..fcfe389f82a8 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp @@ -1,10 +1,15 @@ -//===---- DebugObjectManagerPlugin.h - JITLink debug objects ---*- C++ -*-===// +//===------- DebugObjectManagerPlugin.cpp - JITLink debug objects ---------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// +// +// FIXME: Update Plugin to poke the debug object into a new JITLink section, +// rather than creating a new allocation. +// +//===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" @@ -108,70 +113,77 @@ void ELFDebugObjectSection<ELFT>::dump(raw_ostream &OS, StringRef Name) { } } -static constexpr sys::Memory::ProtectionFlags ReadOnly = - static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ); - enum class Requirement { // Request final target memory load-addresses for all sections. ReportFinalSectionLoadAddresses, }; -/// The plugin creates a debug object from JITLinkContext when JITLink starts -/// processing the corresponding LinkGraph. It provides access to the pass -/// configuration of the LinkGraph and calls the finalization function, once -/// the resulting link artifact was emitted. +/// The plugin creates a debug object from when JITLink starts processing the +/// corresponding LinkGraph. It provides access to the pass configuration of +/// the LinkGraph and calls the finalization function, once the resulting link +/// artifact was emitted. /// class DebugObject { public: - DebugObject(JITLinkContext &Ctx, ExecutionSession &ES) : Ctx(Ctx), ES(ES) {} + DebugObject(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD, + ExecutionSession &ES) + : MemMgr(MemMgr), JD(JD), ES(ES) {} void set(Requirement Req) { Reqs.insert(Req); } bool has(Requirement Req) const { return Reqs.count(Req) > 0; } - using FinalizeContinuation = std::function<void(Expected<sys::MemoryBlock>)>; + using FinalizeContinuation = std::function<void(Expected<ExecutorAddrRange>)>; + void finalizeAsync(FinalizeContinuation OnFinalize); virtual ~DebugObject() { - if (Alloc) - if (Error Err = Alloc->deallocate()) + if (Alloc) { + std::vector<FinalizedAlloc> Allocs; + Allocs.push_back(std::move(Alloc)); + if (Error Err = MemMgr.deallocate(std::move(Allocs))) ES.reportError(std::move(Err)); + } } virtual void reportSectionTargetMemoryRange(StringRef Name, SectionRange TargetMem) {} protected: - using Allocation = JITLinkMemoryManager::Allocation; + using InFlightAlloc = JITLinkMemoryManager::InFlightAlloc; + using FinalizedAlloc = JITLinkMemoryManager::FinalizedAlloc; - virtual Expected<std::unique_ptr<Allocation>> - finalizeWorkingMemory(JITLinkContext &Ctx) = 0; + virtual Expected<SimpleSegmentAlloc> finalizeWorkingMemory() = 0; + + JITLinkMemoryManager &MemMgr; + const JITLinkDylib *JD = nullptr; private: - JITLinkContext &Ctx; ExecutionSession &ES; std::set<Requirement> Reqs; - std::unique_ptr<Allocation> Alloc{nullptr}; + FinalizedAlloc Alloc; }; // Finalize working memory and take ownership of the resulting allocation. Start // copying memory over to the target and pass on the result once we're done. // Ownership of the allocation remains with us for the rest of our lifetime. void DebugObject::finalizeAsync(FinalizeContinuation OnFinalize) { - assert(Alloc == nullptr && "Cannot finalize more than once"); - - auto AllocOrErr = finalizeWorkingMemory(Ctx); - if (!AllocOrErr) - OnFinalize(AllocOrErr.takeError()); - Alloc = std::move(*AllocOrErr); - - Alloc->finalizeAsync([this, OnFinalize](Error Err) { - if (Err) - OnFinalize(std::move(Err)); - else - OnFinalize(sys::MemoryBlock( - jitTargetAddressToPointer<void *>(Alloc->getTargetMemory(ReadOnly)), - Alloc->getWorkingMemory(ReadOnly).size())); - }); + assert(!Alloc && "Cannot finalize more than once"); + + if (auto SimpleSegAlloc = finalizeWorkingMemory()) { + auto ROSeg = SimpleSegAlloc->getSegInfo(MemProt::Read); + ExecutorAddrRange DebugObjRange(ExecutorAddr(ROSeg.Addr), + ExecutorAddrDiff(ROSeg.WorkingMem.size())); + SimpleSegAlloc->finalize( + [this, DebugObjRange, + OnFinalize = std::move(OnFinalize)](Expected<FinalizedAlloc> FA) { + if (FA) { + Alloc = std::move(*FA); + OnFinalize(DebugObjRange); + } else + OnFinalize(FA.takeError()); + }); + } else + OnFinalize(SimpleSegAlloc.takeError()); } /// The current implementation of ELFDebugObject replicates the approach used in @@ -190,8 +202,7 @@ public: StringRef getBuffer() const { return Buffer->getMemBufferRef().getBuffer(); } protected: - Expected<std::unique_ptr<Allocation>> - finalizeWorkingMemory(JITLinkContext &Ctx) override; + Expected<SimpleSegmentAlloc> finalizeWorkingMemory() override; template <typename ELFT> Error recordSection(StringRef Name, @@ -201,15 +212,16 @@ protected: private: template <typename ELFT> static Expected<std::unique_ptr<ELFDebugObject>> - CreateArchType(MemoryBufferRef Buffer, JITLinkContext &Ctx, - ExecutionSession &ES); + CreateArchType(MemoryBufferRef Buffer, JITLinkMemoryManager &MemMgr, + const JITLinkDylib *JD, ExecutionSession &ES); static std::unique_ptr<WritableMemoryBuffer> CopyBuffer(MemoryBufferRef Buffer, Error &Err); ELFDebugObject(std::unique_ptr<WritableMemoryBuffer> Buffer, - JITLinkContext &Ctx, ExecutionSession &ES) - : DebugObject(Ctx, ES), Buffer(std::move(Buffer)) { + JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD, + ExecutionSession &ES) + : DebugObject(MemMgr, JD, ES), Buffer(std::move(Buffer)) { set(Requirement::ReportFinalSectionLoadAddresses); } @@ -244,13 +256,14 @@ ELFDebugObject::CopyBuffer(MemoryBufferRef Buffer, Error &Err) { template <typename ELFT> Expected<std::unique_ptr<ELFDebugObject>> -ELFDebugObject::CreateArchType(MemoryBufferRef Buffer, JITLinkContext &Ctx, - ExecutionSession &ES) { +ELFDebugObject::CreateArchType(MemoryBufferRef Buffer, + JITLinkMemoryManager &MemMgr, + const JITLinkDylib *JD, ExecutionSession &ES) { using SectionHeader = typename ELFT::Shdr; Error Err = Error::success(); std::unique_ptr<ELFDebugObject> DebugObj( - new ELFDebugObject(CopyBuffer(Buffer, Err), Ctx, ES)); + new ELFDebugObject(CopyBuffer(Buffer, Err), MemMgr, JD, ES)); if (Err) return std::move(Err); @@ -299,23 +312,26 @@ ELFDebugObject::Create(MemoryBufferRef Buffer, JITLinkContext &Ctx, if (Class == ELF::ELFCLASS32) { if (Endian == ELF::ELFDATA2LSB) - return CreateArchType<ELF32LE>(Buffer, Ctx, ES); + return CreateArchType<ELF32LE>(Buffer, Ctx.getMemoryManager(), + Ctx.getJITLinkDylib(), ES); if (Endian == ELF::ELFDATA2MSB) - return CreateArchType<ELF32BE>(Buffer, Ctx, ES); + return CreateArchType<ELF32BE>(Buffer, Ctx.getMemoryManager(), + Ctx.getJITLinkDylib(), ES); return nullptr; } if (Class == ELF::ELFCLASS64) { if (Endian == ELF::ELFDATA2LSB) - return CreateArchType<ELF64LE>(Buffer, Ctx, ES); + return CreateArchType<ELF64LE>(Buffer, Ctx.getMemoryManager(), + Ctx.getJITLinkDylib(), ES); if (Endian == ELF::ELFDATA2MSB) - return CreateArchType<ELF64BE>(Buffer, Ctx, ES); + return CreateArchType<ELF64BE>(Buffer, Ctx.getMemoryManager(), + Ctx.getJITLinkDylib(), ES); return nullptr; } return nullptr; } -Expected<std::unique_ptr<DebugObject::Allocation>> -ELFDebugObject::finalizeWorkingMemory(JITLinkContext &Ctx) { +Expected<SimpleSegmentAlloc> ELFDebugObject::finalizeWorkingMemory() { LLVM_DEBUG({ dbgs() << "Section load-addresses in debug object for \"" << Buffer->getBufferIdentifier() << "\":\n"; @@ -324,28 +340,21 @@ ELFDebugObject::finalizeWorkingMemory(JITLinkContext &Ctx) { }); // TODO: This works, but what actual alignment requirements do we have? - unsigned Alignment = sys::Process::getPageSizeEstimate(); - JITLinkMemoryManager &MemMgr = Ctx.getMemoryManager(); - const JITLinkDylib *JD = Ctx.getJITLinkDylib(); + unsigned PageSize = sys::Process::getPageSizeEstimate(); size_t Size = Buffer->getBufferSize(); // Allocate working memory for debug object in read-only segment. - JITLinkMemoryManager::SegmentsRequestMap SingleReadOnlySegment; - SingleReadOnlySegment[ReadOnly] = - JITLinkMemoryManager::SegmentRequest(Alignment, Size, 0); - - auto AllocOrErr = MemMgr.allocate(JD, SingleReadOnlySegment); - if (!AllocOrErr) - return AllocOrErr.takeError(); + auto Alloc = SimpleSegmentAlloc::Create( + MemMgr, JD, {{MemProt::Read, {Size, Align(PageSize)}}}); + if (!Alloc) + return Alloc; // Initialize working memory with a copy of our object buffer. - // TODO: Use our buffer as working memory directly. - std::unique_ptr<Allocation> Alloc = std::move(*AllocOrErr); - MutableArrayRef<char> WorkingMem = Alloc->getWorkingMemory(ReadOnly); - memcpy(WorkingMem.data(), Buffer->getBufferStart(), Size); + auto SegInfo = Alloc->getSegInfo(MemProt::Read); + memcpy(SegInfo.WorkingMem.data(), Buffer->getBufferStart(), Size); Buffer.reset(); - return std::move(Alloc); + return Alloc; } void ELFDebugObject::reportSectionTargetMemoryRange(StringRef Name, @@ -447,7 +456,7 @@ Error DebugObjectManagerPlugin::notifyEmitted( std::future<MSVCPError> FinalizeErr = FinalizePromise.get_future(); It->second->finalizeAsync( - [this, &FinalizePromise, &MR](Expected<sys::MemoryBlock> TargetMem) { + [this, &FinalizePromise, &MR](Expected<ExecutorAddrRange> TargetMem) { // Any failure here will fail materialization. if (!TargetMem) { FinalizePromise.set_value(TargetMem.takeError()); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp new file mode 100644 index 000000000000..8479495623b8 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp @@ -0,0 +1,450 @@ +//===------- DebuggerSupportPlugin.cpp - Utils for debugger support -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h" + +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringSet.h" +#include "llvm/BinaryFormat/MachO.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +static const char *SynthDebugSectionName = "__jitlink_synth_debug_object"; + +namespace { + +struct MachO64LE { + using UIntPtr = uint64_t; + + using Header = MachO::mach_header_64; + using SegmentLC = MachO::segment_command_64; + using Section = MachO::section_64; + using NList = MachO::nlist_64; + + static constexpr support::endianness Endianness = support::little; + static constexpr const uint32_t Magic = MachO::MH_MAGIC_64; + static constexpr const uint32_t SegmentCmd = MachO::LC_SEGMENT_64; +}; + +class MachODebugObjectSynthesizerBase + : public GDBJITDebugInfoRegistrationPlugin::DebugSectionSynthesizer { +public: + static bool isDebugSection(Section &Sec) { + return Sec.getName().startswith("__DWARF,"); + } + + MachODebugObjectSynthesizerBase(LinkGraph &G, ExecutorAddr RegisterActionAddr) + : G(G), RegisterActionAddr(RegisterActionAddr) {} + virtual ~MachODebugObjectSynthesizerBase() {} + + Error preserveDebugSections() { + if (G.findSectionByName(SynthDebugSectionName)) { + LLVM_DEBUG({ + dbgs() << "MachODebugObjectSynthesizer skipping graph " << G.getName() + << " which contains an unexpected existing " + << SynthDebugSectionName << " section.\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "MachODebugObjectSynthesizer visiting graph " << G.getName() + << "\n"; + }); + for (auto &Sec : G.sections()) { + if (!isDebugSection(Sec)) + continue; + // Preserve blocks in this debug section by marking one existing symbol + // live for each block, and introducing a new live, anonymous symbol for + // each currently unreferenced block. + LLVM_DEBUG({ + dbgs() << " Preserving debug section " << Sec.getName() << "\n"; + }); + SmallSet<Block *, 8> PreservedBlocks; + for (auto *Sym : Sec.symbols()) { + bool NewPreservedBlock = + PreservedBlocks.insert(&Sym->getBlock()).second; + if (NewPreservedBlock) + Sym->setLive(true); + } + for (auto *B : Sec.blocks()) + if (!PreservedBlocks.count(B)) + G.addAnonymousSymbol(*B, 0, 0, false, true); + } + return Error::success(); + } + +protected: + LinkGraph &G; + ExecutorAddr RegisterActionAddr; +}; + +template <typename MachOTraits> +class MachODebugObjectSynthesizer : public MachODebugObjectSynthesizerBase { +private: + class MachOStructWriter { + public: + MachOStructWriter(MutableArrayRef<char> Buffer) : Buffer(Buffer) {} + + size_t getOffset() const { return Offset; } + + template <typename MachOStruct> void write(MachOStruct S) { + assert(Offset + sizeof(S) <= Buffer.size() && + "Container block overflow while constructing debug MachO"); + if (MachOTraits::Endianness != support::endian::system_endianness()) + MachO::swapStruct(S); + memcpy(Buffer.data() + Offset, &S, sizeof(S)); + Offset += sizeof(S); + } + + private: + MutableArrayRef<char> Buffer; + size_t Offset = 0; + }; + +public: + using MachODebugObjectSynthesizerBase::MachODebugObjectSynthesizerBase; + + Error startSynthesis() override { + LLVM_DEBUG({ + dbgs() << "Creating " << SynthDebugSectionName << " for " << G.getName() + << "\n"; + }); + auto &SDOSec = G.createSection(SynthDebugSectionName, MemProt::Read); + + struct DebugSectionInfo { + Section *Sec = nullptr; + StringRef SegName; + StringRef SecName; + JITTargetAddress Alignment = 0; + JITTargetAddress StartAddr = 0; + uint64_t Size = 0; + }; + + SmallVector<DebugSectionInfo, 12> DebugSecInfos; + size_t NumSections = 0; + for (auto &Sec : G.sections()) { + if (llvm::empty(Sec.blocks())) + continue; + + ++NumSections; + if (isDebugSection(Sec)) { + size_t SepPos = Sec.getName().find(','); + if (SepPos > 16 || (Sec.getName().size() - (SepPos + 1) > 16)) { + LLVM_DEBUG({ + dbgs() << "Skipping debug object synthesis for graph " + << G.getName() + << ": encountered non-standard DWARF section name \"" + << Sec.getName() << "\"\n"; + }); + return Error::success(); + } + DebugSecInfos.push_back({&Sec, Sec.getName().substr(0, SepPos), + Sec.getName().substr(SepPos + 1), 0, 0}); + } else + NonDebugSections.push_back(&Sec); + } + + // Create container block. + size_t SectionsCmdSize = + sizeof(typename MachOTraits::Section) * NumSections; + size_t SegmentLCSize = + sizeof(typename MachOTraits::SegmentLC) + SectionsCmdSize; + size_t ContainerBlockSize = + sizeof(typename MachOTraits::Header) + SegmentLCSize; + auto ContainerBlockContent = G.allocateBuffer(ContainerBlockSize); + MachOContainerBlock = + &G.createMutableContentBlock(SDOSec, ContainerBlockContent, 0, 8, 0); + + // Copy debug section blocks and symbols. + JITTargetAddress NextBlockAddr = MachOContainerBlock->getSize(); + for (auto &SI : DebugSecInfos) { + assert(!llvm::empty(SI.Sec->blocks()) && "Empty debug info section?"); + + // Update addresses in debug section. + LLVM_DEBUG({ + dbgs() << " Appending " << SI.Sec->getName() << " (" + << SI.Sec->blocks_size() << " block(s)) at " + << formatv("{0:x8}", NextBlockAddr) << "\n"; + }); + for (auto *B : SI.Sec->blocks()) { + NextBlockAddr = alignToBlock(NextBlockAddr, *B); + B->setAddress(NextBlockAddr); + NextBlockAddr += B->getSize(); + } + + auto &FirstBlock = **SI.Sec->blocks().begin(); + if (FirstBlock.getAlignmentOffset() != 0) + return make_error<StringError>( + "First block in " + SI.Sec->getName() + + " section has non-zero alignment offset", + inconvertibleErrorCode()); + if (FirstBlock.getAlignment() > std::numeric_limits<uint32_t>::max()) + return make_error<StringError>("First block in " + SI.Sec->getName() + + " has alignment >4Gb", + inconvertibleErrorCode()); + + SI.Alignment = FirstBlock.getAlignment(); + SI.StartAddr = FirstBlock.getAddress(); + SI.Size = NextBlockAddr - SI.StartAddr; + G.mergeSections(SDOSec, *SI.Sec); + SI.Sec = nullptr; + } + size_t DebugSectionsSize = NextBlockAddr - MachOContainerBlock->getSize(); + + // Write MachO header and debug section load commands. + MachOStructWriter Writer(MachOContainerBlock->getAlreadyMutableContent()); + typename MachOTraits::Header Hdr; + memset(&Hdr, 0, sizeof(Hdr)); + Hdr.magic = MachOTraits::Magic; + switch (G.getTargetTriple().getArch()) { + case Triple::x86_64: + Hdr.cputype = MachO::CPU_TYPE_X86_64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; + break; + case Triple::aarch64: + Hdr.cputype = MachO::CPU_TYPE_ARM64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; + break; + default: + llvm_unreachable("Unsupported architecture"); + } + Hdr.filetype = MachO::MH_OBJECT; + Hdr.ncmds = 1; + Hdr.sizeofcmds = SegmentLCSize; + Hdr.flags = 0; + Writer.write(Hdr); + + typename MachOTraits::SegmentLC SegLC; + memset(&SegLC, 0, sizeof(SegLC)); + SegLC.cmd = MachOTraits::SegmentCmd; + SegLC.cmdsize = SegmentLCSize; + SegLC.vmaddr = ContainerBlockSize; + SegLC.vmsize = DebugSectionsSize; + SegLC.fileoff = ContainerBlockSize; + SegLC.filesize = DebugSectionsSize; + SegLC.maxprot = + MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE; + SegLC.initprot = + MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE; + SegLC.nsects = NumSections; + SegLC.flags = 0; + Writer.write(SegLC); + + StringSet<> ExistingLongNames; + for (auto &SI : DebugSecInfos) { + typename MachOTraits::Section Sec; + memset(&Sec, 0, sizeof(Sec)); + memcpy(Sec.sectname, SI.SecName.data(), SI.SecName.size()); + memcpy(Sec.segname, SI.SegName.data(), SI.SegName.size()); + Sec.addr = SI.StartAddr; + Sec.size = SI.Size; + Sec.offset = SI.StartAddr; + Sec.align = SI.Alignment; + Sec.reloff = 0; + Sec.nreloc = 0; + Sec.flags = MachO::S_ATTR_DEBUG; + Writer.write(Sec); + } + + // Set MachOContainerBlock to indicate success to + // completeSynthesisAndRegister. + NonDebugSectionsStart = Writer.getOffset(); + return Error::success(); + } + + Error completeSynthesisAndRegister() override { + if (!MachOContainerBlock) { + LLVM_DEBUG({ + dbgs() << "Not writing MachO debug object header for " << G.getName() + << " since createDebugSection failed\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "Writing MachO debug object header for " << G.getName() << "\n"; + }); + + MachOStructWriter Writer( + MachOContainerBlock->getAlreadyMutableContent().drop_front( + NonDebugSectionsStart)); + + unsigned LongSectionNameIdx = 0; + for (auto *Sec : NonDebugSections) { + size_t SepPos = Sec->getName().find(','); + StringRef SegName, SecName; + std::string CustomSecName; + + if ((SepPos == StringRef::npos && Sec->getName().size() <= 16)) { + // No embedded segment name, short section name. + SegName = "__JITLINK_CUSTOM"; + SecName = Sec->getName(); + } else if (SepPos < 16 && (Sec->getName().size() - (SepPos + 1) <= 16)) { + // Canonical embedded segment and section name. + SegName = Sec->getName().substr(0, SepPos); + SecName = Sec->getName().substr(SepPos + 1); + } else { + // Long section name that needs to be truncated. + assert(Sec->getName().size() > 16 && + "Short section name should have been handled above"); + SegName = "__JITLINK_CUSTOM"; + auto IdxStr = std::to_string(++LongSectionNameIdx); + CustomSecName = Sec->getName().substr(0, 15 - IdxStr.size()).str(); + CustomSecName += "."; + CustomSecName += IdxStr; + SecName = StringRef(CustomSecName.data(), 16); + } + + SectionRange R(*Sec); + if (R.getFirstBlock()->getAlignmentOffset() != 0) + return make_error<StringError>( + "While building MachO debug object for " + G.getName() + + " first block has non-zero alignment offset", + inconvertibleErrorCode()); + + typename MachOTraits::Section SecCmd; + memset(&SecCmd, 0, sizeof(SecCmd)); + memcpy(SecCmd.sectname, SecName.data(), SecName.size()); + memcpy(SecCmd.segname, SegName.data(), SegName.size()); + SecCmd.addr = R.getStart(); + SecCmd.size = R.getSize(); + SecCmd.offset = 0; + SecCmd.align = R.getFirstBlock()->getAlignment(); + SecCmd.reloff = 0; + SecCmd.nreloc = 0; + SecCmd.flags = 0; + Writer.write(SecCmd); + } + + SectionRange R(MachOContainerBlock->getSection()); + G.allocActions().push_back( + {{RegisterActionAddr.getValue(), R.getStart(), R.getSize()}, {}}); + return Error::success(); + } + +private: + Block *MachOContainerBlock = nullptr; + SmallVector<Section *, 16> NonDebugSections; + size_t NonDebugSectionsStart = 0; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<GDBJITDebugInfoRegistrationPlugin>> +GDBJITDebugInfoRegistrationPlugin::Create(ExecutionSession &ES, + JITDylib &ProcessJD, + const Triple &TT) { + auto RegisterActionAddr = + TT.isOSBinFormatMachO() + ? ES.intern("_llvm_orc_registerJITLoaderGDBAllocAction") + : ES.intern("llvm_orc_registerJITLoaderGDBAllocAction"); + + if (auto Addr = ES.lookup({&ProcessJD}, RegisterActionAddr)) + return std::make_unique<GDBJITDebugInfoRegistrationPlugin>( + ExecutorAddr(Addr->getAddress())); + else + return Addr.takeError(); +} + +Error GDBJITDebugInfoRegistrationPlugin::notifyFailed( + MaterializationResponsibility &MR) { + return Error::success(); +} + +Error GDBJITDebugInfoRegistrationPlugin::notifyRemovingResources( + ResourceKey K) { + return Error::success(); +} + +void GDBJITDebugInfoRegistrationPlugin::notifyTransferringResources( + ResourceKey DstKey, ResourceKey SrcKey) {} + +void GDBJITDebugInfoRegistrationPlugin::modifyPassConfig( + MaterializationResponsibility &MR, LinkGraph &LG, + PassConfiguration &PassConfig) { + + if (LG.getTargetTriple().getObjectFormat() == Triple::MachO) + modifyPassConfigForMachO(MR, LG, PassConfig); + else { + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unspported graph " + << LG.getName() << "(triple = " << LG.getTargetTriple().str() + << "\n"; + }); + } +} + +void GDBJITDebugInfoRegistrationPlugin::modifyPassConfigForMachO( + MaterializationResponsibility &MR, jitlink::LinkGraph &LG, + jitlink::PassConfiguration &PassConfig) { + + switch (LG.getTargetTriple().getArch()) { + case Triple::x86_64: + case Triple::aarch64: + // Supported, continue. + assert(LG.getPointerSize() == 8 && "Graph has incorrect pointer size"); + assert(LG.getEndianness() == support::little && + "Graph has incorrect endianness"); + break; + default: + // Unsupported. + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unsupported " + << "MachO graph " << LG.getName() + << "(triple = " << LG.getTargetTriple().str() + << ", pointer size = " << LG.getPointerSize() << ", endianness = " + << (LG.getEndianness() == support::big ? "big" : "little") + << ")\n"; + }); + return; + } + + // Scan for debug sections. If we find one then install passes. + bool HasDebugSections = false; + for (auto &Sec : LG.sections()) + if (MachODebugObjectSynthesizerBase::isDebugSection(Sec)) { + HasDebugSections = true; + break; + } + + if (HasDebugSections) { + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() + << " contains debug info. Installing debugger support passes.\n"; + }); + + auto MDOS = std::make_shared<MachODebugObjectSynthesizer<MachO64LE>>( + LG, RegisterActionAddr); + PassConfig.PrePrunePasses.push_back( + [=](LinkGraph &G) { return MDOS->preserveDebugSections(); }); + PassConfig.PostPrunePasses.push_back( + [=](LinkGraph &G) { return MDOS->startSynthesis(); }); + PassConfig.PreFixupPasses.push_back( + [=](LinkGraph &G) { return MDOS->completeSynthesisAndRegister(); }); + } else { + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() + << " contains no debug info. Skipping.\n"; + }); + } +} + +} // namespace orc +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp new file mode 100644 index 000000000000..b17d196f01b6 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp @@ -0,0 +1,818 @@ +//===------ ELFNixPlatform.cpp - Utilities for executing MachO in Orc -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" + +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/Support/BinaryByteStream.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +namespace { + +class DSOHandleMaterializationUnit : public MaterializationUnit { +public: + DSOHandleMaterializationUnit(ELFNixPlatform &ENP, + const SymbolStringPtr &DSOHandleSymbol) + : MaterializationUnit(createDSOHandleSectionSymbols(ENP, DSOHandleSymbol), + DSOHandleSymbol), + ENP(ENP) {} + + StringRef getName() const override { return "DSOHandleMU"; } + + void materialize(std::unique_ptr<MaterializationResponsibility> R) override { + unsigned PointerSize; + support::endianness Endianness; + jitlink::Edge::Kind EdgeKind; + const auto &TT = + ENP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + + switch (TT.getArch()) { + case Triple::x86_64: + PointerSize = 8; + Endianness = support::endianness::little; + EdgeKind = jitlink::x86_64::Pointer64; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + + // void *__dso_handle = &__dso_handle; + auto G = std::make_unique<jitlink::LinkGraph>( + "<DSOHandleMU>", TT, PointerSize, Endianness, + jitlink::getGenericEdgeKindName); + auto &DSOHandleSection = + G->createSection(".data.__dso_handle", jitlink::MemProt::Read); + auto &DSOHandleBlock = G->createContentBlock( + DSOHandleSection, getDSOHandleContent(PointerSize), 0, 8, 0); + auto &DSOHandleSymbol = G->addDefinedSymbol( + DSOHandleBlock, 0, *R->getInitializerSymbol(), DSOHandleBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, false, true); + DSOHandleBlock.addEdge(EdgeKind, 0, DSOHandleSymbol, 0); + + ENP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + } + + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} + +private: + static SymbolFlagsMap + createDSOHandleSectionSymbols(ELFNixPlatform &ENP, + const SymbolStringPtr &DSOHandleSymbol) { + SymbolFlagsMap SymbolFlags; + SymbolFlags[DSOHandleSymbol] = JITSymbolFlags::Exported; + return SymbolFlags; + } + + ArrayRef<char> getDSOHandleContent(size_t PointerSize) { + static const char Content[8] = {0}; + assert(PointerSize <= sizeof Content); + return {Content, PointerSize}; + } + + ELFNixPlatform &ENP; +}; + +StringRef EHFrameSectionName = ".eh_frame"; +StringRef InitArrayFuncSectionName = ".init_array"; + +StringRef ThreadBSSSectionName = ".tbss"; +StringRef ThreadDataSectionName = ".tdata"; + +StringRef InitSectionNames[] = {InitArrayFuncSectionName}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<ELFNixPlatform>> +ELFNixPlatform::Create(ExecutionSession &ES, + ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + Optional<SymbolAliasMap> RuntimeAliases) { + + auto &EPC = ES.getExecutorProcessControl(); + + // If the target is not supported then bail out immediately. + if (!supportedTarget(EPC.getTargetTriple())) + return make_error<StringError>("Unsupported ELFNixPlatform triple: " + + EPC.getTargetTriple().str(), + inconvertibleErrorCode()); + + // Create default aliases if the caller didn't supply any. + if (!RuntimeAliases) + RuntimeAliases = standardPlatformAliases(ES); + + // Define the aliases. + if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases)))) + return std::move(Err); + + // Add JIT-dispatch function support symbols. + if (auto Err = PlatformJD.define(absoluteSymbols( + {{ES.intern("__orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), + JITSymbolFlags::Exported}}, + {ES.intern("__orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), + JITSymbolFlags::Exported}}}))) + return std::move(Err); + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load( + ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple()); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + // Create the instance. + Error Err = Error::success(); + auto P = std::unique_ptr<ELFNixPlatform>( + new ELFNixPlatform(ES, ObjLinkingLayer, PlatformJD, + std::move(*OrcRuntimeArchiveGenerator), Err)); + if (Err) + return std::move(Err); + return std::move(P); +} + +Error ELFNixPlatform::setupJITDylib(JITDylib &JD) { + return JD.define( + std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol)); +} + +Error ELFNixPlatform::notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) { + auto &JD = RT.getJITDylib(); + const auto &InitSym = MU.getInitializerSymbol(); + if (!InitSym) + return Error::success(); + + RegisteredInitSymbols[&JD].add(InitSym, + SymbolLookupFlags::WeaklyReferencedSymbol); + LLVM_DEBUG({ + dbgs() << "ELFNixPlatform: Registered init symbol " << *InitSym + << " for MU " << MU.getName() << "\n"; + }); + return Error::success(); +} + +Error ELFNixPlatform::notifyRemoving(ResourceTracker &RT) { + llvm_unreachable("Not supported yet"); +} + +static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, + ArrayRef<std::pair<const char *, const char *>> AL) { + for (auto &KV : AL) { + auto AliasName = ES.intern(KV.first); + assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map"); + Aliases[std::move(AliasName)] = {ES.intern(KV.second), + JITSymbolFlags::Exported}; + } +} + +SymbolAliasMap ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES) { + SymbolAliasMap Aliases; + addAliases(ES, Aliases, requiredCXXAliases()); + addAliases(ES, Aliases, standardRuntimeUtilityAliases()); + return Aliases; +} + +ArrayRef<std::pair<const char *, const char *>> +ELFNixPlatform::requiredCXXAliases() { + static const std::pair<const char *, const char *> RequiredCXXAliases[] = { + {"__cxa_atexit", "__orc_rt_elfnix_cxa_atexit"}, + {"atexit", "__orc_rt_elfnix_atexit"}}; + + return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases); +} + +ArrayRef<std::pair<const char *, const char *>> +ELFNixPlatform::standardRuntimeUtilityAliases() { + static const std::pair<const char *, const char *> + StandardRuntimeUtilityAliases[] = { + {"__orc_rt_run_program", "__orc_rt_elfnix_run_program"}, + {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}}; + + return ArrayRef<std::pair<const char *, const char *>>( + StandardRuntimeUtilityAliases); +} + +bool ELFNixPlatform::isInitializerSection(StringRef SecName) { + for (auto &Name : InitSectionNames) { + if (Name.equals(SecName)) + return true; + } + return false; +} + +bool ELFNixPlatform::supportedTarget(const Triple &TT) { + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } +} + +ELFNixPlatform::ELFNixPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, + std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), + DSOHandleSymbol(ES.intern("__dso_handle")) { + ErrorAsOutParameter _(&Err); + + ObjLinkingLayer.addPlugin(std::make_unique<ELFNixPlatformPlugin>(*this)); + + PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); + + // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating + // the platform now), so set it up. + if (auto E2 = setupJITDylib(PlatformJD)) { + Err = std::move(E2); + return; + } + + RegisteredInitSymbols[&PlatformJD].add( + DSOHandleSymbol, SymbolLookupFlags::WeaklyReferencedSymbol); + + // Associate wrapper function tags with JIT-side function implementations. + if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { + Err = std::move(E2); + return; + } + + // Lookup addresses of runtime functions callable by the platform, + // call the platform bootstrap function to initialize the platform-state + // object in the executor. + if (auto E2 = bootstrapELFNixRuntime(PlatformJD)) { + Err = std::move(E2); + return; + } +} + +Error ELFNixPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { + ExecutionSession::JITDispatchHandlerAssociationMap WFs; + + using GetInitializersSPSSig = + SPSExpected<SPSELFNixJITDylibInitializerSequence>(SPSString); + WFs[ES.intern("__orc_rt_elfnix_get_initializers_tag")] = + ES.wrapAsyncWithSPS<GetInitializersSPSSig>( + this, &ELFNixPlatform::rt_getInitializers); + + using GetDeinitializersSPSSig = + SPSExpected<SPSELFJITDylibDeinitializerSequence>(SPSExecutorAddr); + WFs[ES.intern("__orc_rt_elfnix_get_deinitializers_tag")] = + ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>( + this, &ELFNixPlatform::rt_getDeinitializers); + + using LookupSymbolSPSSig = + SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString); + WFs[ES.intern("__orc_rt_elfnix_symbol_lookup_tag")] = + ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this, + &ELFNixPlatform::rt_lookupSymbol); + + return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs)); +} + +void ELFNixPlatform::getInitializersBuildSequencePhase( + SendInitializerSequenceFn SendResult, JITDylib &JD, + std::vector<JITDylibSP> DFSLinkOrder) { + ELFNixJITDylibInitializerSequence FullInitSeq; + { + std::lock_guard<std::mutex> Lock(PlatformMutex); + for (auto &InitJD : reverse(DFSLinkOrder)) { + LLVM_DEBUG({ + dbgs() << "ELFNixPlatform: Appending inits for \"" << InitJD->getName() + << "\" to sequence\n"; + }); + auto ISItr = InitSeqs.find(InitJD.get()); + if (ISItr != InitSeqs.end()) { + FullInitSeq.emplace_back(std::move(ISItr->second)); + InitSeqs.erase(ISItr); + } + } + } + + SendResult(std::move(FullInitSeq)); +} + +void ELFNixPlatform::getInitializersLookupPhase( + SendInitializerSequenceFn SendResult, JITDylib &JD) { + + auto DFSLinkOrder = JD.getDFSLinkOrder(); + DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; + ES.runSessionLocked([&]() { + for (auto &InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD.get()); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[InitJD.get()] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } + } + }); + + // If there are no further init symbols to look up then move on to the next + // phase. + if (NewInitSymbols.empty()) { + getInitializersBuildSequencePhase(std::move(SendResult), JD, + std::move(DFSLinkOrder)); + return; + } + + // Otherwise issue a lookup and re-run this phase when it completes. + lookupInitSymbolsAsync( + [this, SendResult = std::move(SendResult), &JD](Error Err) mutable { + if (Err) + SendResult(std::move(Err)); + else + getInitializersLookupPhase(std::move(SendResult), JD); + }, + ES, std::move(NewInitSymbols)); +} + +void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName) { + LLVM_DEBUG({ + dbgs() << "ELFNixPlatform::rt_getInitializers(\"" << JDName << "\")\n"; + }); + + JITDylib *JD = ES.getJITDylibByName(JDName); + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n"; + }); + SendResult(make_error<StringError>("No JITDylib named " + JDName, + inconvertibleErrorCode())); + return; + } + + getInitializersLookupPhase(std::move(SendResult), *JD); +} + +void ELFNixPlatform::rt_getDeinitializers( + SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) { + LLVM_DEBUG({ + dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; + + { + std::lock_guard<std::mutex> Lock(PlatformMutex); + auto I = HandleAddrToJITDylib.find(Handle.getValue()); + if (I != HandleAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error<StringError>("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + SendResult(ELFNixJITDylibDeinitializerSequence()); +} + +void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, + ExecutorAddr Handle, + StringRef SymbolName) { + LLVM_DEBUG({ + dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; + + { + std::lock_guard<std::mutex> Lock(PlatformMutex); + auto I = HandleAddrToJITDylib.find(Handle.getValue()); + if (I != HandleAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error<StringError>("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + // Use functor class to work around XL build compiler issue on AIX. + class RtLookupNotifyComplete { + public: + RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult) + : SendResult(std::move(SendResult)) {} + void operator()(Expected<SymbolMap> Result) { + if (Result) { + assert(Result->size() == 1 && "Unexpected result map count"); + SendResult(ExecutorAddr(Result->begin()->second.getAddress())); + } else { + SendResult(Result.takeError()); + } + } + + private: + SendSymbolAddressFn SendResult; + }; + + ES.lookup( + LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, + SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready, + RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister); +} + +Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) { + + std::pair<const char *, ExecutorAddr *> Symbols[] = { + {"__orc_rt_elfnix_platform_bootstrap", &orc_rt_elfnix_platform_bootstrap}, + {"__orc_rt_elfnix_platform_shutdown", &orc_rt_elfnix_platform_shutdown}, + {"__orc_rt_elfnix_register_object_sections", + &orc_rt_elfnix_register_object_sections}, + {"__orc_rt_elfnix_create_pthread_key", + &orc_rt_elfnix_create_pthread_key}}; + + SymbolLookupSet RuntimeSymbols; + std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> AddrsToRecord; + for (const auto &KV : Symbols) { + auto Name = ES.intern(KV.first); + RuntimeSymbols.add(Name); + AddrsToRecord.push_back({std::move(Name), KV.second}); + } + + auto RuntimeSymbolAddrs = ES.lookup( + {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols); + if (!RuntimeSymbolAddrs) + return RuntimeSymbolAddrs.takeError(); + + for (const auto &KV : AddrsToRecord) { + auto &Name = KV.first; + assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?"); + KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress()); + } + + auto PJDDSOHandle = ES.lookup( + {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, DSOHandleSymbol); + if (!PJDDSOHandle) + return PJDDSOHandle.takeError(); + + if (auto Err = ES.callSPSWrapper<void(uint64_t)>( + orc_rt_elfnix_platform_bootstrap, PJDDSOHandle->getAddress())) + return Err; + + // FIXME: Ordering is fuzzy here. We're probably best off saying + // "behavior is undefined if code that uses the runtime is added before + // the platform constructor returns", then move all this to the constructor. + RuntimeBootstrapped = true; + std::vector<ELFPerObjectSectionsToRegister> DeferredPOSRs; + { + std::lock_guard<std::mutex> Lock(PlatformMutex); + DeferredPOSRs = std::move(BootstrapPOSRs); + } + + for (auto &D : DeferredPOSRs) + if (auto Err = registerPerObjectSections(D)) + return Err; + + return Error::success(); +} + +Error ELFNixPlatform::registerInitInfo( + JITDylib &JD, ArrayRef<jitlink::Section *> InitSections) { + + std::unique_lock<std::mutex> Lock(PlatformMutex); + + ELFNixJITDylibInitializers *InitSeq = nullptr; + { + auto I = InitSeqs.find(&JD); + if (I == InitSeqs.end()) { + // If there's no init sequence entry yet then we need to look up the + // header symbol to force creation of one. + Lock.unlock(); + + auto SearchOrder = + JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; }); + if (auto Err = ES.lookup(SearchOrder, DSOHandleSymbol).takeError()) + return Err; + + Lock.lock(); + I = InitSeqs.find(&JD); + assert(I != InitSeqs.end() && + "Entry missing after header symbol lookup?"); + } + InitSeq = &I->second; + } + + for (auto *Sec : InitSections) { + // FIXME: Avoid copy here. + jitlink::SectionRange R(*Sec); + InitSeq->InitSections[Sec->getName()].push_back( + {ExecutorAddr(R.getStart()), ExecutorAddr(R.getEnd())}); + } + + return Error::success(); +} + +Error ELFNixPlatform::registerPerObjectSections( + const ELFPerObjectSectionsToRegister &POSR) { + + if (!orc_rt_elfnix_register_object_sections) + return make_error<StringError>("Attempting to register per-object " + "sections, but runtime support has not " + "been loaded yet", + inconvertibleErrorCode()); + + Error ErrResult = Error::success(); + if (auto Err = ES.callSPSWrapper<shared::SPSError( + SPSELFPerObjectSectionsToRegister)>( + orc_rt_elfnix_register_object_sections, ErrResult, POSR)) + return Err; + return ErrResult; +} + +Expected<uint64_t> ELFNixPlatform::createPThreadKey() { + if (!orc_rt_elfnix_create_pthread_key) + return make_error<StringError>( + "Attempting to create pthread key in target, but runtime support has " + "not been loaded yet", + inconvertibleErrorCode()); + + Expected<uint64_t> Result(0); + if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>( + orc_rt_elfnix_create_pthread_key, Result)) + return std::move(Err); + return Result; +} + +void ELFNixPlatform::ELFNixPlatformPlugin::modifyPassConfig( + MaterializationResponsibility &MR, jitlink::LinkGraph &LG, + jitlink::PassConfiguration &Config) { + + // If the initializer symbol is the __dso_handle symbol then just add + // the DSO handle support passes. + if (MR.getInitializerSymbol() == MP.DSOHandleSymbol) { + addDSOHandleSupportPasses(MR, Config); + // The DSOHandle materialization unit doesn't require any other + // support, so we can bail out early. + return; + } + + // If the object contains initializers then add passes to record them. + if (MR.getInitializerSymbol()) + addInitializerSupportPasses(MR, Config); + + // Add passes for eh-frame and TLV support. + addEHAndTLVSupportPasses(MR, Config); +} + +ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap +ELFNixPlatform::ELFNixPlatformPlugin::getSyntheticSymbolDependencies( + MaterializationResponsibility &MR) { + std::lock_guard<std::mutex> Lock(PluginMutex); + auto I = InitSymbolDeps.find(&MR); + if (I != InitSymbolDeps.end()) { + SyntheticSymbolDependenciesMap Result; + Result[MR.getInitializerSymbol()] = std::move(I->second); + InitSymbolDeps.erase(&MR); + return Result; + } + return SyntheticSymbolDependenciesMap(); +} + +void ELFNixPlatform::ELFNixPlatformPlugin::addInitializerSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + /// Preserve init sections. + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { + if (auto Err = preserveInitSections(G, MR)) + return Err; + return Error::success(); + }); + + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerInitSections(G, JD); + }); +} + +void ELFNixPlatform::ELFNixPlatformPlugin::addDSOHandleSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()]( + jitlink::LinkGraph &G) -> Error { + auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { + return Sym->getName() == *MP.DSOHandleSymbol; + }); + assert(I != G.defined_symbols().end() && "Missing DSO handle symbol"); + { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + JITTargetAddress HandleAddr = (*I)->getAddress(); + MP.HandleAddrToJITDylib[HandleAddr] = &JD; + assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists"); + MP.InitSeqs.insert(std::make_pair( + &JD, + ELFNixJITDylibInitializers(JD.getName(), ExecutorAddr(HandleAddr)))); + } + return Error::success(); + }); +} + +void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + // Insert TLV lowering at the start of the PostPrunePasses, since we want + // it to run before GOT/PLT lowering. + + // TODO: Check that before the fixTLVSectionsAndEdges pass, the GOT/PLT build + // pass has done. Because the TLS descriptor need to be allocate in GOT. + Config.PostPrunePasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return fixTLVSectionsAndEdges(G, JD); + }); + + // Add a pass to register the final addresses of the eh-frame and TLV sections + // with the runtime. + Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error { + ELFPerObjectSectionsToRegister POSR; + + if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { + jitlink::SectionRange R(*EHFrameSection); + if (!R.empty()) + POSR.EHFrameSection = {ExecutorAddr(R.getStart()), + ExecutorAddr(R.getEnd())}; + } + + // Get a pointer to the thread data section if there is one. It will be used + // below. + jitlink::Section *ThreadDataSection = + G.findSectionByName(ThreadDataSectionName); + + // Handle thread BSS section if there is one. + if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) { + // If there's already a thread data section in this graph then merge the + // thread BSS section content into it, otherwise just treat the thread + // BSS section as the thread data section. + if (ThreadDataSection) + G.mergeSections(*ThreadDataSection, *ThreadBSSSection); + else + ThreadDataSection = ThreadBSSSection; + } + + // Having merged thread BSS (if present) and thread data (if present), + // record the resulting section range. + if (ThreadDataSection) { + jitlink::SectionRange R(*ThreadDataSection); + if (!R.empty()) + POSR.ThreadDataSection = {ExecutorAddr(R.getStart()), + ExecutorAddr(R.getEnd())}; + } + + if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) { + + // If we're still bootstrapping the runtime then just record this + // frame for now. + if (!MP.RuntimeBootstrapped) { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + MP.BootstrapPOSRs.push_back(POSR); + return Error::success(); + } + + // Otherwise register it immediately. + if (auto Err = MP.registerPerObjectSections(POSR)) + return Err; + } + + return Error::success(); + }); +} + +Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + + JITLinkSymbolSet InitSectionSymbols; + for (auto &InitSectionName : InitSectionNames) { + // Skip non-init sections. + auto *InitSection = G.findSectionByName(InitSectionName); + if (!InitSection) + continue; + + // Make a pass over live symbols in the section: those blocks are already + // preserved. + DenseSet<jitlink::Block *> AlreadyLiveBlocks; + for (auto &Sym : InitSection->symbols()) { + auto &B = Sym->getBlock(); + if (Sym->isLive() && Sym->getOffset() == 0 && + Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) { + InitSectionSymbols.insert(Sym); + AlreadyLiveBlocks.insert(&B); + } + } + + // Add anonymous symbols to preserve any not-already-preserved blocks. + for (auto *B : InitSection->blocks()) + if (!AlreadyLiveBlocks.count(B)) + InitSectionSymbols.insert( + &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true)); + } + + if (!InitSectionSymbols.empty()) { + std::lock_guard<std::mutex> Lock(PluginMutex); + InitSymbolDeps[&MR] = std::move(InitSectionSymbols); + } + + return Error::success(); +} + +Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections( + jitlink::LinkGraph &G, JITDylib &JD) { + + SmallVector<jitlink::Section *> InitSections; + + LLVM_DEBUG({ dbgs() << "ELFNixPlatform::registerInitSections\n"; }); + + for (auto InitSectionName : InitSectionNames) { + if (auto *Sec = G.findSectionByName(InitSectionName)) { + InitSections.push_back(Sec); + } + } + + // Dump the scraped inits. + LLVM_DEBUG({ + dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n"; + for (auto *Sec : InitSections) { + jitlink::SectionRange R(*Sec); + dbgs() << " " << Sec->getName() << ": " + << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n"; + } + }); + + return MP.registerInitInfo(JD, InitSections); +} + +Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges( + jitlink::LinkGraph &G, JITDylib &JD) { + + // TODO implement TLV support + for (auto *Sym : G.external_symbols()) + if (Sym->getName() == "__tls_get_addr") { + Sym->setName("___orc_rt_elfnix_tls_get_addr"); + } + + auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO"); + + if (TLSInfoEntrySection) { + Optional<uint64_t> Key; + { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + auto I = MP.JITDylibToPThreadKey.find(&JD); + if (I != MP.JITDylibToPThreadKey.end()) + Key = I->second; + } + if (!Key) { + if (auto KeyOrErr = MP.createPThreadKey()) + Key = *KeyOrErr; + else + return KeyOrErr.takeError(); + } + + uint64_t PlatformKeyBits = + support::endian::byte_swap(*Key, G.getEndianness()); + + for (auto *B : TLSInfoEntrySection->blocks()) { + // FIXME: The TLS descriptor byte length may different with different + // ISA + assert(B->getSize() == (G.getPointerSize() * 2) && + "TLS descriptor must be 2 words length"); + auto TLSInfoEntryContent = B->getMutableContent(G); + memcpy(TLSInfoEntryContent.data(), &PlatformKeyBits, G.getPointerSize()); + } + } + + return Error::success(); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp index 5715eda71eee..f3fe0555fa75 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp @@ -39,13 +39,13 @@ createJITLoaderGDBRegistrar(ExecutionSession &ES) { assert((*Result)[0].size() == 1 && "Unexpected number of addresses in result"); - return std::make_unique<EPCDebugObjectRegistrar>(ES, (*Result)[0][0]); + return std::make_unique<EPCDebugObjectRegistrar>( + ES, ExecutorAddr((*Result)[0][0])); } -Error EPCDebugObjectRegistrar::registerDebugObject(sys::MemoryBlock TargetMem) { - return ES.callSPSWrapper<void(SPSExecutorAddress, uint64_t)>( - RegisterFn, ExecutorAddress::fromPtr(TargetMem.base()), - static_cast<uint64_t>(TargetMem.allocatedSize())); +Error EPCDebugObjectRegistrar::registerDebugObject( + ExecutorAddrRange TargetMem) { + return ES.callSPSWrapper<void(SPSExecutorAddrRange)>(RegisterFn, TargetMem); } } // namespace orc diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp index 8cdda9ab5a15..4c0fab8aa9fa 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp @@ -51,21 +51,22 @@ EPCEHFrameRegistrar::Create(ExecutionSession &ES) { auto RegisterEHFrameWrapperFnAddr = (*Result)[0][0]; auto DeregisterEHFrameWrapperFnAddr = (*Result)[0][1]; - return std::make_unique<EPCEHFrameRegistrar>(ES, RegisterEHFrameWrapperFnAddr, - DeregisterEHFrameWrapperFnAddr); + return std::make_unique<EPCEHFrameRegistrar>( + ES, ExecutorAddr(RegisterEHFrameWrapperFnAddr), + ExecutorAddr(DeregisterEHFrameWrapperFnAddr)); } Error EPCEHFrameRegistrar::registerEHFrames(JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { - return ES.callSPSWrapper<void(SPSExecutorAddress, uint64_t)>( - RegisterEHFrameWrapperFnAddr, EHFrameSectionAddr, + return ES.callSPSWrapper<void(SPSExecutorAddr, uint64_t)>( + RegisterEHFrameWrapperFnAddr, ExecutorAddr(EHFrameSectionAddr), static_cast<uint64_t>(EHFrameSectionSize)); } Error EPCEHFrameRegistrar::deregisterEHFrames( JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { - return ES.callSPSWrapper<void(SPSExecutorAddress, uint64_t)>( - DeregisterEHFrameWrapperFnAddr, EHFrameSectionAddr, + return ES.callSPSWrapper<void(SPSExecutorAddr, uint64_t)>( + DeregisterEHFrameWrapperFnAddr, ExecutorAddr(EHFrameSectionAddr), static_cast<uint64_t>(EHFrameSectionSize)); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp new file mode 100644 index 000000000000..6c47c5c5f7bb --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp @@ -0,0 +1,107 @@ +//===------- EPCGenericDylibManager.cpp -- Dylib management via EPC -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/EPCGenericDylibManager.h" + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" + +namespace llvm { +namespace orc { +namespace shared { + +template <> +class SPSSerializationTraits<SPSRemoteSymbolLookupSetElement, + SymbolLookupSet::value_type> { +public: + static size_t size(const SymbolLookupSet::value_type &V) { + return SPSArgList<SPSString, bool>::size( + *V.first, V.second == SymbolLookupFlags::RequiredSymbol); + } + + static bool serialize(SPSOutputBuffer &OB, + const SymbolLookupSet::value_type &V) { + return SPSArgList<SPSString, bool>::serialize( + OB, *V.first, V.second == SymbolLookupFlags::RequiredSymbol); + } +}; + +template <> +class TrivialSPSSequenceSerialization<SPSRemoteSymbolLookupSetElement, + SymbolLookupSet> { +public: + static constexpr bool available = true; +}; + +template <> +class SPSSerializationTraits<SPSRemoteSymbolLookup, + ExecutorProcessControl::LookupRequest> { + using MemberSerialization = + SPSArgList<SPSExecutorAddr, SPSRemoteSymbolLookupSet>; + +public: + static size_t size(const ExecutorProcessControl::LookupRequest &LR) { + return MemberSerialization::size(ExecutorAddr(LR.Handle), LR.Symbols); + } + + static bool serialize(SPSOutputBuffer &OB, + const ExecutorProcessControl::LookupRequest &LR) { + return MemberSerialization::serialize(OB, ExecutorAddr(LR.Handle), + LR.Symbols); + } +}; + +} // end namespace shared + +Expected<EPCGenericDylibManager> +EPCGenericDylibManager::CreateWithDefaultBootstrapSymbols( + ExecutorProcessControl &EPC) { + SymbolAddrs SAs; + if (auto Err = EPC.getBootstrapSymbols( + {{SAs.Instance, rt::SimpleExecutorDylibManagerInstanceName}, + {SAs.Open, rt::SimpleExecutorDylibManagerOpenWrapperName}, + {SAs.Lookup, rt::SimpleExecutorDylibManagerLookupWrapperName}})) + return std::move(Err); + return EPCGenericDylibManager(EPC, std::move(SAs)); +} + +Expected<tpctypes::DylibHandle> EPCGenericDylibManager::open(StringRef Path, + uint64_t Mode) { + Expected<tpctypes::DylibHandle> H(0); + if (auto Err = + EPC.callSPSWrapper<rt::SPSSimpleExecutorDylibManagerOpenSignature>( + SAs.Open, H, SAs.Instance, Path, Mode)) + return std::move(Err); + return H; +} + +Expected<std::vector<ExecutorAddr>> +EPCGenericDylibManager::lookup(tpctypes::DylibHandle H, + const SymbolLookupSet &Lookup) { + Expected<std::vector<ExecutorAddr>> Result((std::vector<ExecutorAddr>())); + if (auto Err = + EPC.callSPSWrapper<rt::SPSSimpleExecutorDylibManagerLookupSignature>( + SAs.Lookup, Result, SAs.Instance, H, Lookup)) + return std::move(Err); + return Result; +} + +Expected<std::vector<ExecutorAddr>> +EPCGenericDylibManager::lookup(tpctypes::DylibHandle H, + const RemoteSymbolLookupSet &Lookup) { + Expected<std::vector<ExecutorAddr>> Result((std::vector<ExecutorAddr>())); + if (auto Err = + EPC.callSPSWrapper<rt::SPSSimpleExecutorDylibManagerLookupSignature>( + SAs.Lookup, Result, SAs.Instance, H, Lookup)) + return std::move(Err); + return Result; +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp new file mode 100644 index 000000000000..9b712cb8f7ca --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp @@ -0,0 +1,184 @@ +//===---- EPCGenericJITLinkMemoryManager.cpp -- Mem management via EPC ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h" + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" + +#include <limits> + +using namespace llvm::jitlink; + +namespace llvm { +namespace orc { + +class EPCGenericJITLinkMemoryManager::InFlightAlloc + : public jitlink::JITLinkMemoryManager::InFlightAlloc { +public: + + // FIXME: The C++98 initializer is an attempt to work around compile failures + // due to http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#1397. + // We should be able to switch this back to member initialization once that + // issue is fixed. + struct SegInfo { + SegInfo() : WorkingMem(nullptr), ContentSize(0), ZeroFillSize(0) {} + + char *WorkingMem; + ExecutorAddr Addr; + uint64_t ContentSize; + uint64_t ZeroFillSize; + }; + + using SegInfoMap = AllocGroupSmallMap<SegInfo>; + + InFlightAlloc(EPCGenericJITLinkMemoryManager &Parent, LinkGraph &G, + ExecutorAddr AllocAddr, SegInfoMap Segs) + : Parent(Parent), G(G), AllocAddr(AllocAddr), Segs(std::move(Segs)) {} + + void finalize(OnFinalizedFunction OnFinalize) override { + tpctypes::FinalizeRequest FR; + for (auto &KV : Segs) { + assert(KV.second.ContentSize <= std::numeric_limits<size_t>::max()); + FR.Segments.push_back(tpctypes::SegFinalizeRequest{ + tpctypes::toWireProtectionFlags( + toSysMemoryProtectionFlags(KV.first.getMemProt())), + KV.second.Addr, + alignTo(KV.second.ContentSize + KV.second.ZeroFillSize, + Parent.EPC.getPageSize()), + {KV.second.WorkingMem, static_cast<size_t>(KV.second.ContentSize)}}); + } + + // Transfer allocation actions. + // FIXME: Merge JITLink and ORC SupportFunctionCall and Action list types, + // turn this into a std::swap. + FR.Actions.reserve(G.allocActions().size()); + for (auto &ActPair : G.allocActions()) + FR.Actions.push_back({{ExecutorAddr(ActPair.Finalize.FnAddr), + {ExecutorAddr(ActPair.Finalize.CtxAddr), + ExecutorAddrDiff(ActPair.Finalize.CtxSize)}}, + {ExecutorAddr(ActPair.Dealloc.FnAddr), + {ExecutorAddr(ActPair.Dealloc.CtxAddr), + ExecutorAddrDiff(ActPair.Dealloc.CtxSize)}}}); + G.allocActions().clear(); + + Parent.EPC.callSPSWrapperAsync< + rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>( + Parent.SAs.Finalize, + [OnFinalize = std::move(OnFinalize), AllocAddr = this->AllocAddr]( + Error SerializationErr, Error FinalizeErr) mutable { + // FIXME: Release abandoned alloc. + if (SerializationErr) { + cantFail(std::move(FinalizeErr)); + OnFinalize(std::move(SerializationErr)); + } else if (FinalizeErr) + OnFinalize(std::move(FinalizeErr)); + else + OnFinalize(FinalizedAlloc(AllocAddr.getValue())); + }, + Parent.SAs.Allocator, std::move(FR)); + } + + void abandon(OnAbandonedFunction OnAbandoned) override { + // FIXME: Return memory to pool instead. + Parent.EPC.callSPSWrapperAsync< + rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( + Parent.SAs.Deallocate, + [OnAbandoned = std::move(OnAbandoned)](Error SerializationErr, + Error DeallocateErr) mutable { + if (SerializationErr) { + cantFail(std::move(DeallocateErr)); + OnAbandoned(std::move(SerializationErr)); + } else + OnAbandoned(std::move(DeallocateErr)); + }, + Parent.SAs.Allocator, ArrayRef<ExecutorAddr>(AllocAddr)); + } + +private: + EPCGenericJITLinkMemoryManager &Parent; + LinkGraph &G; + ExecutorAddr AllocAddr; + SegInfoMap Segs; +}; + +void EPCGenericJITLinkMemoryManager::allocate(const JITLinkDylib *JD, + LinkGraph &G, + OnAllocatedFunction OnAllocated) { + BasicLayout BL(G); + + auto Pages = BL.getContiguousPageBasedLayoutSizes(EPC.getPageSize()); + if (!Pages) + return OnAllocated(Pages.takeError()); + + EPC.callSPSWrapperAsync<rt::SPSSimpleExecutorMemoryManagerReserveSignature>( + SAs.Reserve, + [this, BL = std::move(BL), OnAllocated = std::move(OnAllocated)]( + Error SerializationErr, Expected<ExecutorAddr> AllocAddr) mutable { + if (SerializationErr) { + cantFail(AllocAddr.takeError()); + return OnAllocated(std::move(SerializationErr)); + } + if (!AllocAddr) + return OnAllocated(AllocAddr.takeError()); + + completeAllocation(*AllocAddr, std::move(BL), std::move(OnAllocated)); + }, + SAs.Allocator, Pages->total()); +} + +void EPCGenericJITLinkMemoryManager::deallocate( + std::vector<FinalizedAlloc> Allocs, OnDeallocatedFunction OnDeallocated) { + EPC.callSPSWrapperAsync< + rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( + SAs.Deallocate, + [OnDeallocated = std::move(OnDeallocated)](Error SerErr, + Error DeallocErr) mutable { + if (SerErr) { + cantFail(std::move(DeallocErr)); + OnDeallocated(std::move(SerErr)); + } else + OnDeallocated(std::move(DeallocErr)); + }, + SAs.Allocator, Allocs); + for (auto &A : Allocs) + A.release(); +} + +void EPCGenericJITLinkMemoryManager::completeAllocation( + ExecutorAddr AllocAddr, BasicLayout BL, OnAllocatedFunction OnAllocated) { + + InFlightAlloc::SegInfoMap SegInfos; + + ExecutorAddr NextSegAddr = AllocAddr; + for (auto &KV : BL.segments()) { + const auto &AG = KV.first; + auto &Seg = KV.second; + + Seg.Addr = NextSegAddr.getValue(); + KV.second.WorkingMem = BL.getGraph().allocateBuffer(Seg.ContentSize).data(); + NextSegAddr += ExecutorAddrDiff( + alignTo(Seg.ContentSize + Seg.ZeroFillSize, EPC.getPageSize())); + + auto &SegInfo = SegInfos[AG]; + SegInfo.ContentSize = Seg.ContentSize; + SegInfo.ZeroFillSize = Seg.ZeroFillSize; + SegInfo.Addr = ExecutorAddr(Seg.Addr); + SegInfo.WorkingMem = Seg.WorkingMem; + } + + if (auto Err = BL.apply()) + return OnAllocated(std::move(Err)); + + OnAllocated(std::make_unique<InFlightAlloc>(*this, BL.getGraph(), AllocAddr, + std::move(SegInfos))); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp new file mode 100644 index 000000000000..1d98e104a4d7 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp @@ -0,0 +1,317 @@ +//===----- EPCGenericRTDyldMemoryManager.cpp - EPC-bbasde MemMgr -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/Support/Alignment.h" +#include "llvm/Support/FormatVariadic.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<EPCGenericRTDyldMemoryManager>> +EPCGenericRTDyldMemoryManager::CreateWithDefaultBootstrapSymbols( + ExecutorProcessControl &EPC) { + SymbolAddrs SAs; + if (auto Err = EPC.getBootstrapSymbols( + {{SAs.Instance, rt::SimpleExecutorMemoryManagerInstanceName}, + {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName}, + {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName}, + {SAs.Deallocate, + rt::SimpleExecutorMemoryManagerDeallocateWrapperName}, + {SAs.RegisterEHFrame, + rt::RegisterEHFrameSectionCustomDirectWrapperName}, + {SAs.DeregisterEHFrame, + rt::DeregisterEHFrameSectionCustomDirectWrapperName}})) + return std::move(Err); + return std::make_unique<EPCGenericRTDyldMemoryManager>(EPC, std::move(SAs)); +} + +EPCGenericRTDyldMemoryManager::EPCGenericRTDyldMemoryManager( + ExecutorProcessControl &EPC, SymbolAddrs SAs) + : EPC(EPC), SAs(std::move(SAs)) { + LLVM_DEBUG(dbgs() << "Created remote allocator " << (void *)this << "\n"); +} + +EPCGenericRTDyldMemoryManager::~EPCGenericRTDyldMemoryManager() { + LLVM_DEBUG(dbgs() << "Destroyed remote allocator " << (void *)this << "\n"); + if (!ErrMsg.empty()) + errs() << "Destroying with existing errors:\n" << ErrMsg << "\n"; + + Error Err = Error::success(); + if (auto Err2 = EPC.callSPSWrapper< + rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>( + SAs.Reserve, Err, SAs.Instance, FinalizedAllocs)) { + // FIXME: Report errors through EPC once that functionality is available. + logAllUnhandledErrors(std::move(Err2), errs(), ""); + return; + } + + if (Err) + logAllUnhandledErrors(std::move(Err), errs(), ""); +} + +uint8_t *EPCGenericRTDyldMemoryManager::allocateCodeSection( + uintptr_t Size, unsigned Alignment, unsigned SectionID, + StringRef SectionName) { + std::lock_guard<std::mutex> Lock(M); + LLVM_DEBUG({ + dbgs() << "Allocator " << (void *)this << " allocating code section " + << SectionName << ": size = " << formatv("{0:x}", Size) + << " bytes, alignment = " << Alignment << "\n"; + }); + auto &Seg = Unmapped.back().CodeAllocs; + Seg.emplace_back(Size, Alignment); + return reinterpret_cast<uint8_t *>( + alignAddr(Seg.back().Contents.get(), Align(Alignment))); +} + +uint8_t *EPCGenericRTDyldMemoryManager::allocateDataSection( + uintptr_t Size, unsigned Alignment, unsigned SectionID, + StringRef SectionName, bool IsReadOnly) { + std::lock_guard<std::mutex> Lock(M); + LLVM_DEBUG({ + dbgs() << "Allocator " << (void *)this << " allocating " + << (IsReadOnly ? "ro" : "rw") << "-data section " << SectionName + << ": size = " << formatv("{0:x}", Size) << " bytes, alignment " + << Alignment << ")\n"; + }); + + auto &Seg = + IsReadOnly ? Unmapped.back().RODataAllocs : Unmapped.back().RWDataAllocs; + + Seg.emplace_back(Size, Alignment); + return reinterpret_cast<uint8_t *>( + alignAddr(Seg.back().Contents.get(), Align(Alignment))); +} + +void EPCGenericRTDyldMemoryManager::reserveAllocationSpace( + uintptr_t CodeSize, uint32_t CodeAlign, uintptr_t RODataSize, + uint32_t RODataAlign, uintptr_t RWDataSize, uint32_t RWDataAlign) { + + { + std::lock_guard<std::mutex> Lock(M); + // If there's already an error then bail out. + if (!ErrMsg.empty()) + return; + + if (!isPowerOf2_32(CodeAlign) || CodeAlign > EPC.getPageSize()) { + ErrMsg = "Invalid code alignment in reserveAllocationSpace"; + return; + } + if (!isPowerOf2_32(RODataAlign) || RODataAlign > EPC.getPageSize()) { + ErrMsg = "Invalid ro-data alignment in reserveAllocationSpace"; + return; + } + if (!isPowerOf2_32(RWDataAlign) || RWDataAlign > EPC.getPageSize()) { + ErrMsg = "Invalid rw-data alignment in reserveAllocationSpace"; + return; + } + } + + uint64_t TotalSize = 0; + TotalSize += alignTo(CodeSize, EPC.getPageSize()); + TotalSize += alignTo(RODataSize, EPC.getPageSize()); + TotalSize += alignTo(RWDataSize, EPC.getPageSize()); + + LLVM_DEBUG({ + dbgs() << "Allocator " << (void *)this << " reserving " + << formatv("{0:x}", TotalSize) << " bytes.\n"; + }); + + Expected<ExecutorAddr> TargetAllocAddr((ExecutorAddr())); + if (auto Err = EPC.callSPSWrapper< + rt::SPSSimpleExecutorMemoryManagerReserveSignature>( + SAs.Reserve, TargetAllocAddr, SAs.Instance, TotalSize)) { + std::lock_guard<std::mutex> Lock(M); + ErrMsg = toString(std::move(Err)); + return; + } + if (!TargetAllocAddr) { + std::lock_guard<std::mutex> Lock(M); + ErrMsg = toString(TargetAllocAddr.takeError()); + return; + } + + std::lock_guard<std::mutex> Lock(M); + Unmapped.push_back(AllocGroup()); + Unmapped.back().RemoteCode = { + *TargetAllocAddr, ExecutorAddrDiff(alignTo(CodeSize, EPC.getPageSize()))}; + Unmapped.back().RemoteROData = { + Unmapped.back().RemoteCode.End, + ExecutorAddrDiff(alignTo(RODataSize, EPC.getPageSize()))}; + Unmapped.back().RemoteRWData = { + Unmapped.back().RemoteROData.End, + ExecutorAddrDiff(alignTo(RWDataSize, EPC.getPageSize()))}; +} + +bool EPCGenericRTDyldMemoryManager::needsToReserveAllocationSpace() { + return true; +} + +void EPCGenericRTDyldMemoryManager::registerEHFrames(uint8_t *Addr, + uint64_t LoadAddr, + size_t Size) { + LLVM_DEBUG({ + dbgs() << "Allocator " << (void *)this << " added unfinalized eh-frame " + << formatv("[ {0:x} {1:x} ]", LoadAddr, LoadAddr + Size) << "\n"; + }); + std::lock_guard<std::mutex> Lock(M); + // Bail out early if there's already an error. + if (!ErrMsg.empty()) + return; + + ExecutorAddr LA(LoadAddr); + for (auto &Alloc : llvm::reverse(Unfinalized)) { + if (Alloc.RemoteCode.contains(LA) || Alloc.RemoteROData.contains(LA) || + Alloc.RemoteRWData.contains(LA)) { + Alloc.UnfinalizedEHFrames.push_back({LA, Size}); + return; + } + } + ErrMsg = "eh-frame does not lie inside unfinalized alloc"; +} + +void EPCGenericRTDyldMemoryManager::deregisterEHFrames() { + // This is a no-op for us: We've registered a deallocation action for it. +} + +void EPCGenericRTDyldMemoryManager::notifyObjectLoaded( + RuntimeDyld &Dyld, const object::ObjectFile &Obj) { + std::lock_guard<std::mutex> Lock(M); + LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " applied mappings:\n"); + for (auto &ObjAllocs : Unmapped) { + mapAllocsToRemoteAddrs(Dyld, ObjAllocs.CodeAllocs, + ObjAllocs.RemoteCode.Start); + mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RODataAllocs, + ObjAllocs.RemoteROData.Start); + mapAllocsToRemoteAddrs(Dyld, ObjAllocs.RWDataAllocs, + ObjAllocs.RemoteRWData.Start); + Unfinalized.push_back(std::move(ObjAllocs)); + } + Unmapped.clear(); +} + +bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { + LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " finalizing:\n"); + + // If there's an error then bail out here. + std::vector<AllocGroup> Allocs; + { + std::lock_guard<std::mutex> Lock(M); + if (ErrMsg && !this->ErrMsg.empty()) { + *ErrMsg = std::move(this->ErrMsg); + return true; + } + std::swap(Allocs, Unfinalized); + } + + // Loop over unfinalized objects to make finalization requests. + for (auto &ObjAllocs : Allocs) { + + tpctypes::WireProtectionFlags SegProts[3] = { + tpctypes::toWireProtectionFlags( + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC)), + tpctypes::toWireProtectionFlags(sys::Memory::MF_READ), + tpctypes::toWireProtectionFlags( + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE))}; + + ExecutorAddrRange *RemoteAddrs[3] = {&ObjAllocs.RemoteCode, + &ObjAllocs.RemoteROData, + &ObjAllocs.RemoteRWData}; + + std::vector<Alloc> *SegSections[3] = {&ObjAllocs.CodeAllocs, + &ObjAllocs.RODataAllocs, + &ObjAllocs.RWDataAllocs}; + + tpctypes::FinalizeRequest FR; + std::unique_ptr<char[]> AggregateContents[3]; + + for (unsigned I = 0; I != 3; ++I) { + FR.Segments.push_back({}); + auto &Seg = FR.Segments.back(); + Seg.Prot = SegProts[I]; + Seg.Addr = RemoteAddrs[I]->Start; + for (auto &SecAlloc : *SegSections[I]) { + Seg.Size = alignTo(Seg.Size, SecAlloc.Align); + Seg.Size += SecAlloc.Size; + } + AggregateContents[I] = std::make_unique<char[]>(Seg.Size); + size_t SecOffset = 0; + for (auto &SecAlloc : *SegSections[I]) { + SecOffset = alignTo(SecOffset, SecAlloc.Align); + memcpy(&AggregateContents[I][SecOffset], + reinterpret_cast<const char *>( + alignAddr(SecAlloc.Contents.get(), Align(SecAlloc.Align))), + SecAlloc.Size); + SecOffset += SecAlloc.Size; + // FIXME: Can we reset SecAlloc.Content here, now that it's copied into + // the aggregated content? + } + Seg.Content = {AggregateContents[I].get(), SecOffset}; + } + + for (auto &Frame : ObjAllocs.UnfinalizedEHFrames) + FR.Actions.push_back( + {{SAs.RegisterEHFrame, + {ExecutorAddr(Frame.Addr), ExecutorAddrDiff(Frame.Size)}}, + {SAs.DeregisterEHFrame, + {ExecutorAddr(Frame.Addr), ExecutorAddrDiff(Frame.Size)}}}); + + // We'll also need to make an extra allocation for the eh-frame wrapper call + // arguments. + Error FinalizeErr = Error::success(); + if (auto Err = EPC.callSPSWrapper< + rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>( + SAs.Finalize, FinalizeErr, SAs.Instance, std::move(FR))) { + std::lock_guard<std::mutex> Lock(M); + this->ErrMsg = toString(std::move(Err)); + dbgs() << "Serialization error: " << this->ErrMsg << "\n"; + if (ErrMsg) + *ErrMsg = this->ErrMsg; + return true; + } + if (FinalizeErr) { + std::lock_guard<std::mutex> Lock(M); + this->ErrMsg = toString(std::move(FinalizeErr)); + dbgs() << "Finalization error: " << this->ErrMsg << "\n"; + if (ErrMsg) + *ErrMsg = this->ErrMsg; + return true; + } + } + + return false; +} + +void EPCGenericRTDyldMemoryManager::mapAllocsToRemoteAddrs( + RuntimeDyld &Dyld, std::vector<Alloc> &Allocs, ExecutorAddr NextAddr) { + for (auto &Alloc : Allocs) { + NextAddr.setValue(alignTo(NextAddr.getValue(), Alloc.Align)); + LLVM_DEBUG({ + dbgs() << " " << static_cast<void *>(Alloc.Contents.get()) << " -> " + << format("0x%016" PRIx64, NextAddr.getValue()) << "\n"; + }); + Dyld.mapSectionAddress(reinterpret_cast<const void *>(alignAddr( + Alloc.Contents.get(), Align(Alloc.Align))), + NextAddr.getValue()); + Alloc.RemoteAddr = NextAddr; + // Only advance NextAddr if it was non-null to begin with, + // otherwise leave it as null. + if (NextAddr) + NextAddr += ExecutorAddrDiff(Alloc.Size); + } +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp index b9c70b0aeb3c..818b6b52ff83 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp @@ -43,12 +43,12 @@ public: protected: Error grow() override; - using Allocation = jitlink::JITLinkMemoryManager::Allocation; + using FinalizedAlloc = jitlink::JITLinkMemoryManager::FinalizedAlloc; EPCIndirectionUtils &EPCIU; unsigned TrampolineSize = 0; unsigned TrampolinesPerPage = 0; - std::vector<std::unique_ptr<Allocation>> TrampolineBlocks; + std::vector<FinalizedAlloc> TrampolineBlocks; }; class EPCIndirectStubsManager : public IndirectStubsManager, @@ -89,12 +89,19 @@ EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) Error EPCTrampolinePool::deallocatePool() { Error Err = Error::success(); - for (auto &Alloc : TrampolineBlocks) - Err = joinErrors(std::move(Err), Alloc->deallocate()); - return Err; + std::promise<MSVCPError> DeallocResultP; + auto DeallocResultF = DeallocResultP.get_future(); + + EPCIU.getExecutorProcessControl().getMemMgr().deallocate( + std::move(TrampolineBlocks), + [&](Error Err) { DeallocResultP.set_value(std::move(Err)); }); + + return DeallocResultF.get(); } Error EPCTrampolinePool::grow() { + using namespace jitlink; + assert(AvailableTrampolines.empty() && "Grow called with trampolines still available"); @@ -102,34 +109,26 @@ Error EPCTrampolinePool::grow() { assert(ResolverAddress && "Resolver address can not be null"); auto &EPC = EPCIU.getExecutorProcessControl(); - constexpr auto TrampolinePagePermissions = - static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_EXEC); auto PageSize = EPC.getPageSize(); - jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; - Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize), - 0}; - auto Alloc = EPC.getMemMgr().allocate(nullptr, Request); - + auto Alloc = SimpleSegmentAlloc::Create( + EPC.getMemMgr(), nullptr, + {{MemProt::Read | MemProt::Exec, {PageSize, Align(PageSize)}}}); if (!Alloc) return Alloc.takeError(); unsigned NumTrampolines = TrampolinesPerPage; - auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions); - auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions); - - EPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress, - ResolverAddress, NumTrampolines); - - auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions); + auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); + EPCIU.getABISupport().writeTrampolines( + SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines); for (unsigned I = 0; I < NumTrampolines; ++I) - AvailableTrampolines.push_back(TargetAddr + (I * TrampolineSize)); + AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize)); - if (auto Err = (*Alloc)->finalize()) - return Err; + auto FA = Alloc->finalize(); + if (!FA) + return FA.takeError(); - TrampolineBlocks.push_back(std::move(*Alloc)); + TrampolineBlocks.push_back(std::move(*FA)); return Error::success(); } @@ -162,16 +161,18 @@ Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { unsigned ASIdx = 0; std::vector<tpctypes::UInt32Write> PtrUpdates; for (auto &SI : StubInits) - PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, - static_cast<uint32_t>(SI.second.first)}); + PtrUpdates.push_back( + {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), + static_cast<uint32_t>(SI.second.first)}); return MemAccess.writeUInt32s(PtrUpdates); } case 8: { unsigned ASIdx = 0; std::vector<tpctypes::UInt64Write> PtrUpdates; for (auto &SI : StubInits) - PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, - static_cast<uint64_t>(SI.second.first)}); + PtrUpdates.push_back( + {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), + static_cast<uint64_t>(SI.second.first)}); return MemAccess.writeUInt64s(PtrUpdates); } default: @@ -213,11 +214,11 @@ Error EPCIndirectStubsManager::updatePointer(StringRef Name, auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); switch (EPCIU.getABISupport().getPointerSize()) { case 4: { - tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr); + tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); return MemAccess.writeUInt32s(PUpdate); } case 8: { - tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr); + tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); return MemAccess.writeUInt64s(PUpdate); } default: @@ -267,17 +268,17 @@ EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { } Error EPCIndirectionUtils::cleanup() { - Error Err = Error::success(); - for (auto &A : IndirectStubAllocs) - Err = joinErrors(std::move(Err), A->deallocate()); + auto &MemMgr = EPC.getMemMgr(); + auto Err = MemMgr.deallocate(std::move(IndirectStubAllocs)); if (TP) Err = joinErrors(std::move(Err), static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); if (ResolverBlock) - Err = joinErrors(std::move(Err), ResolverBlock->deallocate()); + Err = + joinErrors(std::move(Err), MemMgr.deallocate(std::move(ResolverBlock))); return Err; } @@ -285,29 +286,29 @@ Error EPCIndirectionUtils::cleanup() { Expected<JITTargetAddress> EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, JITTargetAddress ReentryCtxAddr) { + using namespace jitlink; + assert(ABI && "ABI can not be null"); - constexpr auto ResolverBlockPermissions = - static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_EXEC); auto ResolverSize = ABI->getResolverCodeSize(); - jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; - Request[ResolverBlockPermissions] = {EPC.getPageSize(), - static_cast<size_t>(ResolverSize), 0}; - auto Alloc = EPC.getMemMgr().allocate(nullptr, Request); + auto Alloc = + SimpleSegmentAlloc::Create(EPC.getMemMgr(), nullptr, + {{MemProt::Read | MemProt::Exec, + {ResolverSize, Align(EPC.getPageSize())}}}); + if (!Alloc) return Alloc.takeError(); - auto WorkingMemory = (*Alloc)->getWorkingMemory(ResolverBlockPermissions); - ResolverBlockAddr = (*Alloc)->getTargetMemory(ResolverBlockPermissions); - ABI->writeResolverCode(WorkingMemory.data(), ResolverBlockAddr, ReentryFnAddr, + auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); + ABI->writeResolverCode(SegInfo.WorkingMem.data(), SegInfo.Addr, ReentryFnAddr, ReentryCtxAddr); - if (auto Err = (*Alloc)->finalize()) - return std::move(Err); + auto FA = Alloc->finalize(); + if (!FA) + return FA.takeError(); - ResolverBlock = std::move(*Alloc); - return ResolverBlockAddr; + ResolverBlock = std::move(*FA); + return SegInfo.Addr; } std::unique_ptr<IndirectStubsManager> @@ -341,6 +342,7 @@ EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, Expected<EPCIndirectionUtils::IndirectStubInfoVector> EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { + using namespace jitlink; std::lock_guard<std::mutex> Lock(EPCUIMutex); @@ -350,42 +352,40 @@ EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { auto PageSize = EPC.getPageSize(); auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); NumStubsToAllocate = StubBytes / ABI->getStubSize(); - auto PointerBytes = + auto PtrBytes = alignTo(NumStubsToAllocate * ABI->getPointerSize(), PageSize); - constexpr auto StubPagePermissions = - static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_EXEC); - constexpr auto PointerPagePermissions = - static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_WRITE); - - jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; - Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes), - 0}; - Request[PointerPagePermissions] = {PageSize, 0, PointerBytes}; - auto Alloc = EPC.getMemMgr().allocate(nullptr, Request); + auto StubProt = MemProt::Read | MemProt::Exec; + auto PtrProt = MemProt::Read | MemProt::Write; + + auto Alloc = SimpleSegmentAlloc::Create( + EPC.getMemMgr(), nullptr, + {{StubProt, {static_cast<size_t>(StubBytes), Align(PageSize)}}, + {PtrProt, {static_cast<size_t>(PtrBytes), Align(PageSize)}}}); + if (!Alloc) return Alloc.takeError(); - auto StubTargetAddr = (*Alloc)->getTargetMemory(StubPagePermissions); - auto PointerTargetAddr = (*Alloc)->getTargetMemory(PointerPagePermissions); + auto StubSeg = Alloc->getSegInfo(StubProt); + auto PtrSeg = Alloc->getSegInfo(PtrProt); + + ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr, + PtrSeg.Addr, NumStubsToAllocate); - ABI->writeIndirectStubsBlock( - (*Alloc)->getWorkingMemory(StubPagePermissions).data(), StubTargetAddr, - PointerTargetAddr, NumStubsToAllocate); + auto FA = Alloc->finalize(); + if (!FA) + return FA.takeError(); - if (auto Err = (*Alloc)->finalize()) - return std::move(Err); + IndirectStubAllocs.push_back(std::move(*FA)); + auto StubExecutorAddr = StubSeg.Addr; + auto PtrExecutorAddr = PtrSeg.Addr; for (unsigned I = 0; I != NumStubsToAllocate; ++I) { AvailableIndirectStubs.push_back( - IndirectStubInfo(StubTargetAddr, PointerTargetAddr)); - StubTargetAddr += ABI->getStubSize(); - PointerTargetAddr += ABI->getPointerSize(); + IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr)); + StubExecutorAddr += ABI->getStubSize(); + PtrExecutorAddr += ABI->getPointerSize(); } - - IndirectStubAllocs.push_back(std::move(*Alloc)); } assert(NumStubs <= AvailableIndirectStubs.size() && diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 7a76a6ccc122..2ab9ed4f856b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -12,9 +12,9 @@ #include "llvm/IR/Function.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Module.h" +#include "llvm/MC/TargetRegistry.h" #include "llvm/Object/MachOUniversal.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/TargetRegistry.h" #include "llvm/Target/TargetMachine.h" #include <string> diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index 7d86d125d1db..2eb835551adb 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -24,20 +24,22 @@ ExecutorProcessControl::MemoryAccess::~MemoryAccess() {} ExecutorProcessControl::~ExecutorProcessControl() {} SelfExecutorProcessControl::SelfExecutorProcessControl( - std::shared_ptr<SymbolStringPool> SSP, Triple TargetTriple, - unsigned PageSize, std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) - : ExecutorProcessControl(std::move(SSP)) { + std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D, + Triple TargetTriple, unsigned PageSize, + std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) + : ExecutorProcessControl(std::move(SSP), std::move(D)) { OwnedMemMgr = std::move(MemMgr); if (!OwnedMemMgr) - OwnedMemMgr = std::make_unique<jitlink::InProcessMemoryManager>(); + OwnedMemMgr = std::make_unique<jitlink::InProcessMemoryManager>( + sys::Process::getPageSizeEstimate()); this->TargetTriple = std::move(TargetTriple); this->PageSize = PageSize; this->MemMgr = OwnedMemMgr.get(); this->MemAccess = this; - this->JDI = {ExecutorAddress::fromPtr(jitDispatchViaWrapperFunctionManager), - ExecutorAddress::fromPtr(this)}; + this->JDI = {ExecutorAddr::fromPtr(jitDispatchViaWrapperFunctionManager), + ExecutorAddr::fromPtr(this)}; if (this->TargetTriple.isOSBinFormatMachO()) GlobalManglingPrefix = '_'; } @@ -45,11 +47,20 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( Expected<std::unique_ptr<SelfExecutorProcessControl>> SelfExecutorProcessControl::Create( std::shared_ptr<SymbolStringPool> SSP, + std::unique_ptr<TaskDispatcher> D, std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) { if (!SSP) SSP = std::make_shared<SymbolStringPool>(); + if (!D) { +#if LLVM_ENABLE_THREADS + D = std::make_unique<DynamicThreadPoolTaskDispatcher>(); +#else + D = std::make_unique<InPlaceTaskDispatcher>(); +#endif + } + auto PageSize = sys::Process::getPageSize(); if (!PageSize) return PageSize.takeError(); @@ -57,7 +68,8 @@ SelfExecutorProcessControl::Create( Triple TT(sys::getProcessTriple()); return std::make_unique<SelfExecutorProcessControl>( - std::move(SSP), std::move(TT), *PageSize, std::move(MemMgr)); + std::move(SSP), std::move(D), std::move(TT), *PageSize, + std::move(MemMgr)); } Expected<tpctypes::DylibHandle> @@ -93,7 +105,7 @@ SelfExecutorProcessControl::lookupSymbols(ArrayRef<LookupRequest> Request) { // FIXME: Collect all failing symbols before erroring out. SymbolNameVector MissingSymbols; MissingSymbols.push_back(Sym); - return make_error<SymbolsNotFound>(std::move(MissingSymbols)); + return make_error<SymbolsNotFound>(SSP, std::move(MissingSymbols)); } R.back().push_back(pointerToJITTargetAddress(Addr)); } @@ -103,60 +115,62 @@ SelfExecutorProcessControl::lookupSymbols(ArrayRef<LookupRequest> Request) { } Expected<int32_t> -SelfExecutorProcessControl::runAsMain(JITTargetAddress MainFnAddr, +SelfExecutorProcessControl::runAsMain(ExecutorAddr MainFnAddr, ArrayRef<std::string> Args) { using MainTy = int (*)(int, char *[]); - return orc::runAsMain(jitTargetAddressToFunction<MainTy>(MainFnAddr), Args); + return orc::runAsMain(MainFnAddr.toPtr<MainTy>(), Args); } -void SelfExecutorProcessControl::callWrapperAsync( - SendResultFunction SendResult, JITTargetAddress WrapperFnAddr, - ArrayRef<char> ArgBuffer) { +void SelfExecutorProcessControl::callWrapperAsync(ExecutorAddr WrapperFnAddr, + IncomingWFRHandler SendResult, + ArrayRef<char> ArgBuffer) { using WrapperFnTy = - shared::detail::CWrapperFunctionResult (*)(const char *Data, size_t Size); - auto *WrapperFn = jitTargetAddressToFunction<WrapperFnTy>(WrapperFnAddr); + shared::CWrapperFunctionResult (*)(const char *Data, size_t Size); + auto *WrapperFn = WrapperFnAddr.toPtr<WrapperFnTy>(); SendResult(WrapperFn(ArgBuffer.data(), ArgBuffer.size())); } -Error SelfExecutorProcessControl::disconnect() { return Error::success(); } +Error SelfExecutorProcessControl::disconnect() { + D->shutdown(); + return Error::success(); +} -void SelfExecutorProcessControl::writeUInt8s(ArrayRef<tpctypes::UInt8Write> Ws, - WriteResultFn OnWriteComplete) { +void SelfExecutorProcessControl::writeUInt8sAsync( + ArrayRef<tpctypes::UInt8Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) - *jitTargetAddressToPointer<uint8_t *>(W.Address) = W.Value; + *W.Addr.toPtr<uint8_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeUInt16s( +void SelfExecutorProcessControl::writeUInt16sAsync( ArrayRef<tpctypes::UInt16Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) - *jitTargetAddressToPointer<uint16_t *>(W.Address) = W.Value; + *W.Addr.toPtr<uint16_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeUInt32s( +void SelfExecutorProcessControl::writeUInt32sAsync( ArrayRef<tpctypes::UInt32Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) - *jitTargetAddressToPointer<uint32_t *>(W.Address) = W.Value; + *W.Addr.toPtr<uint32_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeUInt64s( +void SelfExecutorProcessControl::writeUInt64sAsync( ArrayRef<tpctypes::UInt64Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) - *jitTargetAddressToPointer<uint64_t *>(W.Address) = W.Value; + *W.Addr.toPtr<uint64_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeBuffers( +void SelfExecutorProcessControl::writeBuffersAsync( ArrayRef<tpctypes::BufferWrite> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) - memcpy(jitTargetAddressToPointer<char *>(W.Address), W.Buffer.data(), - W.Buffer.size()); + memcpy(W.Addr.toPtr<char *>(), W.Buffer.data(), W.Buffer.size()); OnWriteComplete(Error::success()); } -shared::detail::CWrapperFunctionResult +shared::CWrapperFunctionResult SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager( void *Ctx, const void *FnTag, const char *Data, size_t Size) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index e8dd1bb90c9a..ee1630a2ffa8 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -9,12 +9,17 @@ #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/MC/MCDisassembler/MCDisassembler.h" +#include "llvm/MC/MCInstrAnalysis.h" #include "llvm/Support/Format.h" #include "llvm/Transforms/Utils/Cloning.h" #include <sstream> +#define DEBUG_TYPE "orc" + using namespace llvm; using namespace llvm::orc; @@ -372,5 +377,77 @@ void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, Dst.addModuleFlag(MapMetadata(MF, VMap)); } +Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, + jitlink::LinkGraph &G, + MCDisassembler &Disassembler, + MCInstrAnalysis &MIA) { + // AArch64 appears to already come with the necessary relocations. Among other + // architectures, only x86_64 is currently implemented here. + if (G.getTargetTriple().getArch() != Triple::x86_64) + return Error::success(); + + raw_null_ostream CommentStream; + auto &STI = Disassembler.getSubtargetInfo(); + + // Determine the function bounds + auto &B = Sym.getBlock(); + assert(!B.isZeroFill() && "expected content block"); + auto SymAddress = Sym.getAddress(); + auto SymStartInBlock = + (const uint8_t *)B.getContent().data() + Sym.getOffset(); + auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset(); + auto Content = makeArrayRef(SymStartInBlock, SymSize); + + LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n"); + + SmallDenseSet<uintptr_t, 8> ExistingRelocations; + for (auto &E : B.edges()) { + if (E.isRelocation()) + ExistingRelocations.insert(E.getOffset()); + } + + size_t I = 0; + while (I < Content.size()) { + MCInst Instr; + uint64_t InstrSize = 0; + uint64_t InstrStart = SymAddress + I; + auto DecodeStatus = Disassembler.getInstruction( + Instr, InstrSize, Content.drop_front(I), InstrStart, CommentStream); + if (DecodeStatus != MCDisassembler::Success) { + LLVM_DEBUG(dbgs() << "Aborting due to disassembly failure at address " + << InstrStart); + return make_error<StringError>( + formatv("failed to disassemble at address {0:x16}", InstrStart), + inconvertibleErrorCode()); + } + // Advance to the next instruction. + I += InstrSize; + + // Check for a PC-relative address equal to the symbol itself. + auto PCRelAddr = + MIA.evaluateMemoryOperandAddress(Instr, &STI, InstrStart, InstrSize); + if (!PCRelAddr.hasValue() || PCRelAddr.getValue() != SymAddress) + continue; + + auto RelocOffInInstr = + MIA.getMemoryOperandRelocationOffset(Instr, InstrSize); + if (!RelocOffInInstr.hasValue() || + InstrSize - RelocOffInInstr.getValue() != 4) { + LLVM_DEBUG(dbgs() << "Skipping unknown self-relocation at " + << InstrStart); + continue; + } + + auto RelocOffInBlock = + InstrStart + *RelocOffInInstr - SymAddress + Sym.getOffset(); + if (ExistingRelocations.contains(RelocOffInBlock)) + continue; + + LLVM_DEBUG(dbgs() << "Adding delta32 self-relocation at " << InstrStart); + B.addEdge(jitlink::x86_64::Delta32, RelocOffInBlock, Sym, /*Addend=*/-4); + } + return Error::success(); +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index 4257137a2212..0fbf79b8a56d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -8,8 +8,8 @@ #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" +#include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Host.h" -#include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" namespace llvm { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index 2ac32293e4db..0ab0d7d2e2b6 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -105,16 +105,18 @@ private: /// llvm.global_ctors. class GlobalCtorDtorScraper { public: - GlobalCtorDtorScraper(GenericLLVMIRPlatformSupport &PS, - StringRef InitFunctionPrefix) - : PS(PS), InitFunctionPrefix(InitFunctionPrefix) {} + StringRef InitFunctionPrefix, + StringRef DeInitFunctionPrefix) + : PS(PS), InitFunctionPrefix(InitFunctionPrefix), + DeInitFunctionPrefix(DeInitFunctionPrefix) {} Expected<ThreadSafeModule> operator()(ThreadSafeModule TSM, MaterializationResponsibility &R); private: GenericLLVMIRPlatformSupport &PS; StringRef InitFunctionPrefix; + StringRef DeInitFunctionPrefix; }; /// Generic IR Platform Support @@ -125,12 +127,14 @@ private: class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { public: GenericLLVMIRPlatformSupport(LLJIT &J) - : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")) { + : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")), + DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) { getExecutionSession().setPlatform( std::make_unique<GenericLLVMIRPlatform>(*this)); - setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix)); + setInitTransform(J, GlobalCtorDtorScraper(*this, InitFunctionPrefix, + DeInitFunctionPrefix)); SymbolMap StdInterposes; @@ -203,6 +207,8 @@ public: InitSymbols[&JD].add(KV.first, SymbolLookupFlags::WeaklyReferencedSymbol); InitFunctions[&JD].add(KV.first); + } else if ((*KV.first).startswith(DeInitFunctionPrefix)) { + DeInitFunctions[&JD].add(KV.first); } } return Error::success(); @@ -256,6 +262,11 @@ public: }); } + void registerDeInitFunc(JITDylib &JD, SymbolStringPtr DeInitName) { + getExecutionSession().runSessionLocked( + [&]() { DeInitFunctions[&JD].add(DeInitName); }); + } + private: Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) { @@ -438,6 +449,7 @@ private: LLJIT &J; std::string InitFunctionPrefix; + std::string DeInitFunctionPrefix; DenseMap<JITDylib *, SymbolLookupSet> InitSymbols; DenseMap<JITDylib *, SymbolLookupSet> InitFunctions; DenseMap<JITDylib *, SymbolLookupSet> DeInitFunctions; @@ -459,40 +471,63 @@ GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM, auto Err = TSM.withModuleDo([&](Module &M) -> Error { auto &Ctx = M.getContext(); auto *GlobalCtors = M.getNamedGlobal("llvm.global_ctors"); - - // If there's no llvm.global_ctors or it's just a decl then skip. - if (!GlobalCtors || GlobalCtors->isDeclaration()) + auto *GlobalDtors = M.getNamedGlobal("llvm.global_dtors"); + + auto RegisterCOrDtors = [&](GlobalVariable *GlobalCOrDtors, + bool isCtor) -> Error { + // If there's no llvm.global_c/dtor or it's just a decl then skip. + if (!GlobalCOrDtors || GlobalCOrDtors->isDeclaration()) + return Error::success(); + std::string InitOrDeInitFunctionName; + if (isCtor) + raw_string_ostream(InitOrDeInitFunctionName) + << InitFunctionPrefix << M.getModuleIdentifier(); + else + raw_string_ostream(InitOrDeInitFunctionName) + << DeInitFunctionPrefix << M.getModuleIdentifier(); + + MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout()); + auto InternedInitOrDeInitName = Mangle(InitOrDeInitFunctionName); + if (auto Err = R.defineMaterializing( + {{InternedInitOrDeInitName, JITSymbolFlags::Callable}})) + return Err; + + auto *InitOrDeInitFunc = Function::Create( + FunctionType::get(Type::getVoidTy(Ctx), {}, false), + GlobalValue::ExternalLinkage, InitOrDeInitFunctionName, &M); + InitOrDeInitFunc->setVisibility(GlobalValue::HiddenVisibility); + std::vector<std::pair<Function *, unsigned>> InitsOrDeInits; + auto COrDtors = isCtor ? getConstructors(M) : getDestructors(M); + + for (auto E : COrDtors) + InitsOrDeInits.push_back(std::make_pair(E.Func, E.Priority)); + llvm::sort(InitsOrDeInits, + [](const std::pair<Function *, unsigned> &LHS, + const std::pair<Function *, unsigned> &RHS) { + return LHS.first < RHS.first; + }); + + auto *InitOrDeInitFuncEntryBlock = + BasicBlock::Create(Ctx, "entry", InitOrDeInitFunc); + IRBuilder<> IB(InitOrDeInitFuncEntryBlock); + for (auto &KV : InitsOrDeInits) + IB.CreateCall(KV.first); + IB.CreateRetVoid(); + + if (isCtor) + PS.registerInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName); + else + PS.registerDeInitFunc(R.getTargetJITDylib(), InternedInitOrDeInitName); + + GlobalCOrDtors->eraseFromParent(); return Error::success(); + }; - std::string InitFunctionName; - raw_string_ostream(InitFunctionName) - << InitFunctionPrefix << M.getModuleIdentifier(); - - MangleAndInterner Mangle(PS.getExecutionSession(), M.getDataLayout()); - auto InternedName = Mangle(InitFunctionName); - if (auto Err = - R.defineMaterializing({{InternedName, JITSymbolFlags::Callable}})) + if (auto Err = RegisterCOrDtors(GlobalCtors, true)) + return Err; + if (auto Err = RegisterCOrDtors(GlobalDtors, false)) return Err; - auto *InitFunc = - Function::Create(FunctionType::get(Type::getVoidTy(Ctx), {}, false), - GlobalValue::ExternalLinkage, InitFunctionName, &M); - InitFunc->setVisibility(GlobalValue::HiddenVisibility); - std::vector<std::pair<Function *, unsigned>> Inits; - for (auto E : getConstructors(M)) - Inits.push_back(std::make_pair(E.Func, E.Priority)); - llvm::sort(Inits, [](const std::pair<Function *, unsigned> &LHS, - const std::pair<Function *, unsigned> &RHS) { - return LHS.first < RHS.first; - }); - auto *EntryBlock = BasicBlock::Create(Ctx, "entry", InitFunc); - IRBuilder<> IB(EntryBlock); - for (auto &KV : Inits) - IB.CreateCall(KV.first); - IB.CreateRetVoid(); - - PS.registerInitFunc(R.getTargetJITDylib(), InternedName); - GlobalCtors->eraseFromParent(); return Error::success(); }); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp new file mode 100644 index 000000000000..44cb78c773c9 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp @@ -0,0 +1,82 @@ +//===------- LookupAndRecordAddrs.h - Symbol lookup support utility -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" + +#include <future> + +namespace llvm { +namespace orc { + +void lookupAndRecordAddrs( + unique_function<void(Error)> OnRecorded, ExecutionSession &ES, LookupKind K, + const JITDylibSearchOrder &SearchOrder, + std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> Pairs, + SymbolLookupFlags LookupFlags) { + + SymbolLookupSet Symbols; + for (auto &KV : Pairs) + Symbols.add(KV.first, LookupFlags); + + ES.lookup( + K, SearchOrder, Symbols, SymbolState::Ready, + [Pairs = std::move(Pairs), + OnRec = std::move(OnRecorded)](Expected<SymbolMap> Result) mutable { + if (!Result) + return OnRec(Result.takeError()); + for (auto &KV : Pairs) { + auto I = Result->find(KV.first); + KV.second->setValue((I != Result->end()) ? I->second.getAddress() + : 0); + } + OnRec(Error::success()); + }, + NoDependenciesToRegister); +} + +Error lookupAndRecordAddrs( + ExecutionSession &ES, LookupKind K, const JITDylibSearchOrder &SearchOrder, + std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> Pairs, + SymbolLookupFlags LookupFlags) { + + std::promise<MSVCPError> ResultP; + auto ResultF = ResultP.get_future(); + lookupAndRecordAddrs([&](Error Err) { ResultP.set_value(std::move(Err)); }, + ES, K, SearchOrder, Pairs, LookupFlags); + return ResultF.get(); +} + +Error lookupAndRecordAddrs( + ExecutorProcessControl &EPC, tpctypes::DylibHandle H, + std::vector<std::pair<SymbolStringPtr, ExecutorAddr *>> Pairs, + SymbolLookupFlags LookupFlags) { + + SymbolLookupSet Symbols; + for (auto &KV : Pairs) + Symbols.add(KV.first, LookupFlags); + + ExecutorProcessControl::LookupRequest LR(H, Symbols); + auto Result = EPC.lookupSymbols(LR); + if (!Result) + return Result.takeError(); + + if (Result->size() != 1) + return make_error<StringError>("Error in lookup result", + inconvertibleErrorCode()); + if (Result->front().size() != Pairs.size()) + return make_error<StringError>("Error in lookup result elements", + inconvertibleErrorCode()); + + for (unsigned I = 0; I != Pairs.size(); ++I) + Pairs[I].second->setValue(Result->front()[I]); + + return Error::success(); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 66ef835dc34d..46c915dfea9e 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -12,6 +12,7 @@ #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" @@ -52,7 +53,7 @@ public: auto G = std::make_unique<jitlink::LinkGraph>( "<MachOHeaderMU>", TT, PointerSize, Endianness, jitlink::getGenericEdgeKindName); - auto &HeaderSection = G->createSection("__header", sys::Memory::MF_READ); + auto &HeaderSection = G->createSection("__header", jitlink::MemProt::Read); auto &HeaderBlock = createHeaderBlock(*G, HeaderSection); // Init symbol is header-start symbol. @@ -135,13 +136,14 @@ StringRef ObjCImageInfoSectionName = "__DATA,__objc_image_info"; StringRef ObjCSelRefsSectionName = "__DATA,__objc_selrefs"; StringRef Swift5ProtoSectionName = "__TEXT,__swift5_proto"; StringRef Swift5ProtosSectionName = "__TEXT,__swift5_protos"; +StringRef Swift5TypesSectionName = "__TEXT,__swift5_types"; StringRef ThreadBSSSectionName = "__DATA,__thread_bss"; StringRef ThreadDataSectionName = "__DATA,__thread_data"; StringRef ThreadVarsSectionName = "__DATA,__thread_vars"; StringRef InitSectionNames[] = { ModInitFuncSectionName, ObjCSelRefsSectionName, ObjCClassListSectionName, - Swift5ProtosSectionName, Swift5ProtoSectionName}; + Swift5ProtosSectionName, Swift5ProtoSectionName, Swift5TypesSectionName}; } // end anonymous namespace @@ -172,10 +174,10 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, // Add JIT-dispatch function support symbols. if (auto Err = PlatformJD.define(absoluteSymbols( {{ES.intern("___orc_rt_jit_dispatch"), - {EPC.getJITDispatchInfo().JITDispatchFunctionAddress.getValue(), + {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), JITSymbolFlags::Exported}}, {ES.intern("___orc_rt_jit_dispatch_ctx"), - {EPC.getJITDispatchInfo().JITDispatchContextAddress.getValue(), + {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), JITSymbolFlags::Exported}}}))) return std::move(Err); @@ -267,6 +269,7 @@ bool MachOPlatform::isInitializerSection(StringRef SegName, bool MachOPlatform::supportedTarget(const Triple &TT) { switch (TT.getArch()) { + case Triple::aarch64: case Triple::x86_64: return true; default: @@ -286,6 +289,19 @@ MachOPlatform::MachOPlatform( PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); + // Force linking of eh-frame registration functions. + if (auto Err2 = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), + {{ES.intern("___orc_rt_macho_register_ehframe_section"), + &orc_rt_macho_register_ehframe_section}, + {ES.intern("___orc_rt_macho_deregister_ehframe_section"), + &orc_rt_macho_deregister_ehframe_section}})) { + Err = std::move(Err2); + return; + } + + State = BootstrapPhase2; + // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating // the platform now), so set it up. if (auto E2 = setupJITDylib(PlatformJD)) { @@ -309,6 +325,8 @@ MachOPlatform::MachOPlatform( Err = std::move(E2); return; } + + State = Initialized; } Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { @@ -321,13 +339,13 @@ Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { this, &MachOPlatform::rt_getInitializers); using GetDeinitializersSPSSig = - SPSExpected<SPSMachOJITDylibDeinitializerSequence>(SPSExecutorAddress); + SPSExpected<SPSMachOJITDylibDeinitializerSequence>(SPSExecutorAddr); WFs[ES.intern("___orc_rt_macho_get_deinitializers_tag")] = ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>( this, &MachOPlatform::rt_getDeinitializers); using LookupSymbolSPSSig = - SPSExpected<SPSExecutorAddress>(SPSExecutorAddress, SPSString); + SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString); WFs[ES.intern("___orc_rt_macho_symbol_lookup_tag")] = ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this, &MachOPlatform::rt_lookupSymbol); @@ -411,7 +429,7 @@ void MachOPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, } void MachOPlatform::rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, - ExecutorAddress Handle) { + ExecutorAddr Handle) { LLVM_DEBUG({ dbgs() << "MachOPlatform::rt_getDeinitializers(\"" << formatv("{0:x}", Handle.getValue()) << "\")\n"; @@ -441,8 +459,7 @@ void MachOPlatform::rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, } void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, - ExecutorAddress Handle, - StringRef SymbolName) { + ExecutorAddr Handle, StringRef SymbolName) { LLVM_DEBUG({ dbgs() << "MachOPlatform::rt_lookupSymbol(\"" << formatv("{0:x}", Handle.getValue()) << "\")\n"; @@ -476,7 +493,7 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, void operator()(Expected<SymbolMap> Result) { if (Result) { assert(Result->size() == 1 && "Unexpected result map count"); - SendResult(ExecutorAddress(Result->begin()->second.getAddress())); + SendResult(ExecutorAddr(Result->begin()->second.getAddress())); } else { SendResult(Result.takeError()); } @@ -495,56 +512,25 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, } Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) { - - std::pair<const char *, ExecutorAddress *> Symbols[] = { - {"___orc_rt_macho_platform_bootstrap", &orc_rt_macho_platform_bootstrap}, - {"___orc_rt_macho_platform_shutdown", &orc_rt_macho_platform_shutdown}, - {"___orc_rt_macho_register_object_sections", - &orc_rt_macho_register_object_sections}, - {"___orc_rt_macho_create_pthread_key", &orc_rt_macho_create_pthread_key}}; - - SymbolLookupSet RuntimeSymbols; - std::vector<std::pair<SymbolStringPtr, ExecutorAddress *>> AddrsToRecord; - for (const auto &KV : Symbols) { - auto Name = ES.intern(KV.first); - RuntimeSymbols.add(Name); - AddrsToRecord.push_back({std::move(Name), KV.second}); - } - - auto RuntimeSymbolAddrs = ES.lookup( - {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols); - if (!RuntimeSymbolAddrs) - return RuntimeSymbolAddrs.takeError(); - - for (const auto &KV : AddrsToRecord) { - auto &Name = KV.first; - assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?"); - KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress()); - } - - if (auto Err = - ES.callSPSWrapper<void()>(orc_rt_macho_platform_bootstrap.getValue())) + if (auto Err = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), + {{ES.intern("___orc_rt_macho_platform_bootstrap"), + &orc_rt_macho_platform_bootstrap}, + {ES.intern("___orc_rt_macho_platform_shutdown"), + &orc_rt_macho_platform_shutdown}, + {ES.intern("___orc_rt_macho_register_thread_data_section"), + &orc_rt_macho_register_thread_data_section}, + {ES.intern("___orc_rt_macho_deregister_thread_data_section"), + &orc_rt_macho_deregister_thread_data_section}, + {ES.intern("___orc_rt_macho_create_pthread_key"), + &orc_rt_macho_create_pthread_key}})) return Err; - // FIXME: Ordering is fuzzy here. We're probably best off saying - // "behavior is undefined if code that uses the runtime is added before - // the platform constructor returns", then move all this to the constructor. - RuntimeBootstrapped = true; - std::vector<MachOPerObjectSectionsToRegister> DeferredPOSRs; - { - std::lock_guard<std::mutex> Lock(PlatformMutex); - DeferredPOSRs = std::move(BootstrapPOSRs); - } - - for (auto &D : DeferredPOSRs) - if (auto Err = registerPerObjectSections(D)) - return Err; - - return Error::success(); + return ES.callSPSWrapper<void()>(orc_rt_macho_platform_bootstrap); } Error MachOPlatform::registerInitInfo( - JITDylib &JD, ExecutorAddress ObjCImageInfoAddr, + JITDylib &JD, ExecutorAddr ObjCImageInfoAddr, ArrayRef<jitlink::Section *> InitSections) { std::unique_lock<std::mutex> Lock(PlatformMutex); @@ -576,29 +562,12 @@ Error MachOPlatform::registerInitInfo( // FIXME: Avoid copy here. jitlink::SectionRange R(*Sec); InitSeq->InitSections[Sec->getName()].push_back( - {ExecutorAddress(R.getStart()), ExecutorAddress(R.getEnd())}); + {ExecutorAddr(R.getStart()), ExecutorAddr(R.getEnd())}); } return Error::success(); } -Error MachOPlatform::registerPerObjectSections( - const MachOPerObjectSectionsToRegister &POSR) { - - if (!orc_rt_macho_register_object_sections) - return make_error<StringError>("Attempting to register per-object " - "sections, but runtime support has not " - "been loaded yet", - inconvertibleErrorCode()); - - Error ErrResult = Error::success(); - if (auto Err = ES.callSPSWrapper<shared::SPSError( - SPSMachOPerObjectSectionsToRegister)>( - orc_rt_macho_register_object_sections.getValue(), ErrResult, POSR)) - return Err; - return ErrResult; -} - Expected<uint64_t> MachOPlatform::createPThreadKey() { if (!orc_rt_macho_create_pthread_key) return make_error<StringError>( @@ -608,7 +577,7 @@ Expected<uint64_t> MachOPlatform::createPThreadKey() { Expected<uint64_t> Result(0); if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>( - orc_rt_macho_create_pthread_key.getValue(), Result)) + orc_rt_macho_create_pthread_key, Result)) return std::move(Err); return Result; } @@ -617,21 +586,55 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( MaterializationResponsibility &MR, jitlink::LinkGraph &LG, jitlink::PassConfiguration &Config) { - // If the initializer symbol is the MachOHeader start symbol then just add - // the macho header support passes. - if (MR.getInitializerSymbol() == MP.MachOHeaderStartSymbol) { - addMachOHeaderSupportPasses(MR, Config); - // The header materialization unit doesn't require any other support, so we - // can bail out early. + auto PS = MP.State.load(); + + // --- Handle Initializers --- + if (auto InitSymbol = MR.getInitializerSymbol()) { + + // If the initializer symbol is the MachOHeader start symbol then just + // register it and then bail out -- the header materialization unit + // definitely doesn't need any other passes. + if (InitSymbol == MP.MachOHeaderStartSymbol) { + Config.PostAllocationPasses.push_back([this, &MR](jitlink::LinkGraph &G) { + return associateJITDylibHeaderSymbol(G, MR); + }); + return; + } + + // If the object contains an init symbol other than the header start symbol + // then add passes to preserve, process and register the init + // sections/symbols. + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { + if (auto Err = preserveInitSections(G, MR)) + return Err; + return processObjCImageInfo(G, MR); + }); + + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerInitSections(G, JD); + }); + } + + // --- Add passes for eh-frame and TLV support --- + if (PS == MachOPlatform::BootstrapPhase1) { + Config.PostFixupPasses.push_back( + [this](jitlink::LinkGraph &G) { return registerEHSectionsPhase1(G); }); return; } - // If the object contains initializers then add passes to record them. - if (MR.getInitializerSymbol()) - addInitializerSupportPasses(MR, Config); + // Insert TLV lowering at the start of the PostPrunePasses, since we want + // it to run before GOT/PLT lowering. + Config.PostPrunePasses.insert( + Config.PostPrunePasses.begin(), + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return fixTLVSectionsAndEdges(G, JD); + }); - // Add passes for eh-frame and TLV support. - addEHAndTLVSupportPasses(MR, Config); + // Add a pass to register the final addresses of the eh-frame and TLV sections + // with the runtime. + Config.PostFixupPasses.push_back( + [this](jitlink::LinkGraph &G) { return registerEHAndTLVSections(G); }); } ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap @@ -648,111 +651,22 @@ MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies( return SyntheticSymbolDependenciesMap(); } -void MachOPlatform::MachOPlatformPlugin::addInitializerSupportPasses( - MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { - - /// Preserve init sections. - Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { - if (auto Err = preserveInitSections(G, MR)) - return Err; - return processObjCImageInfo(G, MR); - }); - - Config.PostFixupPasses.push_back( - [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { - return registerInitSections(G, JD); - }); -} - -void MachOPlatform::MachOPlatformPlugin::addMachOHeaderSupportPasses( - MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { - - Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()]( - jitlink::LinkGraph &G) -> Error { - auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { - return Sym->getName() == *MP.MachOHeaderStartSymbol; - }); - assert(I != G.defined_symbols().end() && - "Missing MachO header start symbol"); - { - std::lock_guard<std::mutex> Lock(MP.PlatformMutex); - JITTargetAddress HeaderAddr = (*I)->getAddress(); - MP.HeaderAddrToJITDylib[HeaderAddr] = &JD; - assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists"); - MP.InitSeqs.insert( - std::make_pair(&JD, MachOJITDylibInitializers( - JD.getName(), ExecutorAddress(HeaderAddr)))); - } - return Error::success(); - }); -} - -void MachOPlatform::MachOPlatformPlugin::addEHAndTLVSupportPasses( - MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { - - // Insert TLV lowering at the start of the PostPrunePasses, since we want - // it to run before GOT/PLT lowering. - Config.PostPrunePasses.insert( - Config.PostPrunePasses.begin(), - [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { - return fixTLVSectionsAndEdges(G, JD); - }); - - // Add a pass to register the final addresses of the eh-frame and TLV sections - // with the runtime. - Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error { - MachOPerObjectSectionsToRegister POSR; - - if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { - jitlink::SectionRange R(*EHFrameSection); - if (!R.empty()) - POSR.EHFrameSection = {ExecutorAddress(R.getStart()), - ExecutorAddress(R.getEnd())}; - } - - // Get a pointer to the thread data section if there is one. It will be used - // below. - jitlink::Section *ThreadDataSection = - G.findSectionByName(ThreadDataSectionName); - - // Handle thread BSS section if there is one. - if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) { - // If there's already a thread data section in this graph then merge the - // thread BSS section content into it, otherwise just treat the thread - // BSS section as the thread data section. - if (ThreadDataSection) - G.mergeSections(*ThreadDataSection, *ThreadBSSSection); - else - ThreadDataSection = ThreadBSSSection; - } - - // Having merged thread BSS (if present) and thread data (if present), - // record the resulting section range. - if (ThreadDataSection) { - jitlink::SectionRange R(*ThreadDataSection); - if (!R.empty()) - POSR.ThreadDataSection = {ExecutorAddress(R.getStart()), - ExecutorAddress(R.getEnd())}; - } - - if (POSR.EHFrameSection.StartAddress || - POSR.ThreadDataSection.StartAddress) { - - // If we're still bootstrapping the runtime then just record this - // frame for now. - if (!MP.RuntimeBootstrapped) { - std::lock_guard<std::mutex> Lock(MP.PlatformMutex); - MP.BootstrapPOSRs.push_back(POSR); - return Error::success(); - } - - // Otherwise register it immediately. - if (auto Err = MP.registerPerObjectSections(POSR)) - return Err; - } +Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { - return Error::success(); + auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { + return Sym->getName() == *MP.MachOHeaderStartSymbol; }); + assert(I != G.defined_symbols().end() && "Missing MachO header start symbol"); + + auto &JD = MR.getTargetJITDylib(); + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + JITTargetAddress HeaderAddr = (*I)->getAddress(); + MP.HeaderAddrToJITDylib[HeaderAddr] = &JD; + assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists"); + MP.InitSeqs.insert(std::make_pair( + &JD, MachOJITDylibInitializers(JD.getName(), ExecutorAddr(HeaderAddr)))); + return Error::success(); } Error MachOPlatform::MachOPlatformPlugin::preserveInitSections( @@ -873,7 +787,7 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( Error MachOPlatform::MachOPlatformPlugin::registerInitSections( jitlink::LinkGraph &G, JITDylib &JD) { - ExecutorAddress ObjCImageInfoAddr; + ExecutorAddr ObjCImageInfoAddr; SmallVector<jitlink::Section *> InitSections; if (auto *ObjCImageInfoSec = G.findSectionByName(ObjCImageInfoSectionName)) { @@ -950,9 +864,109 @@ Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( for (auto *B : G.blocks()) for (auto &E : B->edges()) if (E.getKind() == - jitlink::x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable) - E.setKind( - jitlink::x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable); + jitlink::x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadREXRelaxable) + E.setKind(jitlink::x86_64:: + RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable); + + return Error::success(); +} + +Error MachOPlatform::MachOPlatformPlugin::registerEHAndTLVSections( + jitlink::LinkGraph &G) { + + // Add a pass to register the final addresses of the eh-frame and TLV sections + // with the runtime. + if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { + jitlink::SectionRange R(*EHFrameSection); + if (!R.empty()) + G.allocActions().push_back( + {{MP.orc_rt_macho_register_ehframe_section.getValue(), R.getStart(), + R.getSize()}, + {MP.orc_rt_macho_deregister_ehframe_section.getValue(), R.getStart(), + R.getSize()}}); + } + + // Get a pointer to the thread data section if there is one. It will be used + // below. + jitlink::Section *ThreadDataSection = + G.findSectionByName(ThreadDataSectionName); + + // Handle thread BSS section if there is one. + if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) { + // If there's already a thread data section in this graph then merge the + // thread BSS section content into it, otherwise just treat the thread + // BSS section as the thread data section. + if (ThreadDataSection) + G.mergeSections(*ThreadDataSection, *ThreadBSSSection); + else + ThreadDataSection = ThreadBSSSection; + } + + // Having merged thread BSS (if present) and thread data (if present), + // record the resulting section range. + if (ThreadDataSection) { + jitlink::SectionRange R(*ThreadDataSection); + if (!R.empty()) { + if (MP.State != MachOPlatform::Initialized) + return make_error<StringError>("__thread_data section encountered, but " + "MachOPlatform has not finished booting", + inconvertibleErrorCode()); + + G.allocActions().push_back( + {{MP.orc_rt_macho_register_thread_data_section.getValue(), + R.getStart(), R.getSize()}, + {MP.orc_rt_macho_deregister_thread_data_section.getValue(), + R.getStart(), R.getSize()}}); + } + } + return Error::success(); +} + +Error MachOPlatform::MachOPlatformPlugin::registerEHSectionsPhase1( + jitlink::LinkGraph &G) { + + // If there's no eh-frame there's nothing to do. + auto *EHFrameSection = G.findSectionByName(EHFrameSectionName); + if (!EHFrameSection) + return Error::success(); + + // If the eh-frame section is empty there's nothing to do. + jitlink::SectionRange R(*EHFrameSection); + if (R.empty()) + return Error::success(); + + // Since we're linking the object containing the registration code now the + // addresses won't be ready in the platform. We'll have to find them in this + // graph instead. + ExecutorAddr orc_rt_macho_register_ehframe_section; + ExecutorAddr orc_rt_macho_deregister_ehframe_section; + for (auto *Sym : G.defined_symbols()) { + if (!Sym->hasName()) + continue; + if (Sym->getName() == "___orc_rt_macho_register_ehframe_section") + orc_rt_macho_register_ehframe_section = ExecutorAddr(Sym->getAddress()); + else if (Sym->getName() == "___orc_rt_macho_deregister_ehframe_section") + orc_rt_macho_deregister_ehframe_section = ExecutorAddr(Sym->getAddress()); + + if (orc_rt_macho_register_ehframe_section && + orc_rt_macho_deregister_ehframe_section) + break; + } + + // If we failed to find the required functions then bail out. + if (!orc_rt_macho_register_ehframe_section || + !orc_rt_macho_deregister_ehframe_section) + return make_error<StringError>("Could not find eh-frame registration " + "functions during platform bootstrap", + inconvertibleErrorCode()); + + // Otherwise, add allocation actions to the graph to register eh-frames for + // this object. + G.allocActions().push_back( + {{orc_rt_macho_register_ehframe_section.getValue(), R.getStart(), + R.getSize()}, + {orc_rt_macho_deregister_ehframe_section.getValue(), R.getStart(), + R.getSize()}}); return Error::success(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Mangling.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Mangling.cpp index 14b22880ab7e..7b21e6a684ca 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Mangling.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Mangling.cpp @@ -7,9 +7,11 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Mangling.h" +#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Mangler.h" +#include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/MachO.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" @@ -83,17 +85,29 @@ void IRSymbolMapper::add(ExecutionSession &ES, const ManglingOptions &MO, } } -Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>> -getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { - auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); +static SymbolStringPtr addInitSymbol(SymbolFlagsMap &SymbolFlags, + ExecutionSession &ES, + StringRef ObjFileName) { + SymbolStringPtr InitSymbol; + size_t Counter = 0; - if (!Obj) - return Obj.takeError(); + do { + std::string InitSymString; + raw_string_ostream(InitSymString) + << "$." << ObjFileName << ".__inits." << Counter++; + InitSymbol = ES.intern(InitSymString); + } while (SymbolFlags.count(InitSymbol)); - bool IsMachO = isa<object::MachOObjectFile>(Obj->get()); + SymbolFlags[InitSymbol] = JITSymbolFlags::MaterializationSideEffectsOnly; + return InitSymbol; +} +static Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>> +getMachOObjectFileSymbolInfo(ExecutionSession &ES, + const object::MachOObjectFile &Obj) { SymbolFlagsMap SymbolFlags; - for (auto &Sym : (*Obj)->symbols()) { + + for (auto &Sym : Obj.symbols()) { Expected<uint32_t> SymFlagsOrErr = Sym.getFlags(); if (!SymFlagsOrErr) // TODO: Test this error. @@ -123,48 +137,135 @@ getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { return SymFlags.takeError(); // Strip the 'exported' flag from MachO linker-private symbols. - if (IsMachO && Name->startswith("l")) + if (Name->startswith("l")) *SymFlags &= ~JITSymbolFlags::Exported; SymbolFlags[InternedName] = std::move(*SymFlags); } SymbolStringPtr InitSymbol; + for (auto &Sec : Obj.sections()) { + auto SecType = Obj.getSectionType(Sec); + if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) { + InitSymbol = addInitSymbol(SymbolFlags, ES, Obj.getFileName()); + break; + } + auto SegName = Obj.getSectionFinalSegmentName(Sec.getRawDataRefImpl()); + auto SecName = cantFail(Obj.getSectionName(Sec.getRawDataRefImpl())); + if (MachOPlatform::isInitializerSection(SegName, SecName)) { + InitSymbol = addInitSymbol(SymbolFlags, ES, Obj.getFileName()); + break; + } + } - size_t Counter = 0; - auto AddInitSymbol = [&]() { - while (true) { - std::string InitSymString; - raw_string_ostream(InitSymString) - << "$." << ObjBuffer.getBufferIdentifier() << ".__inits." - << Counter++; - InitSymbol = ES.intern(InitSymString); - if (SymbolFlags.count(InitSymbol)) + return std::make_pair(std::move(SymbolFlags), std::move(InitSymbol)); +} + +static Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>> +getELFObjectFileSymbolInfo(ExecutionSession &ES, + const object::ELFObjectFileBase &Obj) { + SymbolFlagsMap SymbolFlags; + for (auto &Sym : Obj.symbols()) { + Expected<uint32_t> SymFlagsOrErr = Sym.getFlags(); + if (!SymFlagsOrErr) + // TODO: Test this error. + return SymFlagsOrErr.takeError(); + + // Skip symbols not defined in this object file. + if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined) + continue; + + // Skip symbols that are not global. + if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global)) + continue; + + // Skip symbols that have type SF_File. + if (auto SymType = Sym.getType()) { + if (*SymType == object::SymbolRef::ST_File) continue; - SymbolFlags[InitSymbol] = JITSymbolFlags::MaterializationSideEffectsOnly; - return; - } - }; - - if (IsMachO) { - auto &MachOObj = cast<object::MachOObjectFile>(*Obj->get()); - for (auto &Sec : MachOObj.sections()) { - auto SecType = MachOObj.getSectionType(Sec); - if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) { - AddInitSymbol(); - break; - } - auto SegName = - MachOObj.getSectionFinalSegmentName(Sec.getRawDataRefImpl()); - auto SecName = cantFail(MachOObj.getSectionName(Sec.getRawDataRefImpl())); - if (MachOPlatform::isInitializerSection(SegName, SecName)) { - AddInitSymbol(); + } else + return SymType.takeError(); + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto InternedName = ES.intern(*Name); + auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); + if (!SymFlags) + return SymFlags.takeError(); + + // ELF STB_GNU_UNIQUE should map to Weak for ORC. + if (Sym.getBinding() == ELF::STB_GNU_UNIQUE) + *SymFlags |= JITSymbolFlags::Weak; + + SymbolFlags[InternedName] = std::move(*SymFlags); + } + + SymbolStringPtr InitSymbol; + for (auto &Sec : Obj.sections()) { + if (auto SecName = Sec.getName()) { + if (ELFNixPlatform::isInitializerSection(*SecName)) { + InitSymbol = addInitSymbol(SymbolFlags, ES, Obj.getFileName()); break; } } } - return std::make_pair(std::move(SymbolFlags), std::move(InitSymbol)); + return std::make_pair(std::move(SymbolFlags), InitSymbol); +} + +Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>> +getGenericObjectFileSymbolInfo(ExecutionSession &ES, + const object::ObjectFile &Obj) { + SymbolFlagsMap SymbolFlags; + for (auto &Sym : Obj.symbols()) { + Expected<uint32_t> SymFlagsOrErr = Sym.getFlags(); + if (!SymFlagsOrErr) + // TODO: Test this error. + return SymFlagsOrErr.takeError(); + + // Skip symbols not defined in this object file. + if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined) + continue; + + // Skip symbols that are not global. + if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global)) + continue; + + // Skip symbols that have type SF_File. + if (auto SymType = Sym.getType()) { + if (*SymType == object::SymbolRef::ST_File) + continue; + } else + return SymType.takeError(); + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto InternedName = ES.intern(*Name); + auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); + if (!SymFlags) + return SymFlags.takeError(); + + SymbolFlags[InternedName] = std::move(*SymFlags); + } + + return std::make_pair(std::move(SymbolFlags), nullptr); +} + +Expected<std::pair<SymbolFlagsMap, SymbolStringPtr>> +getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer); + + if (!Obj) + return Obj.takeError(); + + if (auto *MachOObj = dyn_cast<object::MachOObjectFile>(Obj->get())) + return getMachOObjectFileSymbolInfo(ES, *MachOObj); + else if (auto *ELFObj = dyn_cast<object::ELFObjectFileBase>(Obj->get())) + return getELFObjectFileSymbolInfo(ES, *ELFObj); + + return getGenericObjectFileSymbolInfo(ES, **Obj); } } // End namespace orc. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index fd260089c04b..6f840a079dd1 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -64,9 +64,9 @@ private: LGI.SymbolFlags[ES.intern(Sym->getName())] = Flags; } - if (G.getTargetTriple().isOSBinFormatMachO()) - if (hasMachOInitSection(G)) - LGI.InitSymbol = makeInitSymbol(ES, G); + if ((G.getTargetTriple().isOSBinFormatMachO() && hasMachOInitSection(G)) || + (G.getTargetTriple().isOSBinFormatELF() && hasELFInitSection(G))) + LGI.InitSymbol = makeInitSymbol(ES, G); return LGI; } @@ -77,11 +77,19 @@ private: Sec.getName() == "__DATA,__objc_classlist" || Sec.getName() == "__TEXT,__swift5_protos" || Sec.getName() == "__TEXT,__swift5_proto" || + Sec.getName() == "__TEXT,__swift5_types" || Sec.getName() == "__DATA,__mod_init_func") return true; return false; } + static bool hasELFInitSection(LinkGraph &G) { + for (auto &Sec : G.sections()) + if (Sec.getName() == ".init_array") + return true; + return false; + } + static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) { std::string InitSymString; raw_string_ostream(InitSymString) @@ -272,8 +280,9 @@ public: // If there were missing symbols then report the error. if (!MissingSymbols.empty()) - return make_error<MissingSymbolDefinitions>(G.getName(), - std::move(MissingSymbols)); + return make_error<MissingSymbolDefinitions>( + Layer.getExecutionSession().getSymbolStringPool(), G.getName(), + std::move(MissingSymbols)); // If there are more definitions than expected, add them to the // ExtraSymbols vector. @@ -286,8 +295,9 @@ public: // If there were extra definitions then report the error. if (!ExtraSymbols.empty()) - return make_error<UnexpectedSymbolDefinitions>(G.getName(), - std::move(ExtraSymbols)); + return make_error<UnexpectedSymbolDefinitions>( + Layer.getExecutionSession().getSymbolStringPool(), G.getName(), + std::move(ExtraSymbols)); } if (auto Err = MR->notifyResolved(InternedResult)) @@ -297,8 +307,7 @@ public: return Error::success(); } - void notifyFinalized( - std::unique_ptr<JITLinkMemoryManager::Allocation> A) override { + void notifyFinalized(JITLinkMemoryManager::FinalizedAlloc A) override { if (auto Err = Layer.notifyEmitted(*MR, std::move(A))) { Layer.getExecutionSession().reportError(std::move(Err)); MR->failMaterialization(); @@ -414,7 +423,8 @@ private: std::vector<std::pair<SymbolStringPtr, Symbol *>> NameToSym; auto ProcessSymbol = [&](Symbol *Sym) { - if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) { + if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak && + Sym->getScope() != Scope::Local) { auto Name = ES.intern(Sym->getName()); if (!MR->getSymbols().count(ES.intern(Sym->getName()))) { JITSymbolFlags SF = JITSymbolFlags::Weak; @@ -543,8 +553,7 @@ private: // Propagate block-level dependencies through the block-dependence graph. while (!WorkList.empty()) { - auto *B = WorkList.back(); - WorkList.pop_back(); + auto *B = WorkList.pop_back_val(); auto &BI = BlockInfos[B]; assert(BI.DependenciesChanged && @@ -672,7 +681,7 @@ void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { } Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, - AllocPtr Alloc) { + FinalizedAlloc FA) { Error Err = Error::success(); for (auto &P : Plugins) Err = joinErrors(std::move(Err), P->notifyEmitted(MR)); @@ -681,17 +690,20 @@ Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, return Err; return MR.withResourceKeyDo( - [&](ResourceKey K) { Allocs[K].push_back(std::move(Alloc)); }); + [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); }); } Error ObjectLinkingLayer::handleRemoveResources(ResourceKey K) { - Error Err = Error::success(); - - for (auto &P : Plugins) - Err = joinErrors(std::move(Err), P->notifyRemovingResources(K)); + { + Error Err = Error::success(); + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingResources(K)); + if (Err) + return Err; + } - std::vector<AllocPtr> AllocsToRemove; + std::vector<FinalizedAlloc> AllocsToRemove; getExecutionSession().runSessionLocked([&] { auto I = Allocs.find(K); if (I != Allocs.end()) { @@ -700,12 +712,10 @@ Error ObjectLinkingLayer::handleRemoveResources(ResourceKey K) { } }); - while (!AllocsToRemove.empty()) { - Err = joinErrors(std::move(Err), AllocsToRemove.back()->deallocate()); - AllocsToRemove.pop_back(); - } + if (AllocsToRemove.empty()) + return Error::success(); - return Err; + return MemMgr.deallocate(std::move(AllocsToRemove)); } void ObjectLinkingLayer::handleTransferResources(ResourceKey DstKey, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp index d6f73a8b0864..673f7394450f 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -619,6 +619,61 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( return LLVMErrorSuccess; } +LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForPath( + LLVMOrcDefinitionGeneratorRef *Result, const char *FileName, + char GlobalPrefix, LLVMOrcSymbolPredicate Filter, void *FilterCtx) { + assert(Result && "Result can not be null"); + assert(FileName && "FileName can not be null"); + assert((Filter || !FilterCtx) && + "if Filter is null then FilterCtx must also be null"); + + DynamicLibrarySearchGenerator::SymbolPredicate Pred; + if (Filter) + Pred = [=](const SymbolStringPtr &Name) -> bool { + return Filter(FilterCtx, wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); + }; + + auto LibrarySymsGenerator = + DynamicLibrarySearchGenerator::Load(FileName, GlobalPrefix, Pred); + + if (!LibrarySymsGenerator) { + *Result = 0; + return wrap(LibrarySymsGenerator.takeError()); + } + + *Result = wrap(LibrarySymsGenerator->release()); + return LLVMErrorSuccess; +} + +LLVMErrorRef LLVMOrcCreateStaticLibrarySearchGeneratorForPath( + LLVMOrcDefinitionGeneratorRef *Result, LLVMOrcObjectLayerRef ObjLayer, + const char *FileName, const char *TargetTriple) { + assert(Result && "Result can not be null"); + assert(FileName && "Filename can not be null"); + assert(ObjLayer && "ObjectLayer can not be null"); + + if (TargetTriple) { + auto TT = Triple(TargetTriple); + auto LibrarySymsGenerator = + StaticLibraryDefinitionGenerator::Load(*unwrap(ObjLayer), FileName, TT); + if (!LibrarySymsGenerator) { + *Result = 0; + return wrap(LibrarySymsGenerator.takeError()); + } + *Result = wrap(LibrarySymsGenerator->release()); + return LLVMErrorSuccess; + } else { + auto LibrarySymsGenerator = + StaticLibraryDefinitionGenerator::Load(*unwrap(ObjLayer), FileName); + if (!LibrarySymsGenerator) { + *Result = 0; + return wrap(LibrarySymsGenerator.takeError()); + } + *Result = wrap(LibrarySymsGenerator->release()); + return LLVMErrorSuccess; + } +} + LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void) { return wrap(new ThreadSafeContext(std::make_unique<LLVMContext>())); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp new file mode 100644 index 000000000000..02044e4af29a --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -0,0 +1,47 @@ +//===------ OrcRTBridge.cpp - Executor functions for bootstrap -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" + +namespace llvm { +namespace orc { +namespace rt { + +const char *SimpleExecutorDylibManagerInstanceName = + "__llvm_orc_SimpleExecutorDylibManager_Instance"; +const char *SimpleExecutorDylibManagerOpenWrapperName = + "__llvm_orc_SimpleExecutorDylibManager_open_wrapper"; +const char *SimpleExecutorDylibManagerLookupWrapperName = + "__llvm_orc_SimpleExecutorDylibManager_lookup_wrapper"; +const char *SimpleExecutorMemoryManagerInstanceName = + "__llvm_orc_SimpleExecutorMemoryManager_Instance"; +const char *SimpleExecutorMemoryManagerReserveWrapperName = + "__llvm_orc_SimpleExecutorMemoryManager_reserve_wrapper"; +const char *SimpleExecutorMemoryManagerFinalizeWrapperName = + "__llvm_orc_SimpleExecutorMemoryManager_finalize_wrapper"; +const char *SimpleExecutorMemoryManagerDeallocateWrapperName = + "__llvm_orc_SimpleExecutorMemoryManager_deallocate_wrapper"; +const char *MemoryWriteUInt8sWrapperName = + "__llvm_orc_bootstrap_mem_write_uint8s_wrapper"; +const char *MemoryWriteUInt16sWrapperName = + "__llvm_orc_bootstrap_mem_write_uint16s_wrapper"; +const char *MemoryWriteUInt32sWrapperName = + "__llvm_orc_bootstrap_mem_write_uint32s_wrapper"; +const char *MemoryWriteUInt64sWrapperName = + "__llvm_orc_bootstrap_mem_write_uint64s_wrapper"; +const char *MemoryWriteBuffersWrapperName = + "__llvm_orc_bootstrap_mem_write_buffers_wrapper"; +const char *RegisterEHFrameSectionCustomDirectWrapperName = + "__llvm_orc_bootstrap_register_ehframe_section_custom_direct_wrapper"; +const char *DeregisterEHFrameSectionCustomDirectWrapperName = + "__llvm_orc_bootstrap_deregister_ehframe_section_custom_direct_wrapper"; +const char *RunAsMainWrapperName = "__llvm_orc_bootstrap_run_as_main_wrapper"; + +} // end namespace rt +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/RPCError.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/RPCError.cpp deleted file mode 100644 index a55cb220f218..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/RPCError.cpp +++ /dev/null @@ -1,58 +0,0 @@ -//===--------------- RPCError.cpp - RPCERror implementation ---------------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// RPC Error type implmentations. -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/Orc/Shared/RPCUtils.h" -#include "llvm/Support/Error.h" -#include "llvm/Support/raw_ostream.h" - -#include <string> -#include <system_error> - -char llvm::orc::shared::RPCFatalError::ID = 0; -char llvm::orc::shared::ConnectionClosed::ID = 0; -char llvm::orc::shared::ResponseAbandoned::ID = 0; -char llvm::orc::shared::CouldNotNegotiate::ID = 0; - -namespace llvm { -namespace orc { -namespace shared { - -std::error_code ConnectionClosed::convertToErrorCode() const { - return orcError(OrcErrorCode::RPCConnectionClosed); -} - -void ConnectionClosed::log(raw_ostream &OS) const { - OS << "RPC connection already closed"; -} - -std::error_code ResponseAbandoned::convertToErrorCode() const { - return orcError(OrcErrorCode::RPCResponseAbandoned); -} - -void ResponseAbandoned::log(raw_ostream &OS) const { - OS << "RPC response abandoned"; -} - -CouldNotNegotiate::CouldNotNegotiate(std::string Signature) - : Signature(std::move(Signature)) {} - -std::error_code CouldNotNegotiate::convertToErrorCode() const { - return orcError(OrcErrorCode::RPCCouldNotNegotiateFunction); -} - -void CouldNotNegotiate::log(raw_ostream &OS) const { - OS << "Could not negotiate RPC function " << Signature; -} - -} // end namespace shared -} // end namespace orc -} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.cpp new file mode 100644 index 000000000000..64fc717b7b56 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.cpp @@ -0,0 +1,250 @@ +//===------ SimpleRemoteEPCUtils.cpp - Utils for Simple Remote EPC --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Message definitions and other utilities for SimpleRemoteEPC and +// SimpleRemoteEPCServer. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/FormatVariadic.h" + +#if !defined(_MSC_VER) && !defined(__MINGW32__) +#include <unistd.h> +#else +#include <io.h> +#endif + +namespace { + +struct FDMsgHeader { + static constexpr unsigned MsgSizeOffset = 0; + static constexpr unsigned OpCOffset = MsgSizeOffset + sizeof(uint64_t); + static constexpr unsigned SeqNoOffset = OpCOffset + sizeof(uint64_t); + static constexpr unsigned TagAddrOffset = SeqNoOffset + sizeof(uint64_t); + static constexpr unsigned Size = TagAddrOffset + sizeof(uint64_t); +}; + +} // namespace + +namespace llvm { +namespace orc { +namespace SimpleRemoteEPCDefaultBootstrapSymbolNames { + +const char *ExecutorSessionObjectName = + "__llvm_orc_SimpleRemoteEPC_dispatch_ctx"; +const char *DispatchFnName = "__llvm_orc_SimpleRemoteEPC_dispatch_fn"; + +} // end namespace SimpleRemoteEPCDefaultBootstrapSymbolNames + +SimpleRemoteEPCTransportClient::~SimpleRemoteEPCTransportClient() {} +SimpleRemoteEPCTransport::~SimpleRemoteEPCTransport() {} + +Expected<std::unique_ptr<FDSimpleRemoteEPCTransport>> +FDSimpleRemoteEPCTransport::Create(SimpleRemoteEPCTransportClient &C, int InFD, + int OutFD) { +#if LLVM_ENABLE_THREADS + if (InFD == -1) + return make_error<StringError>("Invalid input file descriptor " + + Twine(InFD), + inconvertibleErrorCode()); + if (OutFD == -1) + return make_error<StringError>("Invalid output file descriptor " + + Twine(OutFD), + inconvertibleErrorCode()); + std::unique_ptr<FDSimpleRemoteEPCTransport> FDT( + new FDSimpleRemoteEPCTransport(C, InFD, OutFD)); + return std::move(FDT); +#else + return make_error<StringError>("FD-based SimpleRemoteEPC transport requires " + "thread support, but llvm was built with " + "LLVM_ENABLE_THREADS=Off", + inconvertibleErrorCode()); +#endif +} + +FDSimpleRemoteEPCTransport::~FDSimpleRemoteEPCTransport() { +#if LLVM_ENABLE_THREADS + ListenerThread.join(); +#endif +} + +Error FDSimpleRemoteEPCTransport::start() { +#if LLVM_ENABLE_THREADS + ListenerThread = std::thread([this]() { listenLoop(); }); + return Error::success(); +#endif + llvm_unreachable("Should not be called with LLVM_ENABLE_THREADS=Off"); +} + +Error FDSimpleRemoteEPCTransport::sendMessage(SimpleRemoteEPCOpcode OpC, + uint64_t SeqNo, + ExecutorAddr TagAddr, + ArrayRef<char> ArgBytes) { + char HeaderBuffer[FDMsgHeader::Size]; + + *((support::ulittle64_t *)(HeaderBuffer + FDMsgHeader::MsgSizeOffset)) = + FDMsgHeader::Size + ArgBytes.size(); + *((support::ulittle64_t *)(HeaderBuffer + FDMsgHeader::OpCOffset)) = + static_cast<uint64_t>(OpC); + *((support::ulittle64_t *)(HeaderBuffer + FDMsgHeader::SeqNoOffset)) = SeqNo; + *((support::ulittle64_t *)(HeaderBuffer + FDMsgHeader::TagAddrOffset)) = + TagAddr.getValue(); + + std::lock_guard<std::mutex> Lock(M); + if (Disconnected) + return make_error<StringError>("FD-transport disconnected", + inconvertibleErrorCode()); + if (int ErrNo = writeBytes(HeaderBuffer, FDMsgHeader::Size)) + return errorCodeToError(std::error_code(ErrNo, std::generic_category())); + if (int ErrNo = writeBytes(ArgBytes.data(), ArgBytes.size())) + return errorCodeToError(std::error_code(ErrNo, std::generic_category())); + return Error::success(); +} + +void FDSimpleRemoteEPCTransport::disconnect() { + if (Disconnected) + return; // Return if already disconnected. + + Disconnected = true; + bool CloseOutFD = InFD != OutFD; + + // Close InFD. + while (close(InFD) == -1) { + if (errno == EBADF) + break; + } + + // Close OutFD. + if (CloseOutFD) { + while (close(OutFD) == -1) { + if (errno == EBADF) + break; + } + } +} + +static Error makeUnexpectedEOFError() { + return make_error<StringError>("Unexpected end-of-file", + inconvertibleErrorCode()); +} + +Error FDSimpleRemoteEPCTransport::readBytes(char *Dst, size_t Size, + bool *IsEOF) { + assert(Dst && "Attempt to read into null."); + ssize_t Completed = 0; + while (Completed < static_cast<ssize_t>(Size)) { + ssize_t Read = ::read(InFD, Dst + Completed, Size - Completed); + if (Read <= 0) { + auto ErrNo = errno; + if (Read == 0) { + if (Completed == 0 && IsEOF) { + *IsEOF = true; + return Error::success(); + } else + return makeUnexpectedEOFError(); + } else if (ErrNo == EAGAIN || ErrNo == EINTR) + continue; + else { + std::lock_guard<std::mutex> Lock(M); + if (Disconnected && IsEOF) { // disconnect called, pretend this is EOF. + *IsEOF = true; + return Error::success(); + } + return errorCodeToError( + std::error_code(ErrNo, std::generic_category())); + } + } + Completed += Read; + } + return Error::success(); +} + +int FDSimpleRemoteEPCTransport::writeBytes(const char *Src, size_t Size) { + assert(Src && "Attempt to append from null."); + ssize_t Completed = 0; + while (Completed < static_cast<ssize_t>(Size)) { + ssize_t Written = ::write(OutFD, Src + Completed, Size - Completed); + if (Written < 0) { + auto ErrNo = errno; + if (ErrNo == EAGAIN || ErrNo == EINTR) + continue; + else + return ErrNo; + } + Completed += Written; + } + return 0; +} + +void FDSimpleRemoteEPCTransport::listenLoop() { + Error Err = Error::success(); + do { + + char HeaderBuffer[FDMsgHeader::Size]; + // Read the header buffer. + { + bool IsEOF = false; + if (auto Err2 = readBytes(HeaderBuffer, FDMsgHeader::Size, &IsEOF)) { + Err = joinErrors(std::move(Err), std::move(Err2)); + break; + } + if (IsEOF) + break; + } + + // Decode header buffer. + uint64_t MsgSize; + SimpleRemoteEPCOpcode OpC; + uint64_t SeqNo; + ExecutorAddr TagAddr; + + MsgSize = + *((support::ulittle64_t *)(HeaderBuffer + FDMsgHeader::MsgSizeOffset)); + OpC = static_cast<SimpleRemoteEPCOpcode>(static_cast<uint64_t>( + *((support::ulittle64_t *)(HeaderBuffer + FDMsgHeader::OpCOffset)))); + SeqNo = + *((support::ulittle64_t *)(HeaderBuffer + FDMsgHeader::SeqNoOffset)); + TagAddr.setValue( + *((support::ulittle64_t *)(HeaderBuffer + FDMsgHeader::TagAddrOffset))); + + if (MsgSize < FDMsgHeader::Size) { + Err = joinErrors(std::move(Err), + make_error<StringError>("Message size too small", + inconvertibleErrorCode())); + break; + } + + // Read the argument bytes. + SimpleRemoteEPCArgBytesVector ArgBytes; + ArgBytes.resize(MsgSize - FDMsgHeader::Size); + if (auto Err2 = readBytes(ArgBytes.data(), ArgBytes.size())) { + Err = joinErrors(std::move(Err), std::move(Err2)); + break; + } + + if (auto Action = C.handleMessage(OpC, SeqNo, TagAddr, ArgBytes)) { + if (*Action == SimpleRemoteEPCTransportClient::EndSession) + break; + } else { + Err = joinErrors(std::move(Err), Action.takeError()); + break; + } + } while (true); + + // Attempt to close FDs, set Disconnected to true so that subsequent + // sendMessage calls fail. + disconnect(); + + // Call up to the client to handle the disconnection. + C.handleDisconnect(std::move(Err)); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp new file mode 100644 index 000000000000..47364a92a451 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp @@ -0,0 +1,406 @@ +//===------- SimpleRemoteEPC.cpp -- Simple remote executor control --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/SimpleRemoteEPC.h" +#include "llvm/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/EPCGenericMemoryAccess.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/Support/FormatVariadic.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { + +SimpleRemoteEPC::~SimpleRemoteEPC() { +#ifndef NDEBUG + std::lock_guard<std::mutex> Lock(SimpleRemoteEPCMutex); + assert(Disconnected && "Destroyed without disconnection"); +#endif // NDEBUG +} + +Expected<tpctypes::DylibHandle> +SimpleRemoteEPC::loadDylib(const char *DylibPath) { + return DylibMgr->open(DylibPath, 0); +} + +Expected<std::vector<tpctypes::LookupResult>> +SimpleRemoteEPC::lookupSymbols(ArrayRef<LookupRequest> Request) { + std::vector<tpctypes::LookupResult> Result; + + for (auto &Element : Request) { + if (auto R = DylibMgr->lookup(Element.Handle, Element.Symbols)) { + Result.push_back({}); + Result.back().reserve(R->size()); + for (auto Addr : *R) + Result.back().push_back(Addr.getValue()); + } else + return R.takeError(); + } + return std::move(Result); +} + +Expected<int32_t> SimpleRemoteEPC::runAsMain(ExecutorAddr MainFnAddr, + ArrayRef<std::string> Args) { + int64_t Result = 0; + if (auto Err = callSPSWrapper<rt::SPSRunAsMainSignature>( + RunAsMainAddr, Result, ExecutorAddr(MainFnAddr), Args)) + return std::move(Err); + return Result; +} + +void SimpleRemoteEPC::callWrapperAsync(ExecutorAddr WrapperFnAddr, + IncomingWFRHandler OnComplete, + ArrayRef<char> ArgBuffer) { + uint64_t SeqNo; + { + std::lock_guard<std::mutex> Lock(SimpleRemoteEPCMutex); + SeqNo = getNextSeqNo(); + assert(!PendingCallWrapperResults.count(SeqNo) && "SeqNo already in use"); + PendingCallWrapperResults[SeqNo] = std::move(OnComplete); + } + + if (auto Err = sendMessage(SimpleRemoteEPCOpcode::CallWrapper, SeqNo, + WrapperFnAddr, ArgBuffer)) { + IncomingWFRHandler H; + + // We just registered OnComplete, but there may be a race between this + // thread returning from sendMessage and handleDisconnect being called from + // the transport's listener thread. If handleDisconnect gets there first + // then it will have failed 'H' for us. If we get there first (or if + // handleDisconnect already ran) then we need to take care of it. + { + std::lock_guard<std::mutex> Lock(SimpleRemoteEPCMutex); + auto I = PendingCallWrapperResults.find(SeqNo); + if (I != PendingCallWrapperResults.end()) { + H = std::move(I->second); + PendingCallWrapperResults.erase(I); + } + } + + if (H) + H(shared::WrapperFunctionResult::createOutOfBandError("disconnecting")); + + getExecutionSession().reportError(std::move(Err)); + } +} + +Error SimpleRemoteEPC::disconnect() { + T->disconnect(); + D->shutdown(); + std::unique_lock<std::mutex> Lock(SimpleRemoteEPCMutex); + DisconnectCV.wait(Lock, [this] { return Disconnected; }); + return std::move(DisconnectErr); +} + +Expected<SimpleRemoteEPCTransportClient::HandleMessageAction> +SimpleRemoteEPC::handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, + ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + + LLVM_DEBUG({ + dbgs() << "SimpleRemoteEPC::handleMessage: opc = "; + switch (OpC) { + case SimpleRemoteEPCOpcode::Setup: + dbgs() << "Setup"; + assert(SeqNo == 0 && "Non-zero SeqNo for Setup?"); + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Setup?"); + break; + case SimpleRemoteEPCOpcode::Hangup: + dbgs() << "Hangup"; + assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + break; + case SimpleRemoteEPCOpcode::Result: + dbgs() << "Result"; + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + break; + case SimpleRemoteEPCOpcode::CallWrapper: + dbgs() << "CallWrapper"; + break; + } + dbgs() << ", seqno = " << SeqNo + << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) + << " bytes\n"; + }); + + using UT = std::underlying_type_t<SimpleRemoteEPCOpcode>; + if (static_cast<UT>(OpC) > static_cast<UT>(SimpleRemoteEPCOpcode::LastOpC)) + return make_error<StringError>("Unexpected opcode", + inconvertibleErrorCode()); + + switch (OpC) { + case SimpleRemoteEPCOpcode::Setup: + if (auto Err = handleSetup(SeqNo, TagAddr, std::move(ArgBytes))) + return std::move(Err); + break; + case SimpleRemoteEPCOpcode::Hangup: + T->disconnect(); + if (auto Err = handleHangup(std::move(ArgBytes))) + return std::move(Err); + return EndSession; + case SimpleRemoteEPCOpcode::Result: + if (auto Err = handleResult(SeqNo, TagAddr, std::move(ArgBytes))) + return std::move(Err); + break; + case SimpleRemoteEPCOpcode::CallWrapper: + handleCallWrapper(SeqNo, TagAddr, std::move(ArgBytes)); + break; + } + return ContinueSession; +} + +void SimpleRemoteEPC::handleDisconnect(Error Err) { + LLVM_DEBUG({ + dbgs() << "SimpleRemoteEPC::handleDisconnect: " + << (Err ? "failure" : "success") << "\n"; + }); + + PendingCallWrapperResultsMap TmpPending; + + { + std::lock_guard<std::mutex> Lock(SimpleRemoteEPCMutex); + std::swap(TmpPending, PendingCallWrapperResults); + } + + for (auto &KV : TmpPending) + KV.second( + shared::WrapperFunctionResult::createOutOfBandError("disconnecting")); + + std::lock_guard<std::mutex> Lock(SimpleRemoteEPCMutex); + DisconnectErr = joinErrors(std::move(DisconnectErr), std::move(Err)); + Disconnected = true; + DisconnectCV.notify_all(); +} + +Expected<std::unique_ptr<jitlink::JITLinkMemoryManager>> +SimpleRemoteEPC::createDefaultMemoryManager(SimpleRemoteEPC &SREPC) { + EPCGenericJITLinkMemoryManager::SymbolAddrs SAs; + if (auto Err = SREPC.getBootstrapSymbols( + {{SAs.Allocator, rt::SimpleExecutorMemoryManagerInstanceName}, + {SAs.Reserve, rt::SimpleExecutorMemoryManagerReserveWrapperName}, + {SAs.Finalize, rt::SimpleExecutorMemoryManagerFinalizeWrapperName}, + {SAs.Deallocate, + rt::SimpleExecutorMemoryManagerDeallocateWrapperName}})) + return std::move(Err); + + return std::make_unique<EPCGenericJITLinkMemoryManager>(SREPC, SAs); +} + +Expected<std::unique_ptr<ExecutorProcessControl::MemoryAccess>> +SimpleRemoteEPC::createDefaultMemoryAccess(SimpleRemoteEPC &SREPC) { + return nullptr; +} + +Error SimpleRemoteEPC::sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, + ExecutorAddr TagAddr, + ArrayRef<char> ArgBytes) { + assert(OpC != SimpleRemoteEPCOpcode::Setup && + "SimpleRemoteEPC sending Setup message? That's the wrong direction."); + + LLVM_DEBUG({ + dbgs() << "SimpleRemoteEPC::sendMessage: opc = "; + switch (OpC) { + case SimpleRemoteEPCOpcode::Hangup: + dbgs() << "Hangup"; + assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + break; + case SimpleRemoteEPCOpcode::Result: + dbgs() << "Result"; + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + break; + case SimpleRemoteEPCOpcode::CallWrapper: + dbgs() << "CallWrapper"; + break; + default: + llvm_unreachable("Invalid opcode"); + } + dbgs() << ", seqno = " << SeqNo + << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) + << " bytes\n"; + }); + auto Err = T->sendMessage(OpC, SeqNo, TagAddr, ArgBytes); + LLVM_DEBUG({ + if (Err) + dbgs() << " \\--> SimpleRemoteEPC::sendMessage failed\n"; + }); + return Err; +} + +Error SimpleRemoteEPC::handleSetup(uint64_t SeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + if (SeqNo != 0) + return make_error<StringError>("Setup packet SeqNo not zero", + inconvertibleErrorCode()); + + if (TagAddr) + return make_error<StringError>("Setup packet TagAddr not zero", + inconvertibleErrorCode()); + + std::lock_guard<std::mutex> Lock(SimpleRemoteEPCMutex); + auto I = PendingCallWrapperResults.find(0); + assert(PendingCallWrapperResults.size() == 1 && + I != PendingCallWrapperResults.end() && + "Setup message handler not connectly set up"); + auto SetupMsgHandler = std::move(I->second); + PendingCallWrapperResults.erase(I); + + auto WFR = + shared::WrapperFunctionResult::copyFrom(ArgBytes.data(), ArgBytes.size()); + SetupMsgHandler(std::move(WFR)); + return Error::success(); +} + +Error SimpleRemoteEPC::setup(Setup S) { + using namespace SimpleRemoteEPCDefaultBootstrapSymbolNames; + + std::promise<MSVCPExpected<SimpleRemoteEPCExecutorInfo>> EIP; + auto EIF = EIP.get_future(); + + // Prepare a handler for the setup packet. + PendingCallWrapperResults[0] = + RunInPlace()( + [&](shared::WrapperFunctionResult SetupMsgBytes) { + if (const char *ErrMsg = SetupMsgBytes.getOutOfBandError()) { + EIP.set_value( + make_error<StringError>(ErrMsg, inconvertibleErrorCode())); + return; + } + using SPSSerialize = + shared::SPSArgList<shared::SPSSimpleRemoteEPCExecutorInfo>; + shared::SPSInputBuffer IB(SetupMsgBytes.data(), SetupMsgBytes.size()); + SimpleRemoteEPCExecutorInfo EI; + if (SPSSerialize::deserialize(IB, EI)) + EIP.set_value(EI); + else + EIP.set_value(make_error<StringError>( + "Could not deserialize setup message", inconvertibleErrorCode())); + }); + + // Start the transport. + if (auto Err = T->start()) + return Err; + + // Wait for setup packet to arrive. + auto EI = EIF.get(); + if (!EI) { + T->disconnect(); + return EI.takeError(); + } + + LLVM_DEBUG({ + dbgs() << "SimpleRemoteEPC received setup message:\n" + << " Triple: " << EI->TargetTriple << "\n" + << " Page size: " << EI->PageSize << "\n" + << " Bootstrap symbols:\n"; + for (const auto &KV : EI->BootstrapSymbols) + dbgs() << " " << KV.first() << ": " + << formatv("{0:x16}", KV.second.getValue()) << "\n"; + }); + TargetTriple = Triple(EI->TargetTriple); + PageSize = EI->PageSize; + BootstrapSymbols = std::move(EI->BootstrapSymbols); + + if (auto Err = getBootstrapSymbols( + {{JDI.JITDispatchContext, ExecutorSessionObjectName}, + {JDI.JITDispatchFunction, DispatchFnName}, + {RunAsMainAddr, rt::RunAsMainWrapperName}})) + return Err; + + if (auto DM = + EPCGenericDylibManager::CreateWithDefaultBootstrapSymbols(*this)) + DylibMgr = std::make_unique<EPCGenericDylibManager>(std::move(*DM)); + else + return DM.takeError(); + + // Set a default CreateMemoryManager if none is specified. + if (!S.CreateMemoryManager) + S.CreateMemoryManager = createDefaultMemoryManager; + + if (auto MemMgr = S.CreateMemoryManager(*this)) { + OwnedMemMgr = std::move(*MemMgr); + this->MemMgr = OwnedMemMgr.get(); + } else + return MemMgr.takeError(); + + // Set a default CreateMemoryAccess if none is specified. + if (!S.CreateMemoryAccess) + S.CreateMemoryAccess = createDefaultMemoryAccess; + + if (auto MemAccess = S.CreateMemoryAccess(*this)) { + OwnedMemAccess = std::move(*MemAccess); + this->MemAccess = OwnedMemAccess.get(); + } else + return MemAccess.takeError(); + + return Error::success(); +} + +Error SimpleRemoteEPC::handleResult(uint64_t SeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + IncomingWFRHandler SendResult; + + if (TagAddr) + return make_error<StringError>("Unexpected TagAddr in result message", + inconvertibleErrorCode()); + + { + std::lock_guard<std::mutex> Lock(SimpleRemoteEPCMutex); + auto I = PendingCallWrapperResults.find(SeqNo); + if (I == PendingCallWrapperResults.end()) + return make_error<StringError>("No call for sequence number " + + Twine(SeqNo), + inconvertibleErrorCode()); + SendResult = std::move(I->second); + PendingCallWrapperResults.erase(I); + releaseSeqNo(SeqNo); + } + + auto WFR = + shared::WrapperFunctionResult::copyFrom(ArgBytes.data(), ArgBytes.size()); + SendResult(std::move(WFR)); + return Error::success(); +} + +void SimpleRemoteEPC::handleCallWrapper( + uint64_t RemoteSeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + assert(ES && "No ExecutionSession attached"); + D->dispatch(makeGenericNamedTask( + [this, RemoteSeqNo, TagAddr, ArgBytes = std::move(ArgBytes)]() { + ES->runJITDispatchHandler( + [this, RemoteSeqNo](shared::WrapperFunctionResult WFR) { + if (auto Err = + sendMessage(SimpleRemoteEPCOpcode::Result, RemoteSeqNo, + ExecutorAddr(), {WFR.data(), WFR.size()})) + getExecutionSession().reportError(std::move(Err)); + }, + TagAddr.getValue(), ArgBytes); + }, + "callWrapper task")); +} + +Error SimpleRemoteEPC::handleHangup(SimpleRemoteEPCArgBytesVector ArgBytes) { + using namespace llvm::orc::shared; + auto WFR = WrapperFunctionResult::copyFrom(ArgBytes.data(), ArgBytes.size()); + if (const char *ErrMsg = WFR.getOutOfBandError()) + return make_error<StringError>(ErrMsg, inconvertibleErrorCode()); + + detail::SPSSerializableError Info; + SPSInputBuffer IB(WFR.data(), WFR.size()); + if (!SPSArgList<SPSError>::deserialize(IB, Info)) + return make_error<StringError>("Could not deserialize hangup info", + inconvertibleErrorCode()); + return fromSPSSerializable(std::move(Info)); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp index 43c2a44835fd..4c15e25b1d89 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp @@ -10,6 +10,7 @@ #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/ManagedStatic.h" #include <cstdint> @@ -64,14 +65,23 @@ LLVM_ATTRIBUTE_NOINLINE void __jit_debug_register_code() { } using namespace llvm; +using namespace llvm::orc; // Serialize rendezvous with the debugger as well as access to shared data. ManagedStatic<std::mutex> JITDebugLock; // Register debug object, return error message or null for success. -static void registerJITLoaderGDBImpl(JITTargetAddress Addr, uint64_t Size) { +static void registerJITLoaderGDBImpl(const char *ObjAddr, size_t Size) { + LLVM_DEBUG({ + dbgs() << "Registering debug object with GDB JIT interface " + << formatv("([{0:x16} -- {1:x16}])", + reinterpret_cast<uintptr_t>(ObjAddr), + reinterpret_cast<uintptr_t>(ObjAddr + Size)) + << "\n"; + }); + jit_code_entry *E = new jit_code_entry; - E->symfile_addr = jitTargetAddressToPointer<const char *>(Addr); + E->symfile_addr = ObjAddr; E->symfile_size = Size; E->prev_entry = nullptr; @@ -92,10 +102,26 @@ static void registerJITLoaderGDBImpl(JITTargetAddress Addr, uint64_t Size) { __jit_debug_register_code(); } -extern "C" orc::shared::detail::CWrapperFunctionResult +extern "C" orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderGDBAllocAction(const char *Data, size_t Size) { + using namespace orc::shared; + return WrapperFunction<SPSError()>::handle(nullptr, 0, + [=]() -> Error { + registerJITLoaderGDBImpl(Data, + Size); + return Error::success(); + }) + .release(); +} + +extern "C" orc::shared::CWrapperFunctionResult llvm_orc_registerJITLoaderGDBWrapper(const char *Data, uint64_t Size) { using namespace orc::shared; - return WrapperFunction<void(SPSExecutorAddress, uint64_t)>::handle( - Data, Size, registerJITLoaderGDBImpl) + return WrapperFunction<void(SPSExecutorAddrRange)>::handle( + Data, Size, + [](ExecutorAddrRange R) { + registerJITLoaderGDBImpl(R.Start.toPtr<char *>(), + R.size().getValue()); + }) .release(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp new file mode 100644 index 000000000000..82aa62a0c0d9 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp @@ -0,0 +1,84 @@ +//===------------------------ OrcRTBootstrap.cpp --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "OrcRTBootstrap.h" + +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm::orc::shared; + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +template <typename WriteT, typename SPSWriteT> +static llvm::orc::shared::CWrapperFunctionResult +writeUIntsWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<void(SPSSequence<SPSWriteT>)>::handle( + ArgData, ArgSize, + [](std::vector<WriteT> Ws) { + for (auto &W : Ws) + *W.Addr.template toPtr<decltype(W.Value) *>() = W.Value; + }) + .release(); +} + +static llvm::orc::shared::CWrapperFunctionResult +writeBuffersWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<void(SPSSequence<SPSMemoryAccessBufferWrite>)>::handle( + ArgData, ArgSize, + [](std::vector<tpctypes::BufferWrite> Ws) { + for (auto &W : Ws) + memcpy(W.Addr.template toPtr<char *>(), W.Buffer.data(), + W.Buffer.size()); + }) + .release(); +} + +static llvm::orc::shared::CWrapperFunctionResult +runAsMainWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<rt::SPSRunAsMainSignature>::handle( + ArgData, ArgSize, + [](ExecutorAddr MainAddr, + std::vector<std::string> Args) -> int64_t { + return runAsMain(MainAddr.toPtr<int (*)(int, char *[])>(), Args); + }) + .release(); +} + +void addTo(StringMap<ExecutorAddr> &M) { + M[rt::MemoryWriteUInt8sWrapperName] = ExecutorAddr::fromPtr( + &writeUIntsWrapper<tpctypes::UInt8Write, + shared::SPSMemoryAccessUInt8Write>); + M[rt::MemoryWriteUInt16sWrapperName] = ExecutorAddr::fromPtr( + &writeUIntsWrapper<tpctypes::UInt16Write, + shared::SPSMemoryAccessUInt16Write>); + M[rt::MemoryWriteUInt32sWrapperName] = ExecutorAddr::fromPtr( + &writeUIntsWrapper<tpctypes::UInt32Write, + shared::SPSMemoryAccessUInt32Write>); + M[rt::MemoryWriteUInt64sWrapperName] = ExecutorAddr::fromPtr( + &writeUIntsWrapper<tpctypes::UInt64Write, + shared::SPSMemoryAccessUInt64Write>); + M[rt::MemoryWriteBuffersWrapperName] = + ExecutorAddr::fromPtr(&writeBuffersWrapper); + M[rt::RegisterEHFrameSectionCustomDirectWrapperName] = ExecutorAddr::fromPtr( + &llvm_orc_registerEHFrameSectionCustomDirectWrapper); + M[rt::DeregisterEHFrameSectionCustomDirectWrapperName] = + ExecutorAddr::fromPtr( + &llvm_orc_deregisterEHFrameSectionCustomDirectWrapper); + M[rt::RunAsMainWrapperName] = ExecutorAddr::fromPtr(&runAsMainWrapper); +} + +} // end namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.h new file mode 100644 index 000000000000..6b7ff79a3efc --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.h @@ -0,0 +1,36 @@ +//===----------------------- OrcRTBootstrap.h -------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// OrcRTPrelinkImpl provides functions that should be linked into the executor +// to bootstrap common JIT functionality (e.g. memory allocation and memory +// access). +// +// Call rt_impl::addTo to add these functions to a bootstrap symbols map. +// +// FIXME: The functionality in this file should probably be moved to an ORC +// runtime bootstrap library in compiler-rt. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_ORC_TARGETPROCESS_ORCRTBOOTSTRAP_H +#define LIB_EXECUTIONENGINE_ORC_TARGETPROCESS_ORCRTBOOTSTRAP_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ExecutionEngine/Orc/Shared/ExecutorAddress.h" + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +void addTo(StringMap<ExecutorAddr> &M); + +} // namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_TARGETPROCESS_ORCRTBOOTSTRAP_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp index 4a408d61ee38..e331bad84200 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp @@ -1,9 +1,8 @@ //===--------- RegisterEHFrames.cpp - Register EH frame sections ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -86,11 +85,11 @@ static Error deregisterFrameWrapper(const void *P) { } #endif -#ifdef __APPLE__ +#if defined(HAVE_UNW_ADD_DYNAMIC_FDE) || defined(__APPLE__) template <typename HandleFDEFn> -Error walkAppleEHFrameSection(const char *const SectionStart, - size_t SectionSize, HandleFDEFn HandleFDE) { +Error walkLibunwindEHFrameSection(const char *const SectionStart, + size_t SectionSize, HandleFDEFn HandleFDE) { const char *CurCFIRecord = SectionStart; const char *End = SectionStart + SectionSize; uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); @@ -124,16 +123,19 @@ Error walkAppleEHFrameSection(const char *const SectionStart, return Error::success(); } -#endif // __APPLE__ +#endif // HAVE_UNW_ADD_DYNAMIC_FDE || __APPLE__ Error registerEHFrameSection(const void *EHFrameSectionAddr, size_t EHFrameSectionSize) { -#ifdef __APPLE__ - // On Darwin __register_frame has to be called for each FDE entry. - return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), - EHFrameSectionSize, registerFrameWrapper); + /* libgcc and libunwind __register_frame behave differently. We use the + * presence of __unw_add_dynamic_fde to detect libunwind. */ +#if defined(HAVE_UNW_ADD_DYNAMIC_FDE) || defined(__APPLE__) + // With libunwind, __register_frame has to be called for each FDE entry. + return walkLibunwindEHFrameSection( + static_cast<const char *>(EHFrameSectionAddr), EHFrameSectionSize, + registerFrameWrapper); #else - // On Linux __register_frame takes a single argument: + // With libgcc, __register_frame takes a single argument: // a pointer to the start of the .eh_frame section. // How can it find the end? Because crtendS.o is linked @@ -144,9 +146,10 @@ Error registerEHFrameSection(const void *EHFrameSectionAddr, Error deregisterEHFrameSection(const void *EHFrameSectionAddr, size_t EHFrameSectionSize) { -#ifdef __APPLE__ - return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), - EHFrameSectionSize, deregisterFrameWrapper); +#if defined(HAVE_UNW_ADD_DYNAMIC_FDE) || defined(__APPLE__) + return walkLibunwindEHFrameSection( + static_cast<const char *>(EHFrameSectionAddr), EHFrameSectionSize, + deregisterFrameWrapper); #else return deregisterFrameWrapper(EHFrameSectionAddr); #endif @@ -155,26 +158,42 @@ Error deregisterEHFrameSection(const void *EHFrameSectionAddr, } // end namespace orc } // end namespace llvm -static Error registerEHFrameWrapper(JITTargetAddress Addr, uint64_t Size) { - return llvm::orc::registerEHFrameSection( - jitTargetAddressToPointer<const void *>(Addr), Size); +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerEHFrameSectionCustomDirectWrapper( + const char *EHFrameSectionAddr, uint64_t Size) { + if (auto Err = registerEHFrameSection(EHFrameSectionAddr, Size)) + return WrapperFunctionResult::createOutOfBandError(toString(std::move(Err))) + .release(); + return llvm::orc::shared::CWrapperFunctionResult(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_deregisterEHFrameSectionCustomDirectWrapper( + const char *EHFrameSectionAddr, uint64_t Size) { + if (auto Err = deregisterEHFrameSection(EHFrameSectionAddr, Size)) + return WrapperFunctionResult::createOutOfBandError(toString(std::move(Err))) + .release(); + return llvm::orc::shared::CWrapperFunctionResult(); +} + +static Error registerEHFrameWrapper(ExecutorAddr Addr, uint64_t Size) { + return llvm::orc::registerEHFrameSection(Addr.toPtr<const void *>(), Size); } -static Error deregisterEHFrameWrapper(JITTargetAddress Addr, uint64_t Size) { - return llvm::orc::deregisterEHFrameSection( - jitTargetAddressToPointer<const void *>(Addr), Size); +static Error deregisterEHFrameWrapper(ExecutorAddr Addr, uint64_t Size) { + return llvm::orc::deregisterEHFrameSection(Addr.toPtr<const void *>(), Size); } -extern "C" orc::shared::detail::CWrapperFunctionResult +extern "C" orc::shared::CWrapperFunctionResult llvm_orc_registerEHFrameSectionWrapper(const char *Data, uint64_t Size) { - return WrapperFunction<SPSError(SPSExecutorAddress, uint64_t)>::handle( + return WrapperFunction<SPSError(SPSExecutorAddr, uint64_t)>::handle( Data, Size, registerEHFrameWrapper) .release(); } -extern "C" orc::shared::detail::CWrapperFunctionResult +extern "C" orc::shared::CWrapperFunctionResult llvm_orc_deregisterEHFrameSectionWrapper(const char *Data, uint64_t Size) { - return WrapperFunction<SPSError(SPSExecutorAddress, uint64_t)>::handle( + return WrapperFunction<SPSError(SPSExecutorAddr, uint64_t)>::handle( Data, Size, deregisterEHFrameWrapper) .release(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp new file mode 100644 index 000000000000..3c9dd21b0832 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp @@ -0,0 +1,129 @@ +//===--- SimpleExecutorDylibManager.cpp - Executor-side dylib management --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.h" + +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/Support/FormatVariadic.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +SimpleExecutorDylibManager::~SimpleExecutorDylibManager() { + assert(Dylibs.empty() && "shutdown not called?"); +} + +Expected<tpctypes::DylibHandle> +SimpleExecutorDylibManager::open(const std::string &Path, uint64_t Mode) { + if (Mode != 0) + return make_error<StringError>("open: non-zero mode bits not yet supported", + inconvertibleErrorCode()); + + const char *PathCStr = Path.empty() ? nullptr : Path.c_str(); + std::string ErrMsg; + + auto DL = sys::DynamicLibrary::getPermanentLibrary(PathCStr, &ErrMsg); + if (!DL.isValid()) + return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + + std::lock_guard<std::mutex> Lock(M); + Dylibs[NextId] = std::move(DL); + return NextId++; +} + +Expected<std::vector<ExecutorAddr>> +SimpleExecutorDylibManager::lookup(tpctypes::DylibHandle H, + const RemoteSymbolLookupSet &L) { + std::vector<ExecutorAddr> Result; + + std::lock_guard<std::mutex> Lock(M); + auto I = Dylibs.find(H); + if (I == Dylibs.end()) + return make_error<StringError>("No dylib for handle " + formatv("{0:x}", H), + inconvertibleErrorCode()); + auto &DL = I->second; + + for (const auto &E : L) { + + if (E.Name.empty()) { + if (E.Required) + return make_error<StringError>("Required address for empty symbol \"\"", + inconvertibleErrorCode()); + else + Result.push_back(ExecutorAddr()); + } else { + + const char *DemangledSymName = E.Name.c_str(); +#ifdef __APPLE__ + if (E.Name.front() != '_') + return make_error<StringError>(Twine("MachO symbol \"") + E.Name + + "\" missing leading '_'", + inconvertibleErrorCode()); + ++DemangledSymName; +#endif + + void *Addr = DL.getAddressOfSymbol(DemangledSymName); + if (!Addr && E.Required) + return make_error<StringError>(Twine("Missing definition for ") + + DemangledSymName, + inconvertibleErrorCode()); + + Result.push_back(ExecutorAddr::fromPtr(Addr)); + } + } + + return Result; +} + +Error SimpleExecutorDylibManager::shutdown() { + + DylibsMap DM; + { + std::lock_guard<std::mutex> Lock(M); + std::swap(DM, Dylibs); + } + + // There is no removal of dylibs at the moment, so nothing to do here. + return Error::success(); +} + +void SimpleExecutorDylibManager::addBootstrapSymbols( + StringMap<ExecutorAddr> &M) { + M[rt::SimpleExecutorDylibManagerInstanceName] = ExecutorAddr::fromPtr(this); + M[rt::SimpleExecutorDylibManagerOpenWrapperName] = + ExecutorAddr::fromPtr(&openWrapper); + M[rt::SimpleExecutorDylibManagerLookupWrapperName] = + ExecutorAddr::fromPtr(&lookupWrapper); +} + +llvm::orc::shared::CWrapperFunctionResult +SimpleExecutorDylibManager::openWrapper(const char *ArgData, size_t ArgSize) { + return shared:: + WrapperFunction<rt::SPSSimpleExecutorDylibManagerOpenSignature>::handle( + ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &SimpleExecutorDylibManager::open)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +SimpleExecutorDylibManager::lookupWrapper(const char *ArgData, size_t ArgSize) { + return shared:: + WrapperFunction<rt::SPSSimpleExecutorDylibManagerLookupSignature>::handle( + ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &SimpleExecutorDylibManager::lookup)) + .release(); +} + +} // namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp new file mode 100644 index 000000000000..232340c22a32 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp @@ -0,0 +1,261 @@ +//===- SimpleExecuorMemoryManagare.cpp - Simple executor-side memory mgmt -===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.h" + +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/Support/FormatVariadic.h" + +#define DEBUG_TYPE "orc" + +namespace llvm { +namespace orc { +namespace rt_bootstrap { + +SimpleExecutorMemoryManager::~SimpleExecutorMemoryManager() { + assert(Allocations.empty() && "shutdown not called?"); +} + +Expected<ExecutorAddr> SimpleExecutorMemoryManager::allocate(uint64_t Size) { + std::error_code EC; + auto MB = sys::Memory::allocateMappedMemory( + Size, 0, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC); + if (EC) + return errorCodeToError(EC); + std::lock_guard<std::mutex> Lock(M); + assert(!Allocations.count(MB.base()) && "Duplicate allocation addr"); + Allocations[MB.base()].Size = Size; + return ExecutorAddr::fromPtr(MB.base()); +} + +Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) { + ExecutorAddr Base(~0ULL); + std::vector<tpctypes::WrapperFunctionCall> DeallocationActions; + size_t SuccessfulFinalizationActions = 0; + + if (FR.Segments.empty()) { + // NOTE: Finalizing nothing is currently a no-op. Should it be an error? + if (FR.Actions.empty()) + return Error::success(); + else + return make_error<StringError>("Finalization actions attached to empty " + "finalization request", + inconvertibleErrorCode()); + } + + for (auto &Seg : FR.Segments) + Base = std::min(Base, Seg.Addr); + + for (auto &ActPair : FR.Actions) + if (ActPair.Deallocate.Func) + DeallocationActions.push_back(ActPair.Deallocate); + + // Get the Allocation for this finalization. + size_t AllocSize = 0; + { + std::lock_guard<std::mutex> Lock(M); + auto I = Allocations.find(Base.toPtr<void *>()); + if (I == Allocations.end()) + return make_error<StringError>("Attempt to finalize unrecognized " + "allocation " + + formatv("{0:x}", Base.getValue()), + inconvertibleErrorCode()); + AllocSize = I->second.Size; + I->second.DeallocationActions = std::move(DeallocationActions); + } + ExecutorAddr AllocEnd = Base + ExecutorAddrDiff(AllocSize); + + // Bail-out function: this will run deallocation actions corresponding to any + // completed finalization actions, then deallocate memory. + auto BailOut = [&](Error Err) { + std::pair<void *, Allocation> AllocToDestroy; + + // Get allocation to destory. + { + std::lock_guard<std::mutex> Lock(M); + auto I = Allocations.find(Base.toPtr<void *>()); + + // Check for missing allocation (effective a double free). + if (I == Allocations.end()) + return joinErrors( + std::move(Err), + make_error<StringError>("No allocation entry found " + "for " + + formatv("{0:x}", Base.getValue()), + inconvertibleErrorCode())); + AllocToDestroy = std::move(*I); + Allocations.erase(I); + } + + // Run deallocation actions for all completed finalization actions. + while (SuccessfulFinalizationActions) + Err = + joinErrors(std::move(Err), FR.Actions[--SuccessfulFinalizationActions] + .Deallocate.runWithSPSRet()); + + // Deallocate memory. + sys::MemoryBlock MB(AllocToDestroy.first, AllocToDestroy.second.Size); + if (auto EC = sys::Memory::releaseMappedMemory(MB)) + Err = joinErrors(std::move(Err), errorCodeToError(EC)); + + return Err; + }; + + // Copy content and apply permissions. + for (auto &Seg : FR.Segments) { + + // Check segment ranges. + if (LLVM_UNLIKELY(Seg.Size < Seg.Content.size())) + return BailOut(make_error<StringError>( + formatv("Segment {0:x} content size ({1:x} bytes) " + "exceeds segment size ({2:x} bytes)", + Seg.Addr.getValue(), Seg.Content.size(), Seg.Size), + inconvertibleErrorCode())); + ExecutorAddr SegEnd = Seg.Addr + ExecutorAddrDiff(Seg.Size); + if (LLVM_UNLIKELY(Seg.Addr < Base || SegEnd > AllocEnd)) + return BailOut(make_error<StringError>( + formatv("Segment {0:x} -- {1:x} crosses boundary of " + "allocation {2:x} -- {3:x}", + Seg.Addr.getValue(), SegEnd.getValue(), Base.getValue(), + AllocEnd.getValue()), + inconvertibleErrorCode())); + + char *Mem = Seg.Addr.toPtr<char *>(); + memcpy(Mem, Seg.Content.data(), Seg.Content.size()); + memset(Mem + Seg.Content.size(), 0, Seg.Size - Seg.Content.size()); + assert(Seg.Size <= std::numeric_limits<size_t>::max()); + if (auto EC = sys::Memory::protectMappedMemory( + {Mem, static_cast<size_t>(Seg.Size)}, + tpctypes::fromWireProtectionFlags(Seg.Prot))) + return BailOut(errorCodeToError(EC)); + if (Seg.Prot & tpctypes::WPF_Exec) + sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); + } + + // Run finalization actions. + for (auto &ActPair : FR.Actions) { + if (auto Err = ActPair.Finalize.runWithSPSRet()) + return BailOut(std::move(Err)); + ++SuccessfulFinalizationActions; + } + + return Error::success(); +} + +Error SimpleExecutorMemoryManager::deallocate( + const std::vector<ExecutorAddr> &Bases) { + std::vector<std::pair<void *, Allocation>> AllocPairs; + AllocPairs.reserve(Bases.size()); + + // Get allocation to destory. + Error Err = Error::success(); + { + std::lock_guard<std::mutex> Lock(M); + for (auto &Base : Bases) { + auto I = Allocations.find(Base.toPtr<void *>()); + + // Check for missing allocation (effective a double free). + if (I != Allocations.end()) { + AllocPairs.push_back(std::move(*I)); + Allocations.erase(I); + } else + Err = joinErrors( + std::move(Err), + make_error<StringError>("No allocation entry found " + "for " + + formatv("{0:x}", Base.getValue()), + inconvertibleErrorCode())); + } + } + + while (!AllocPairs.empty()) { + auto &P = AllocPairs.back(); + Err = joinErrors(std::move(Err), deallocateImpl(P.first, P.second)); + AllocPairs.pop_back(); + } + + return Err; +} + +Error SimpleExecutorMemoryManager::shutdown() { + + AllocationsMap AM; + { + std::lock_guard<std::mutex> Lock(M); + AM = std::move(Allocations); + } + + Error Err = Error::success(); + for (auto &KV : AM) + Err = joinErrors(std::move(Err), deallocateImpl(KV.first, KV.second)); + return Err; +} + +void SimpleExecutorMemoryManager::addBootstrapSymbols( + StringMap<ExecutorAddr> &M) { + M[rt::SimpleExecutorMemoryManagerInstanceName] = ExecutorAddr::fromPtr(this); + M[rt::SimpleExecutorMemoryManagerReserveWrapperName] = + ExecutorAddr::fromPtr(&reserveWrapper); + M[rt::SimpleExecutorMemoryManagerFinalizeWrapperName] = + ExecutorAddr::fromPtr(&finalizeWrapper); + M[rt::SimpleExecutorMemoryManagerDeallocateWrapperName] = + ExecutorAddr::fromPtr(&deallocateWrapper); +} + +Error SimpleExecutorMemoryManager::deallocateImpl(void *Base, Allocation &A) { + Error Err = Error::success(); + + while (!A.DeallocationActions.empty()) { + Err = joinErrors(std::move(Err), + A.DeallocationActions.back().runWithSPSRet()); + A.DeallocationActions.pop_back(); + } + + sys::MemoryBlock MB(Base, A.Size); + if (auto EC = sys::Memory::releaseMappedMemory(MB)) + Err = joinErrors(std::move(Err), errorCodeToError(EC)); + + return Err; +} + +llvm::orc::shared::CWrapperFunctionResult +SimpleExecutorMemoryManager::reserveWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSSimpleExecutorMemoryManagerReserveSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &SimpleExecutorMemoryManager::allocate)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +SimpleExecutorMemoryManager::finalizeWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSSimpleExecutorMemoryManagerFinalizeSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &SimpleExecutorMemoryManager::finalize)) + .release(); +} + +llvm::orc::shared::CWrapperFunctionResult +SimpleExecutorMemoryManager::deallocateWrapper(const char *ArgData, + size_t ArgSize) { + return shared::WrapperFunction< + rt::SPSSimpleExecutorMemoryManagerDeallocateSignature>:: + handle(ArgData, ArgSize, + shared::makeMethodWrapperHandler( + &SimpleExecutorMemoryManager::deallocate)) + .release(); +} + +} // namespace rt_bootstrap +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp new file mode 100644 index 000000000000..b6b21bde1182 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp @@ -0,0 +1,293 @@ +//===------- SimpleEPCServer.cpp - EPC over simple abstract channel -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h" + +#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/Host.h" +#include "llvm/Support/Process.h" + +#include "OrcRTBootstrap.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm::orc::shared; + +namespace llvm { +namespace orc { + +ExecutorBootstrapService::~ExecutorBootstrapService() {} + +SimpleRemoteEPCServer::Dispatcher::~Dispatcher() {} + +#if LLVM_ENABLE_THREADS +void SimpleRemoteEPCServer::ThreadDispatcher::dispatch( + unique_function<void()> Work) { + { + std::lock_guard<std::mutex> Lock(DispatchMutex); + if (!Running) + return; + ++Outstanding; + } + + std::thread([this, Work = std::move(Work)]() mutable { + Work(); + std::lock_guard<std::mutex> Lock(DispatchMutex); + --Outstanding; + OutstandingCV.notify_all(); + }).detach(); +} + +void SimpleRemoteEPCServer::ThreadDispatcher::shutdown() { + std::unique_lock<std::mutex> Lock(DispatchMutex); + Running = false; + OutstandingCV.wait(Lock, [this]() { return Outstanding == 0; }); +} +#endif + +StringMap<ExecutorAddr> SimpleRemoteEPCServer::defaultBootstrapSymbols() { + StringMap<ExecutorAddr> DBS; + rt_bootstrap::addTo(DBS); + return DBS; +} + +Expected<SimpleRemoteEPCTransportClient::HandleMessageAction> +SimpleRemoteEPCServer::handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, + ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + + LLVM_DEBUG({ + dbgs() << "SimpleRemoteEPCServer::handleMessage: opc = "; + switch (OpC) { + case SimpleRemoteEPCOpcode::Setup: + dbgs() << "Setup"; + assert(SeqNo == 0 && "Non-zero SeqNo for Setup?"); + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Setup?"); + break; + case SimpleRemoteEPCOpcode::Hangup: + dbgs() << "Hangup"; + assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + break; + case SimpleRemoteEPCOpcode::Result: + dbgs() << "Result"; + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + break; + case SimpleRemoteEPCOpcode::CallWrapper: + dbgs() << "CallWrapper"; + break; + } + dbgs() << ", seqno = " << SeqNo + << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) + << " bytes\n"; + }); + + using UT = std::underlying_type_t<SimpleRemoteEPCOpcode>; + if (static_cast<UT>(OpC) > static_cast<UT>(SimpleRemoteEPCOpcode::LastOpC)) + return make_error<StringError>("Unexpected opcode", + inconvertibleErrorCode()); + + // TODO: Clean detach message? + switch (OpC) { + case SimpleRemoteEPCOpcode::Setup: + return make_error<StringError>("Unexpected Setup opcode", + inconvertibleErrorCode()); + case SimpleRemoteEPCOpcode::Hangup: + return SimpleRemoteEPCTransportClient::EndSession; + case SimpleRemoteEPCOpcode::Result: + if (auto Err = handleResult(SeqNo, TagAddr, std::move(ArgBytes))) + return std::move(Err); + break; + case SimpleRemoteEPCOpcode::CallWrapper: + handleCallWrapper(SeqNo, TagAddr, std::move(ArgBytes)); + break; + } + return ContinueSession; +} + +Error SimpleRemoteEPCServer::waitForDisconnect() { + std::unique_lock<std::mutex> Lock(ServerStateMutex); + ShutdownCV.wait(Lock, [this]() { return RunState == ServerShutDown; }); + return std::move(ShutdownErr); +} + +void SimpleRemoteEPCServer::handleDisconnect(Error Err) { + PendingJITDispatchResultsMap TmpPending; + + { + std::lock_guard<std::mutex> Lock(ServerStateMutex); + std::swap(TmpPending, PendingJITDispatchResults); + RunState = ServerShuttingDown; + } + + // Send out-of-band errors to any waiting threads. + for (auto &KV : TmpPending) + KV.second->set_value( + shared::WrapperFunctionResult::createOutOfBandError("disconnecting")); + + // Wait for dispatcher to clear. + D->shutdown(); + + // Shut down services. + while (!Services.empty()) { + ShutdownErr = + joinErrors(std::move(ShutdownErr), Services.back()->shutdown()); + Services.pop_back(); + } + + std::lock_guard<std::mutex> Lock(ServerStateMutex); + ShutdownErr = joinErrors(std::move(ShutdownErr), std::move(Err)); + RunState = ServerShutDown; + ShutdownCV.notify_all(); +} + +Error SimpleRemoteEPCServer::sendMessage(SimpleRemoteEPCOpcode OpC, + uint64_t SeqNo, ExecutorAddr TagAddr, + ArrayRef<char> ArgBytes) { + + LLVM_DEBUG({ + dbgs() << "SimpleRemoteEPCServer::sendMessage: opc = "; + switch (OpC) { + case SimpleRemoteEPCOpcode::Setup: + dbgs() << "Setup"; + assert(SeqNo == 0 && "Non-zero SeqNo for Setup?"); + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Setup?"); + break; + case SimpleRemoteEPCOpcode::Hangup: + dbgs() << "Hangup"; + assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + break; + case SimpleRemoteEPCOpcode::Result: + dbgs() << "Result"; + assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + break; + case SimpleRemoteEPCOpcode::CallWrapper: + dbgs() << "CallWrapper"; + break; + } + dbgs() << ", seqno = " << SeqNo + << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) + << " bytes\n"; + }); + auto Err = T->sendMessage(OpC, SeqNo, TagAddr, ArgBytes); + LLVM_DEBUG({ + if (Err) + dbgs() << " \\--> SimpleRemoteEPC::sendMessage failed\n"; + }); + return Err; +} + +Error SimpleRemoteEPCServer::sendSetupMessage( + StringMap<ExecutorAddr> BootstrapSymbols) { + + using namespace SimpleRemoteEPCDefaultBootstrapSymbolNames; + + std::vector<char> SetupPacket; + SimpleRemoteEPCExecutorInfo EI; + EI.TargetTriple = sys::getProcessTriple(); + if (auto PageSize = sys::Process::getPageSize()) + EI.PageSize = *PageSize; + else + return PageSize.takeError(); + EI.BootstrapSymbols = std::move(BootstrapSymbols); + + assert(!EI.BootstrapSymbols.count(ExecutorSessionObjectName) && + "Dispatch context name should not be set"); + assert(!EI.BootstrapSymbols.count(DispatchFnName) && + "Dispatch function name should not be set"); + EI.BootstrapSymbols[ExecutorSessionObjectName] = ExecutorAddr::fromPtr(this); + EI.BootstrapSymbols[DispatchFnName] = ExecutorAddr::fromPtr(jitDispatchEntry); + + using SPSSerialize = + shared::SPSArgList<shared::SPSSimpleRemoteEPCExecutorInfo>; + auto SetupPacketBytes = + shared::WrapperFunctionResult::allocate(SPSSerialize::size(EI)); + shared::SPSOutputBuffer OB(SetupPacketBytes.data(), SetupPacketBytes.size()); + if (!SPSSerialize::serialize(OB, EI)) + return make_error<StringError>("Could not send setup packet", + inconvertibleErrorCode()); + + return sendMessage(SimpleRemoteEPCOpcode::Setup, 0, ExecutorAddr(), + {SetupPacketBytes.data(), SetupPacketBytes.size()}); +} + +Error SimpleRemoteEPCServer::handleResult( + uint64_t SeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + std::promise<shared::WrapperFunctionResult> *P = nullptr; + { + std::lock_guard<std::mutex> Lock(ServerStateMutex); + auto I = PendingJITDispatchResults.find(SeqNo); + if (I == PendingJITDispatchResults.end()) + return make_error<StringError>("No call for sequence number " + + Twine(SeqNo), + inconvertibleErrorCode()); + P = I->second; + PendingJITDispatchResults.erase(I); + releaseSeqNo(SeqNo); + } + auto R = shared::WrapperFunctionResult::allocate(ArgBytes.size()); + memcpy(R.data(), ArgBytes.data(), ArgBytes.size()); + P->set_value(std::move(R)); + return Error::success(); +} + +void SimpleRemoteEPCServer::handleCallWrapper( + uint64_t RemoteSeqNo, ExecutorAddr TagAddr, + SimpleRemoteEPCArgBytesVector ArgBytes) { + D->dispatch([this, RemoteSeqNo, TagAddr, ArgBytes = std::move(ArgBytes)]() { + using WrapperFnTy = + shared::CWrapperFunctionResult (*)(const char *, size_t); + auto *Fn = TagAddr.toPtr<WrapperFnTy>(); + shared::WrapperFunctionResult ResultBytes( + Fn(ArgBytes.data(), ArgBytes.size())); + if (auto Err = sendMessage(SimpleRemoteEPCOpcode::Result, RemoteSeqNo, + ExecutorAddr(), + {ResultBytes.data(), ResultBytes.size()})) + ReportError(std::move(Err)); + }); +} + +shared::WrapperFunctionResult +SimpleRemoteEPCServer::doJITDispatch(const void *FnTag, const char *ArgData, + size_t ArgSize) { + uint64_t SeqNo; + std::promise<shared::WrapperFunctionResult> ResultP; + auto ResultF = ResultP.get_future(); + { + std::lock_guard<std::mutex> Lock(ServerStateMutex); + if (RunState != ServerRunning) + return shared::WrapperFunctionResult::createOutOfBandError( + "jit_dispatch not available (EPC server shut down)"); + + SeqNo = getNextSeqNo(); + assert(!PendingJITDispatchResults.count(SeqNo) && "SeqNo already in use"); + PendingJITDispatchResults[SeqNo] = &ResultP; + } + + if (auto Err = sendMessage(SimpleRemoteEPCOpcode::CallWrapper, SeqNo, + ExecutorAddr::fromPtr(FnTag), {ArgData, ArgSize})) + ReportError(std::move(Err)); + + return ResultF.get(); +} + +shared::CWrapperFunctionResult +SimpleRemoteEPCServer::jitDispatchEntry(void *DispatchCtx, const void *FnTag, + const char *ArgData, size_t ArgSize) { + return reinterpret_cast<SimpleRemoteEPCServer *>(DispatchCtx) + ->doJITDispatch(FnTag, ArgData, ArgSize) + .release(); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TaskDispatch.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TaskDispatch.cpp new file mode 100644 index 000000000000..111c84ec87ed --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TaskDispatch.cpp @@ -0,0 +1,48 @@ +//===------------ TaskDispatch.cpp - ORC task dispatch utils --------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TaskDispatch.h" + +namespace llvm { +namespace orc { + +char Task::ID = 0; +char GenericNamedTask::ID = 0; +const char *GenericNamedTask::DefaultDescription = "Generic Task"; + +void Task::anchor() {} +TaskDispatcher::~TaskDispatcher() {} + +void InPlaceTaskDispatcher::dispatch(std::unique_ptr<Task> T) { T->run(); } + +void InPlaceTaskDispatcher::shutdown() {} + +#if LLVM_ENABLE_THREADS +void DynamicThreadPoolTaskDispatcher::dispatch(std::unique_ptr<Task> T) { + { + std::lock_guard<std::mutex> Lock(DispatchMutex); + ++Outstanding; + } + + std::thread([this, T = std::move(T)]() mutable { + T->run(); + std::lock_guard<std::mutex> Lock(DispatchMutex); + --Outstanding; + OutstandingCV.notify_all(); + }).detach(); +} + +void DynamicThreadPoolTaskDispatcher::shutdown() { + std::unique_lock<std::mutex> Lock(DispatchMutex); + Running = false; + OutstandingCV.wait(Lock, [this]() { return Outstanding == 0; }); +} +#endif + +} // namespace orc +} // namespace llvm |