diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2023-12-18 20:30:12 +0000 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2024-04-19 21:12:03 +0000 |
| commit | c9157d925c489f07ba9c0b2ce47e5149b75969a5 (patch) | |
| tree | 08bc4a3d9cad3f9ebffa558ddf140b9d9257b219 /contrib/llvm-project/llvm/lib/ExecutionEngine/Orc | |
| parent | 2a66844f606a35d68ad8a8061f4bea204274b3bc (diff) | |
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ExecutionEngine/Orc')
28 files changed, 2179 insertions, 970 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp index 7c869bead0b0..c8f5a99099ea 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp @@ -54,13 +54,13 @@ public: void materialize(std::unique_ptr<MaterializationResponsibility> R) override { unsigned PointerSize; - support::endianness Endianness; + llvm::endianness Endianness; const auto &TT = CP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::x86_64: PointerSize = 8; - Endianness = support::endianness::little; + Endianness = llvm::endianness::little; break; default: llvm_unreachable("Unrecognized architecture"); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp index 0c23f2b25219..56838e9bc86d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -31,6 +31,7 @@ char SymbolsCouldNotBeRemoved::ID = 0; char MissingSymbolDefinitions::ID = 0; char UnexpectedSymbolDefinitions::ID = 0; char MaterializationTask::ID = 0; +char LookupTask::ID = 0; RegisterDependenciesFunction NoDependenciesToRegister = RegisterDependenciesFunction(); @@ -348,7 +349,7 @@ void ReExportsMaterializationUnit::materialize( } } - // The OnResolveInfo struct will hold the aliases and responsibilty for each + // The OnResolveInfo struct will hold the aliases and responsibility for each // query in the list. struct OnResolveInfo { OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R, @@ -529,11 +530,16 @@ public: SymbolLookupSet LookupSet; SymbolState RequiredState; - std::unique_lock<std::mutex> GeneratorLock; size_t CurSearchOrderIndex = 0; bool NewJITDylib = true; SymbolLookupSet DefGeneratorCandidates; SymbolLookupSet DefGeneratorNonCandidates; + + enum { + NotInGenerator, // Not currently using a generator. + ResumedForGenerator, // Resumed after being auto-suspended before generator. + InGenerator // Currently using generator. + } GenState = NotInGenerator; std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack; }; @@ -547,15 +553,11 @@ public: OnComplete(std::move(OnComplete)) {} void complete(std::unique_ptr<InProgressLookupState> IPLS) override { - GeneratorLock = {}; // Unlock and release. auto &ES = SearchOrder.front().first->getExecutionSession(); ES.OL_completeLookupFlags(std::move(IPLS), std::move(OnComplete)); } - void fail(Error Err) override { - GeneratorLock = {}; // Unlock and release. - OnComplete(std::move(Err)); - } + void fail(Error Err) override { OnComplete(std::move(Err)); } private: unique_function<void(Expected<SymbolFlagsMap>)> OnComplete; @@ -574,14 +576,12 @@ public: } void complete(std::unique_ptr<InProgressLookupState> IPLS) override { - GeneratorLock = {}; // Unlock and release. auto &ES = SearchOrder.front().first->getExecutionSession(); ES.OL_completeLookup(std::move(IPLS), std::move(Q), std::move(RegisterDependencies)); } void fail(Error Err) override { - GeneratorLock = {}; Q->detach(); Q->handleFailed(std::move(Err)); } @@ -638,7 +638,19 @@ void LookupState::continueLookup(Error Err) { ES.OL_applyQueryPhase1(std::move(IPLS), std::move(Err)); } -DefinitionGenerator::~DefinitionGenerator() = default; +DefinitionGenerator::~DefinitionGenerator() { + std::deque<LookupState> LookupsToFail; + { + std::lock_guard<std::mutex> Lock(M); + std::swap(PendingLookups, LookupsToFail); + InUse = false; + } + + for (auto &LS : LookupsToFail) + LS.continueLookup(make_error<StringError>( + "Query waiting on DefinitionGenerator that was destroyed", + inconvertibleErrorCode())); +} JITDylib::~JITDylib() { LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n"); @@ -677,6 +689,10 @@ ResourceTrackerSP JITDylib::createResourceTracker() { } void JITDylib::removeGenerator(DefinitionGenerator &G) { + // DefGenerator moved into TmpDG to ensure that it's destroyed outside the + // session lock (since it may have to send errors to pending queries). + std::shared_ptr<DefinitionGenerator> TmpDG; + ES.runSessionLocked([&] { assert(State == Open && "JD is defunct"); auto I = llvm::find_if(DefGenerators, @@ -684,6 +700,7 @@ void JITDylib::removeGenerator(DefinitionGenerator &G) { return H.get() == &G; }); assert(I != DefGenerators.end() && "Generator not found"); + TmpDG = std::move(*I); DefGenerators.erase(I); }); } @@ -1336,7 +1353,7 @@ void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) { ES.runSessionLocked([&]() { for (auto &KV : NewLinks) { // Skip elements of NewLinks that are already in the link order. - if (llvm::find(LinkOrder, KV) != LinkOrder.end()) + if (llvm::is_contained(LinkOrder, KV)) continue; LinkOrder.push_back(std::move(KV)); @@ -1903,6 +1920,10 @@ void MaterializationTask::printDescription(raw_ostream &OS) { void MaterializationTask::run() { MU->materialize(std::move(MR)); } +void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; } + +void LookupTask::run() { LS.continueLookup(Error::success()); } + ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC) : EPC(std::move(EPC)) { // Associated EPC and this. @@ -1918,16 +1939,14 @@ ExecutionSession::~ExecutionSession() { Error ExecutionSession::endSession() { LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n"); - std::vector<JITDylibSP> JITDylibsToClose = runSessionLocked([&] { + auto JDsToRemove = runSessionLocked([&] { SessionOpen = false; - return std::move(JDs); + return JDs; }); - // TODO: notifiy platform? run static deinits? + std::reverse(JDsToRemove.begin(), JDsToRemove.end()); - Error Err = Error::success(); - for (auto &JD : reverse(JITDylibsToClose)) - Err = joinErrors(std::move(Err), JD->clear()); + auto Err = removeJITDylibs(std::move(JDsToRemove)); Err = joinErrors(std::move(Err), EPC->disconnect()); @@ -1977,42 +1996,44 @@ Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) { return JD; } -Error ExecutionSession::removeJITDylib(JITDylib &JD) { - // Keep JD alive throughout this routine, even if all other references - // have been dropped. - JITDylibSP JDKeepAlive = &JD; +Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) { // Set JD to 'Closing' state and remove JD from the ExecutionSession. runSessionLocked([&] { - assert(JD.State == JITDylib::Open && "JD already closed"); - JD.State = JITDylib::Closing; - auto I = llvm::find(JDs, &JD); - assert(I != JDs.end() && "JD does not appear in session JDs"); - JDs.erase(I); + for (auto &JD : JDsToRemove) { + assert(JD->State == JITDylib::Open && "JD already closed"); + JD->State = JITDylib::Closing; + auto I = llvm::find(JDs, JD); + assert(I != JDs.end() && "JD does not appear in session JDs"); + JDs.erase(I); + } }); - // Clear the JITDylib. Hold on to any error while we clean up the - // JITDylib members below. - auto Err = JD.clear(); - - // Notify the platform of the teardown. - if (P) - Err = joinErrors(std::move(Err), P->teardownJITDylib(JD)); + // Clear JITDylibs and notify the platform. + Error Err = Error::success(); + for (auto JD : JDsToRemove) { + Err = joinErrors(std::move(Err), JD->clear()); + if (P) + Err = joinErrors(std::move(Err), P->teardownJITDylib(*JD)); + } // Set JD to closed state. Clear remaining data structures. runSessionLocked([&] { - assert(JD.State == JITDylib::Closing && "JD should be closing"); - JD.State = JITDylib::Closed; - assert(JD.Symbols.empty() && "JD.Symbols is not empty after clear"); - assert(JD.UnmaterializedInfos.empty() && - "JD.UnmaterializedInfos is not empty after clear"); - assert(JD.MaterializingInfos.empty() && - "JD.MaterializingInfos is not empty after clear"); - assert(JD.TrackerSymbols.empty() && - "TrackerSymbols is not empty after clear"); - JD.DefGenerators.clear(); - JD.LinkOrder.clear(); + for (auto &JD : JDsToRemove) { + assert(JD->State == JITDylib::Closing && "JD should be closing"); + JD->State = JITDylib::Closed; + assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear"); + assert(JD->UnmaterializedInfos.empty() && + "JD.UnmaterializedInfos is not empty after clear"); + assert(JD->MaterializingInfos.empty() && + "JD.MaterializingInfos is not empty after clear"); + assert(JD->TrackerSymbols.empty() && + "TrackerSymbols is not empty after clear"); + JD->DefGenerators.clear(); + JD->LinkOrder.clear(); + } }); + return Err; } @@ -2406,6 +2427,37 @@ Error ExecutionSession::IL_updateCandidatesFor( }); } +void ExecutionSession::OL_resumeLookupAfterGeneration( + InProgressLookupState &IPLS) { + + assert(IPLS.GenState != InProgressLookupState::NotInGenerator && + "Should not be called for not-in-generator lookups"); + IPLS.GenState = InProgressLookupState::NotInGenerator; + + LookupState LS; + + if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) { + IPLS.CurDefGeneratorStack.pop_back(); + std::lock_guard<std::mutex> Lock(DG->M); + + // If there are no pending lookups then mark the generator as free and + // return. + if (DG->PendingLookups.empty()) { + DG->InUse = false; + return; + } + + // Otherwise resume the next lookup. + LS = std::move(DG->PendingLookups.front()); + DG->PendingLookups.pop_front(); + } + + if (LS.IPLS) { + LS.IPLS->GenState = InProgressLookupState::ResumedForGenerator; + dispatchTask(std::make_unique<LookupTask>(std::move(LS))); + } +} + void ExecutionSession::OL_applyQueryPhase1( std::unique_ptr<InProgressLookupState> IPLS, Error Err) { @@ -2422,6 +2474,12 @@ void ExecutionSession::OL_applyQueryPhase1( << IPLS->DefGeneratorNonCandidates << "\n"; }); + if (IPLS->GenState == InProgressLookupState::InGenerator) + OL_resumeLookupAfterGeneration(*IPLS); + + assert(IPLS->GenState != InProgressLookupState::InGenerator && + "Lookup should not be in InGenerator state here"); + // FIXME: We should attach the query as we go: This provides a result in a // single pass in the common case where all symbols have already reached the // required state. The query could be detached again in the 'fail' method on @@ -2447,10 +2505,6 @@ void ExecutionSession::OL_applyQueryPhase1( // If we've just reached a new JITDylib then perform some setup. if (IPLS->NewJITDylib) { - - // Acquire the generator lock for this JITDylib. - IPLS->GeneratorLock = std::unique_lock<std::mutex>(JD.GeneratorsMutex); - // Add any non-candidates from the last JITDylib (if any) back on to the // list of definition candidates for this JITDylib, reset definition // non-candidates to the empty set. @@ -2488,6 +2542,13 @@ void ExecutionSession::OL_applyQueryPhase1( dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates << "\n"; }); + + // If this lookup was resumed after auto-suspension but all candidates + // have already been generated (by some previous call to the generator) + // treat the lookup as if it had completed generation. + if (IPLS->GenState == InProgressLookupState::ResumedForGenerator && + IPLS->DefGeneratorCandidates.empty()) + OL_resumeLookupAfterGeneration(*IPLS); }); // If we encountered an error while filtering generation candidates then @@ -2509,13 +2570,32 @@ void ExecutionSession::OL_applyQueryPhase1( while (!IPLS->CurDefGeneratorStack.empty() && !IPLS->DefGeneratorCandidates.empty()) { auto DG = IPLS->CurDefGeneratorStack.back().lock(); - IPLS->CurDefGeneratorStack.pop_back(); if (!DG) return IPLS->fail(make_error<StringError>( "DefinitionGenerator removed while lookup in progress", inconvertibleErrorCode())); + // At this point the lookup is in either the NotInGenerator state, or in + // the ResumedForGenerator state. + // If this lookup is in the NotInGenerator state then check whether the + // generator is in use. If the generator is not in use then move the + // lookup to the InGenerator state and continue. If the generator is + // already in use then just add this lookup to the pending lookups list + // and bail out. + // If this lookup is in the ResumedForGenerator state then just move it + // to InGenerator and continue. + if (IPLS->GenState == InProgressLookupState::NotInGenerator) { + std::lock_guard<std::mutex> Lock(DG->M); + if (DG->InUse) { + DG->PendingLookups.push_back(std::move(IPLS)); + return; + } + DG->InUse = true; + } + + IPLS->GenState = InProgressLookupState::InGenerator; + auto K = IPLS->K; auto &LookupSet = IPLS->DefGeneratorCandidates; @@ -2528,6 +2608,11 @@ void ExecutionSession::OL_applyQueryPhase1( IPLS = std::move(LS.IPLS); } + // If the lookup returned then pop the generator stack and unblock the + // next lookup on this generator (if any). + if (IPLS) + OL_resumeLookupAfterGeneration(*IPLS); + // If there was an error then fail the query. if (Err) { LLVM_DEBUG({ @@ -2677,7 +2762,7 @@ void ExecutionSession::OL_completeLookup( // Otherwise this is a match. - // If this symbol is already in the requried state then notify the + // If this symbol is already in the required state then notify the // query, remove the symbol and continue. if (SymI->second.getState() >= Q->getRequiredState()) { LLVM_DEBUG(dbgs() diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp deleted file mode 100644 index 830582bb3649..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp +++ /dev/null @@ -1,472 +0,0 @@ -//===------- DebuggerSupportPlugin.cpp - Utils for debugger support -------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h" - -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringSet.h" -#include "llvm/BinaryFormat/MachO.h" - -#define DEBUG_TYPE "orc" - -using namespace llvm; -using namespace llvm::jitlink; -using namespace llvm::orc; - -static const char *SynthDebugSectionName = "__jitlink_synth_debug_object"; - -namespace { - -struct MachO64LE { - using UIntPtr = uint64_t; - - using Header = MachO::mach_header_64; - using SegmentLC = MachO::segment_command_64; - using Section = MachO::section_64; - using NList = MachO::nlist_64; - - static constexpr support::endianness Endianness = support::little; - static constexpr const uint32_t Magic = MachO::MH_MAGIC_64; - static constexpr const uint32_t SegmentCmd = MachO::LC_SEGMENT_64; -}; - -class MachODebugObjectSynthesizerBase - : public GDBJITDebugInfoRegistrationPlugin::DebugSectionSynthesizer { -public: - static bool isDebugSection(Section &Sec) { - return Sec.getName().startswith("__DWARF,"); - } - - MachODebugObjectSynthesizerBase(LinkGraph &G, ExecutorAddr RegisterActionAddr) - : G(G), RegisterActionAddr(RegisterActionAddr) {} - virtual ~MachODebugObjectSynthesizerBase() = default; - - Error preserveDebugSections() { - if (G.findSectionByName(SynthDebugSectionName)) { - LLVM_DEBUG({ - dbgs() << "MachODebugObjectSynthesizer skipping graph " << G.getName() - << " which contains an unexpected existing " - << SynthDebugSectionName << " section.\n"; - }); - return Error::success(); - } - - LLVM_DEBUG({ - dbgs() << "MachODebugObjectSynthesizer visiting graph " << G.getName() - << "\n"; - }); - for (auto &Sec : G.sections()) { - if (!isDebugSection(Sec)) - continue; - // Preserve blocks in this debug section by marking one existing symbol - // live for each block, and introducing a new live, anonymous symbol for - // each currently unreferenced block. - LLVM_DEBUG({ - dbgs() << " Preserving debug section " << Sec.getName() << "\n"; - }); - SmallSet<Block *, 8> PreservedBlocks; - for (auto *Sym : Sec.symbols()) { - bool NewPreservedBlock = - PreservedBlocks.insert(&Sym->getBlock()).second; - if (NewPreservedBlock) - Sym->setLive(true); - } - for (auto *B : Sec.blocks()) - if (!PreservedBlocks.count(B)) - G.addAnonymousSymbol(*B, 0, 0, false, true); - } - return Error::success(); - } - -protected: - LinkGraph &G; - ExecutorAddr RegisterActionAddr; -}; - -template <typename MachOTraits> -class MachODebugObjectSynthesizer : public MachODebugObjectSynthesizerBase { -private: - class MachOStructWriter { - public: - MachOStructWriter(MutableArrayRef<char> Buffer) : Buffer(Buffer) {} - - size_t getOffset() const { return Offset; } - - template <typename MachOStruct> void write(MachOStruct S) { - assert(Offset + sizeof(S) <= Buffer.size() && - "Container block overflow while constructing debug MachO"); - if (MachOTraits::Endianness != support::endian::system_endianness()) - MachO::swapStruct(S); - memcpy(Buffer.data() + Offset, &S, sizeof(S)); - Offset += sizeof(S); - } - - private: - MutableArrayRef<char> Buffer; - size_t Offset = 0; - }; - -public: - using MachODebugObjectSynthesizerBase::MachODebugObjectSynthesizerBase; - - Error startSynthesis() override { - LLVM_DEBUG({ - dbgs() << "Creating " << SynthDebugSectionName << " for " << G.getName() - << "\n"; - }); - auto &SDOSec = G.createSection(SynthDebugSectionName, MemProt::Read); - - struct DebugSectionInfo { - Section *Sec = nullptr; - StringRef SegName; - StringRef SecName; - uint64_t Alignment = 0; - orc::ExecutorAddr StartAddr; - uint64_t Size = 0; - }; - - SmallVector<DebugSectionInfo, 12> DebugSecInfos; - size_t NumSections = 0; - for (auto &Sec : G.sections()) { - if (Sec.blocks().empty()) - continue; - - ++NumSections; - if (isDebugSection(Sec)) { - size_t SepPos = Sec.getName().find(','); - if (SepPos > 16 || (Sec.getName().size() - (SepPos + 1) > 16)) { - LLVM_DEBUG({ - dbgs() << "Skipping debug object synthesis for graph " - << G.getName() - << ": encountered non-standard DWARF section name \"" - << Sec.getName() << "\"\n"; - }); - return Error::success(); - } - DebugSecInfos.push_back({&Sec, Sec.getName().substr(0, SepPos), - Sec.getName().substr(SepPos + 1), 0, - orc::ExecutorAddr(), 0}); - } else { - NonDebugSections.push_back(&Sec); - - // If the first block in the section has a non-zero alignment offset - // then we need to add a padding block, since the section command in - // the header doesn't allow for aligment offsets. - SectionRange R(Sec); - if (!R.empty()) { - auto &FB = *R.getFirstBlock(); - if (FB.getAlignmentOffset() != 0) { - auto Padding = G.allocateBuffer(FB.getAlignmentOffset()); - memset(Padding.data(), 0, Padding.size()); - G.createContentBlock(Sec, Padding, - FB.getAddress() - FB.getAlignmentOffset(), - FB.getAlignment(), 0); - } - } - } - } - - // Create container block. - size_t SectionsCmdSize = - sizeof(typename MachOTraits::Section) * NumSections; - size_t SegmentLCSize = - sizeof(typename MachOTraits::SegmentLC) + SectionsCmdSize; - size_t ContainerBlockSize = - sizeof(typename MachOTraits::Header) + SegmentLCSize; - auto ContainerBlockContent = G.allocateBuffer(ContainerBlockSize); - MachOContainerBlock = &G.createMutableContentBlock( - SDOSec, ContainerBlockContent, orc::ExecutorAddr(), 8, 0); - - // Copy debug section blocks and symbols. - orc::ExecutorAddr NextBlockAddr(MachOContainerBlock->getSize()); - for (auto &SI : DebugSecInfos) { - assert(!SI.Sec->blocks().empty() && "Empty debug info section?"); - - // Update addresses in debug section. - LLVM_DEBUG({ - dbgs() << " Appending " << SI.Sec->getName() << " (" - << SI.Sec->blocks_size() << " block(s)) at " - << formatv("{0:x8}", NextBlockAddr) << "\n"; - }); - for (auto *B : SI.Sec->blocks()) { - NextBlockAddr = alignToBlock(NextBlockAddr, *B); - B->setAddress(NextBlockAddr); - NextBlockAddr += B->getSize(); - } - - auto &FirstBlock = **SI.Sec->blocks().begin(); - if (FirstBlock.getAlignmentOffset() != 0) - return make_error<StringError>( - "First block in " + SI.Sec->getName() + - " section has non-zero alignment offset", - inconvertibleErrorCode()); - if (FirstBlock.getAlignment() > std::numeric_limits<uint32_t>::max()) - return make_error<StringError>("First block in " + SI.Sec->getName() + - " has alignment >4Gb", - inconvertibleErrorCode()); - - SI.Alignment = FirstBlock.getAlignment(); - SI.StartAddr = FirstBlock.getAddress(); - SI.Size = NextBlockAddr - SI.StartAddr; - G.mergeSections(SDOSec, *SI.Sec); - SI.Sec = nullptr; - } - size_t DebugSectionsSize = - NextBlockAddr - orc::ExecutorAddr(MachOContainerBlock->getSize()); - - // Write MachO header and debug section load commands. - MachOStructWriter Writer(MachOContainerBlock->getAlreadyMutableContent()); - typename MachOTraits::Header Hdr; - memset(&Hdr, 0, sizeof(Hdr)); - Hdr.magic = MachOTraits::Magic; - switch (G.getTargetTriple().getArch()) { - case Triple::x86_64: - Hdr.cputype = MachO::CPU_TYPE_X86_64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; - break; - case Triple::aarch64: - Hdr.cputype = MachO::CPU_TYPE_ARM64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; - break; - default: - llvm_unreachable("Unsupported architecture"); - } - Hdr.filetype = MachO::MH_OBJECT; - Hdr.ncmds = 1; - Hdr.sizeofcmds = SegmentLCSize; - Hdr.flags = 0; - Writer.write(Hdr); - - typename MachOTraits::SegmentLC SegLC; - memset(&SegLC, 0, sizeof(SegLC)); - SegLC.cmd = MachOTraits::SegmentCmd; - SegLC.cmdsize = SegmentLCSize; - SegLC.vmaddr = ContainerBlockSize; - SegLC.vmsize = DebugSectionsSize; - SegLC.fileoff = ContainerBlockSize; - SegLC.filesize = DebugSectionsSize; - SegLC.maxprot = - MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE; - SegLC.initprot = - MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE; - SegLC.nsects = NumSections; - SegLC.flags = 0; - Writer.write(SegLC); - - StringSet<> ExistingLongNames; - for (auto &SI : DebugSecInfos) { - typename MachOTraits::Section Sec; - memset(&Sec, 0, sizeof(Sec)); - memcpy(Sec.sectname, SI.SecName.data(), SI.SecName.size()); - memcpy(Sec.segname, SI.SegName.data(), SI.SegName.size()); - Sec.addr = SI.StartAddr.getValue(); - Sec.size = SI.Size; - Sec.offset = SI.StartAddr.getValue(); - Sec.align = SI.Alignment; - Sec.reloff = 0; - Sec.nreloc = 0; - Sec.flags = MachO::S_ATTR_DEBUG; - Writer.write(Sec); - } - - // Set MachOContainerBlock to indicate success to - // completeSynthesisAndRegister. - NonDebugSectionsStart = Writer.getOffset(); - return Error::success(); - } - - Error completeSynthesisAndRegister() override { - if (!MachOContainerBlock) { - LLVM_DEBUG({ - dbgs() << "Not writing MachO debug object header for " << G.getName() - << " since createDebugSection failed\n"; - }); - return Error::success(); - } - - LLVM_DEBUG({ - dbgs() << "Writing MachO debug object header for " << G.getName() << "\n"; - }); - - MachOStructWriter Writer( - MachOContainerBlock->getAlreadyMutableContent().drop_front( - NonDebugSectionsStart)); - - unsigned LongSectionNameIdx = 0; - for (auto *Sec : NonDebugSections) { - size_t SepPos = Sec->getName().find(','); - StringRef SegName, SecName; - std::string CustomSecName; - - if ((SepPos == StringRef::npos && Sec->getName().size() <= 16)) { - // No embedded segment name, short section name. - SegName = "__JITLINK_CUSTOM"; - SecName = Sec->getName(); - } else if (SepPos < 16 && (Sec->getName().size() - (SepPos + 1) <= 16)) { - // Canonical embedded segment and section name. - SegName = Sec->getName().substr(0, SepPos); - SecName = Sec->getName().substr(SepPos + 1); - } else { - // Long section name that needs to be truncated. - assert(Sec->getName().size() > 16 && - "Short section name should have been handled above"); - SegName = "__JITLINK_CUSTOM"; - auto IdxStr = std::to_string(++LongSectionNameIdx); - CustomSecName = Sec->getName().substr(0, 15 - IdxStr.size()).str(); - CustomSecName += "."; - CustomSecName += IdxStr; - SecName = StringRef(CustomSecName.data(), 16); - } - - SectionRange R(*Sec); - if (R.getFirstBlock()->getAlignmentOffset() != 0) - return make_error<StringError>( - "While building MachO debug object for " + G.getName() + - " first block has non-zero alignment offset", - inconvertibleErrorCode()); - - typename MachOTraits::Section SecCmd; - memset(&SecCmd, 0, sizeof(SecCmd)); - memcpy(SecCmd.sectname, SecName.data(), SecName.size()); - memcpy(SecCmd.segname, SegName.data(), SegName.size()); - SecCmd.addr = R.getStart().getValue(); - SecCmd.size = R.getSize(); - SecCmd.offset = 0; - SecCmd.align = R.getFirstBlock()->getAlignment(); - SecCmd.reloff = 0; - SecCmd.nreloc = 0; - SecCmd.flags = 0; - Writer.write(SecCmd); - } - - static constexpr bool AutoRegisterCode = true; - SectionRange R(MachOContainerBlock->getSection()); - G.allocActions().push_back( - {cantFail(shared::WrapperFunctionCall::Create< - shared::SPSArgList<shared::SPSExecutorAddrRange, bool>>( - RegisterActionAddr, R.getRange(), AutoRegisterCode)), - {}}); - return Error::success(); - } - -private: - Block *MachOContainerBlock = nullptr; - SmallVector<Section *, 16> NonDebugSections; - size_t NonDebugSectionsStart = 0; -}; - -} // end anonymous namespace - -namespace llvm { -namespace orc { - -Expected<std::unique_ptr<GDBJITDebugInfoRegistrationPlugin>> -GDBJITDebugInfoRegistrationPlugin::Create(ExecutionSession &ES, - JITDylib &ProcessJD, - const Triple &TT) { - auto RegisterActionAddr = - TT.isOSBinFormatMachO() - ? ES.intern("_llvm_orc_registerJITLoaderGDBAllocAction") - : ES.intern("llvm_orc_registerJITLoaderGDBAllocAction"); - - if (auto RegisterSym = ES.lookup({&ProcessJD}, RegisterActionAddr)) - return std::make_unique<GDBJITDebugInfoRegistrationPlugin>( - RegisterSym->getAddress()); - else - return RegisterSym.takeError(); -} - -Error GDBJITDebugInfoRegistrationPlugin::notifyFailed( - MaterializationResponsibility &MR) { - return Error::success(); -} - -Error GDBJITDebugInfoRegistrationPlugin::notifyRemovingResources( - JITDylib &JD, ResourceKey K) { - return Error::success(); -} - -void GDBJITDebugInfoRegistrationPlugin::notifyTransferringResources( - JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) {} - -void GDBJITDebugInfoRegistrationPlugin::modifyPassConfig( - MaterializationResponsibility &MR, LinkGraph &LG, - PassConfiguration &PassConfig) { - - if (LG.getTargetTriple().getObjectFormat() == Triple::MachO) - modifyPassConfigForMachO(MR, LG, PassConfig); - else { - LLVM_DEBUG({ - dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unspported graph " - << LG.getName() << "(triple = " << LG.getTargetTriple().str() - << "\n"; - }); - } -} - -void GDBJITDebugInfoRegistrationPlugin::modifyPassConfigForMachO( - MaterializationResponsibility &MR, jitlink::LinkGraph &LG, - jitlink::PassConfiguration &PassConfig) { - - switch (LG.getTargetTriple().getArch()) { - case Triple::x86_64: - case Triple::aarch64: - // Supported, continue. - assert(LG.getPointerSize() == 8 && "Graph has incorrect pointer size"); - assert(LG.getEndianness() == support::little && - "Graph has incorrect endianness"); - break; - default: - // Unsupported. - LLVM_DEBUG({ - dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unsupported " - << "MachO graph " << LG.getName() - << "(triple = " << LG.getTargetTriple().str() - << ", pointer size = " << LG.getPointerSize() << ", endianness = " - << (LG.getEndianness() == support::big ? "big" : "little") - << ")\n"; - }); - return; - } - - // Scan for debug sections. If we find one then install passes. - bool HasDebugSections = false; - for (auto &Sec : LG.sections()) - if (MachODebugObjectSynthesizerBase::isDebugSection(Sec)) { - HasDebugSections = true; - break; - } - - if (HasDebugSections) { - LLVM_DEBUG({ - dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() - << " contains debug info. Installing debugger support passes.\n"; - }); - - auto MDOS = std::make_shared<MachODebugObjectSynthesizer<MachO64LE>>( - LG, RegisterActionAddr); - PassConfig.PrePrunePasses.push_back( - [=](LinkGraph &G) { return MDOS->preserveDebugSections(); }); - PassConfig.PostPrunePasses.push_back( - [=](LinkGraph &G) { return MDOS->startSynthesis(); }); - PassConfig.PreFixupPasses.push_back( - [=](LinkGraph &G) { return MDOS->completeSynthesisAndRegister(); }); - } else { - LLVM_DEBUG({ - dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() - << " contains no debug info. Skipping.\n"; - }); - } -} - -} // namespace orc -} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp new file mode 100644 index 000000000000..f65ec27ff875 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp @@ -0,0 +1,121 @@ +//===--- DebugInfoSupport.cpp -- Utils for debug info support ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities to preserve and parse debug info from LinkGraphs. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" + +#include "llvm/Support/SmallVectorMemoryBuffer.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::jitlink; + +namespace { +static DenseSet<StringRef> DWARFSectionNames = { +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ + StringRef(ELF_NAME), +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION +}; + +// We might be able to drop relocations to symbols that do end up +// being pruned by the linker, but for now we just preserve all +static void preserveDWARFSection(LinkGraph &G, Section &Sec) { + DenseMap<Block *, Symbol *> Preserved; + for (auto Sym : Sec.symbols()) { + if (Sym->isLive()) + Preserved[&Sym->getBlock()] = Sym; + else if (!Preserved.count(&Sym->getBlock())) + Preserved[&Sym->getBlock()] = Sym; + } + for (auto Block : Sec.blocks()) { + auto &PSym = Preserved[Block]; + if (!PSym) + PSym = &G.addAnonymousSymbol(*Block, 0, 0, false, true); + else if (!PSym->isLive()) + PSym->setLive(true); + } +} + +static SmallVector<char, 0> getSectionData(Section &Sec) { + SmallVector<char, 0> SecData; + SmallVector<Block *, 8> SecBlocks(Sec.blocks().begin(), Sec.blocks().end()); + std::sort(SecBlocks.begin(), SecBlocks.end(), [](Block *LHS, Block *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + // Convert back to what object file would have, one blob of section content + // Assumes all zerofill + // TODO handle alignment? + // TODO handle alignment offset? + for (auto *Block : SecBlocks) { + if (Block->isZeroFill()) + SecData.resize(SecData.size() + Block->getSize(), 0); + else + SecData.append(Block->getContent().begin(), Block->getContent().end()); + } + return SecData; +} + +static void dumpDWARFContext(DWARFContext &DC) { + auto options = llvm::DIDumpOptions(); + options.DumpType &= ~DIDT_UUID; + options.DumpType &= ~(1 << DIDT_ID_DebugFrame); + LLVM_DEBUG(DC.dump(dbgs(), options)); +} + +} // namespace + +Error llvm::orc::preserveDebugSections(LinkGraph &G) { + if (!G.getTargetTriple().isOSBinFormatELF()) { + return make_error<StringError>( + "preserveDebugSections only supports ELF LinkGraphs!", + inconvertibleErrorCode()); + } + for (auto &Sec : G.sections()) { + if (DWARFSectionNames.count(Sec.getName())) { + LLVM_DEBUG(dbgs() << "Preserving DWARF section " << Sec.getName() + << "\n"); + preserveDWARFSection(G, Sec); + } + } + return Error::success(); +} + +Expected<std::pair<std::unique_ptr<DWARFContext>, + StringMap<std::unique_ptr<MemoryBuffer>>>> +llvm::orc::createDWARFContext(LinkGraph &G) { + if (!G.getTargetTriple().isOSBinFormatELF()) { + return make_error<StringError>( + "createDWARFContext only supports ELF LinkGraphs!", + inconvertibleErrorCode()); + } + StringMap<std::unique_ptr<MemoryBuffer>> DWARFSectionData; + for (auto &Sec : G.sections()) { + if (DWARFSectionNames.count(Sec.getName())) { + auto SecData = getSectionData(Sec); + auto Name = Sec.getName(); + // DWARFContext expects the section name to not start with a dot + if (Name.starts_with(".")) + Name = Name.drop_front(); + LLVM_DEBUG(dbgs() << "Creating DWARFContext section " << Name + << " with size " << SecData.size() << "\n"); + DWARFSectionData[Name] = + std::make_unique<SmallVectorMemoryBuffer>(std::move(SecData)); + } + } + auto Ctx = + DWARFContext::create(DWARFSectionData, G.getPointerSize(), + G.getEndianness() == llvm::endianness::little); + dumpDWARFContext(*Ctx); + return std::make_pair(std::move(Ctx), std::move(DWARFSectionData)); +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp new file mode 100644 index 000000000000..1668473c0eb4 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp @@ -0,0 +1,61 @@ +//===------ DebuggerSupport.cpp - Utils for enabling debugger support -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" +#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; + +namespace llvm::orc { + +Error enableDebuggerSupport(LLJIT &J) { + auto *ObjLinkingLayer = dyn_cast<ObjectLinkingLayer>(&J.getObjLinkingLayer()); + if (!ObjLinkingLayer) + return make_error<StringError>("Cannot enable LLJIT debugger support: " + "Debugger support requires JITLink", + inconvertibleErrorCode()); + auto ProcessSymsJD = J.getProcessSymbolsJITDylib(); + if (!ProcessSymsJD) + return make_error<StringError>("Cannot enable LLJIT debugger support: " + "Process symbols are not available", + inconvertibleErrorCode()); + + auto &ES = J.getExecutionSession(); + const auto &TT = J.getTargetTriple(); + + switch (TT.getObjectFormat()) { + case Triple::ELF: { + auto Registrar = createJITLoaderGDBRegistrar(ES); + if (!Registrar) + return Registrar.takeError(); + ObjLinkingLayer->addPlugin(std::make_unique<DebugObjectManagerPlugin>( + ES, std::move(*Registrar), false, true)); + return Error::success(); + } + case Triple::MachO: { + auto DS = GDBJITDebugInfoRegistrationPlugin::Create(ES, *ProcessSymsJD, TT); + if (!DS) + return DS.takeError(); + ObjLinkingLayer->addPlugin(std::move(*DS)); + return Error::success(); + } + default: + return make_error<StringError>( + "Cannot enable LLJIT debugger support: " + + Triple::getObjectFormatTypeName(TT.getObjectFormat()) + + " is not supported", + inconvertibleErrorCode()); + } +} + +} // namespace llvm::orc diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp new file mode 100644 index 000000000000..e387b06ee934 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp @@ -0,0 +1,423 @@ +//===------- DebuggerSupportPlugin.cpp - Utils for debugger support -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h" +#include "llvm/ExecutionEngine/Orc/MachOBuilder.h" + +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h" + +#include <chrono> + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +static const char *SynthDebugSectionName = "__jitlink_synth_debug_object"; + +namespace { + +class MachODebugObjectSynthesizerBase + : public GDBJITDebugInfoRegistrationPlugin::DebugSectionSynthesizer { +public: + static bool isDebugSection(Section &Sec) { + return Sec.getName().starts_with("__DWARF,"); + } + + MachODebugObjectSynthesizerBase(LinkGraph &G, ExecutorAddr RegisterActionAddr) + : G(G), RegisterActionAddr(RegisterActionAddr) {} + virtual ~MachODebugObjectSynthesizerBase() = default; + + Error preserveDebugSections() { + if (G.findSectionByName(SynthDebugSectionName)) { + LLVM_DEBUG({ + dbgs() << "MachODebugObjectSynthesizer skipping graph " << G.getName() + << " which contains an unexpected existing " + << SynthDebugSectionName << " section.\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "MachODebugObjectSynthesizer visiting graph " << G.getName() + << "\n"; + }); + for (auto &Sec : G.sections()) { + if (!isDebugSection(Sec)) + continue; + // Preserve blocks in this debug section by marking one existing symbol + // live for each block, and introducing a new live, anonymous symbol for + // each currently unreferenced block. + LLVM_DEBUG({ + dbgs() << " Preserving debug section " << Sec.getName() << "\n"; + }); + SmallSet<Block *, 8> PreservedBlocks; + for (auto *Sym : Sec.symbols()) { + bool NewPreservedBlock = + PreservedBlocks.insert(&Sym->getBlock()).second; + if (NewPreservedBlock) + Sym->setLive(true); + } + for (auto *B : Sec.blocks()) + if (!PreservedBlocks.count(B)) + G.addAnonymousSymbol(*B, 0, 0, false, true); + } + + return Error::success(); + } + +protected: + LinkGraph &G; + ExecutorAddr RegisterActionAddr; +}; + +template <typename MachOTraits> +class MachODebugObjectSynthesizer : public MachODebugObjectSynthesizerBase { +public: + MachODebugObjectSynthesizer(ExecutionSession &ES, LinkGraph &G, + ExecutorAddr RegisterActionAddr) + : MachODebugObjectSynthesizerBase(G, RegisterActionAddr), + Builder(ES.getPageSize()) {} + + using MachODebugObjectSynthesizerBase::MachODebugObjectSynthesizerBase; + + Error startSynthesis() override { + LLVM_DEBUG({ + dbgs() << "Creating " << SynthDebugSectionName << " for " << G.getName() + << "\n"; + }); + + for (auto &Sec : G.sections()) { + if (Sec.blocks().empty()) + continue; + + // Skip sections whose name's don't fit the MachO standard. + if (Sec.getName().empty() || Sec.getName().size() > 33 || + Sec.getName().find(',') > 16) + continue; + + if (isDebugSection(Sec)) + DebugSections.push_back({&Sec, nullptr}); + else if (Sec.getMemLifetime() != MemLifetime::NoAlloc) + NonDebugSections.push_back({&Sec, nullptr}); + } + + // Bail out early if no debug sections. + if (DebugSections.empty()) + return Error::success(); + + // Write MachO header and debug section load commands. + Builder.Header.filetype = MachO::MH_OBJECT; + switch (G.getTargetTriple().getArch()) { + case Triple::x86_64: + Builder.Header.cputype = MachO::CPU_TYPE_X86_64; + Builder.Header.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; + break; + case Triple::aarch64: + Builder.Header.cputype = MachO::CPU_TYPE_ARM64; + Builder.Header.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; + break; + default: + llvm_unreachable("Unsupported architecture"); + } + + Seg = &Builder.addSegment(""); + + StringMap<std::unique_ptr<MemoryBuffer>> DebugSectionMap; + StringRef DebugLineSectionData; + for (auto &DSec : DebugSections) { + auto [SegName, SecName] = DSec.GraphSec->getName().split(','); + DSec.BuilderSec = &Seg->addSection(SecName, SegName); + + SectionRange SR(*DSec.GraphSec); + DSec.BuilderSec->Content.Size = SR.getSize(); + if (!SR.empty()) { + DSec.BuilderSec->align = Log2_64(SR.getFirstBlock()->getAlignment()); + StringRef SectionData(SR.getFirstBlock()->getContent().data(), + SR.getFirstBlock()->getSize()); + DebugSectionMap[SecName] = + MemoryBuffer::getMemBuffer(SectionData, G.getName(), false); + if (SecName == "__debug_line") + DebugLineSectionData = SectionData; + } + } + + std::optional<StringRef> FileName; + if (!DebugLineSectionData.empty()) { + assert((G.getEndianness() == llvm::endianness::big || + G.getEndianness() == llvm::endianness::little) && + "G.getEndianness() must be either big or little"); + auto DWARFCtx = + DWARFContext::create(DebugSectionMap, G.getPointerSize(), + G.getEndianness() == llvm::endianness::little); + DWARFDataExtractor DebugLineData( + DebugLineSectionData, G.getEndianness() == llvm::endianness::little, + G.getPointerSize()); + uint64_t Offset = 0; + DWARFDebugLine::LineTable LineTable; + + // Try to parse line data. Consume error on failure. + if (auto Err = LineTable.parse(DebugLineData, &Offset, *DWARFCtx, nullptr, + consumeError)) { + handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { + LLVM_DEBUG({ + dbgs() << "Cannot parse line table for \"" << G.getName() << "\": "; + EIB.log(dbgs()); + dbgs() << "\n"; + }); + }); + } else { + if (!LineTable.Prologue.FileNames.empty()) + FileName = *dwarf::toString(LineTable.Prologue.FileNames[0].Name); + } + } + + // If no line table (or unable to use) then use graph name. + // FIXME: There are probably other debug sections we should look in first. + if (!FileName) + FileName = StringRef(G.getName()); + + Builder.addSymbol("", MachO::N_SO, 0, 0, 0); + Builder.addSymbol(*FileName, MachO::N_SO, 0, 0, 0); + auto TimeStamp = std::chrono::duration_cast<std::chrono::seconds>( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + Builder.addSymbol("", MachO::N_OSO, 3, 1, TimeStamp); + + for (auto &NDSP : NonDebugSections) { + auto [SegName, SecName] = NDSP.GraphSec->getName().split(','); + NDSP.BuilderSec = &Seg->addSection(SecName, SegName); + SectionRange SR(*NDSP.GraphSec); + if (!SR.empty()) + NDSP.BuilderSec->align = Log2_64(SR.getFirstBlock()->getAlignment()); + + // Add stabs. + for (auto *Sym : NDSP.GraphSec->symbols()) { + // Skip anonymous symbols. + if (!Sym->hasName()) + continue; + + uint8_t SymType = Sym->isCallable() ? MachO::N_FUN : MachO::N_GSYM; + + Builder.addSymbol("", MachO::N_BNSYM, 1, 0, 0); + StabSymbols.push_back( + {*Sym, Builder.addSymbol(Sym->getName(), SymType, 1, 0, 0), + Builder.addSymbol(Sym->getName(), SymType, 0, 0, 0)}); + Builder.addSymbol("", MachO::N_ENSYM, 1, 0, 0); + } + } + + Builder.addSymbol("", MachO::N_SO, 1, 0, 0); + + // Lay out the debug object, create a section and block for it. + size_t DebugObjectSize = Builder.layout(); + + auto &SDOSec = G.createSection(SynthDebugSectionName, MemProt::Read); + MachOContainerBlock = &G.createMutableContentBlock( + SDOSec, G.allocateBuffer(DebugObjectSize), orc::ExecutorAddr(), 8, 0); + + return Error::success(); + } + + Error completeSynthesisAndRegister() override { + if (!MachOContainerBlock) { + LLVM_DEBUG({ + dbgs() << "Not writing MachO debug object header for " << G.getName() + << " since createDebugSection failed\n"; + }); + + return Error::success(); + } + ExecutorAddr MaxAddr; + for (auto &NDSec : NonDebugSections) { + SectionRange SR(*NDSec.GraphSec); + NDSec.BuilderSec->addr = SR.getStart().getValue(); + NDSec.BuilderSec->size = SR.getSize(); + NDSec.BuilderSec->offset = SR.getStart().getValue(); + if (SR.getEnd() > MaxAddr) + MaxAddr = SR.getEnd(); + } + + for (auto &DSec : DebugSections) { + if (DSec.GraphSec->blocks_size() != 1) + return make_error<StringError>( + "Unexpected number of blocks in debug info section", + inconvertibleErrorCode()); + + if (ExecutorAddr(DSec.BuilderSec->addr) + DSec.BuilderSec->size > MaxAddr) + MaxAddr = ExecutorAddr(DSec.BuilderSec->addr) + DSec.BuilderSec->size; + + auto &B = **DSec.GraphSec->blocks().begin(); + DSec.BuilderSec->Content.Data = B.getContent().data(); + DSec.BuilderSec->Content.Size = B.getContent().size(); + DSec.BuilderSec->flags |= MachO::S_ATTR_DEBUG; + } + + LLVM_DEBUG({ + dbgs() << "Writing MachO debug object header for " << G.getName() << "\n"; + }); + + // Update stab symbol addresses. + for (auto &SS : StabSymbols) { + SS.StartStab.nlist().n_value = SS.Sym.getAddress().getValue(); + SS.EndStab.nlist().n_value = SS.Sym.getSize(); + } + + Builder.write(MachOContainerBlock->getAlreadyMutableContent()); + + static constexpr bool AutoRegisterCode = true; + SectionRange R(MachOContainerBlock->getSection()); + G.allocActions().push_back( + {cantFail(shared::WrapperFunctionCall::Create< + shared::SPSArgList<shared::SPSExecutorAddrRange, bool>>( + RegisterActionAddr, R.getRange(), AutoRegisterCode)), + {}}); + + return Error::success(); + } + +private: + struct SectionPair { + Section *GraphSec = nullptr; + typename MachOBuilder<MachOTraits>::Section *BuilderSec = nullptr; + }; + + struct StabSymbolsEntry { + using RelocTarget = typename MachOBuilder<MachOTraits>::RelocTarget; + + StabSymbolsEntry(Symbol &Sym, RelocTarget StartStab, RelocTarget EndStab) + : Sym(Sym), StartStab(StartStab), EndStab(EndStab) {} + + Symbol &Sym; + RelocTarget StartStab, EndStab; + }; + + using BuilderType = MachOBuilder<MachOTraits>; + + Block *MachOContainerBlock = nullptr; + MachOBuilder<MachOTraits> Builder; + typename MachOBuilder<MachOTraits>::Segment *Seg = nullptr; + std::vector<StabSymbolsEntry> StabSymbols; + SmallVector<SectionPair, 16> DebugSections; + SmallVector<SectionPair, 16> NonDebugSections; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<GDBJITDebugInfoRegistrationPlugin>> +GDBJITDebugInfoRegistrationPlugin::Create(ExecutionSession &ES, + JITDylib &ProcessJD, + const Triple &TT) { + auto RegisterActionAddr = + TT.isOSBinFormatMachO() + ? ES.intern("_llvm_orc_registerJITLoaderGDBAllocAction") + : ES.intern("llvm_orc_registerJITLoaderGDBAllocAction"); + + if (auto RegisterSym = ES.lookup({&ProcessJD}, RegisterActionAddr)) + return std::make_unique<GDBJITDebugInfoRegistrationPlugin>( + RegisterSym->getAddress()); + else + return RegisterSym.takeError(); +} + +Error GDBJITDebugInfoRegistrationPlugin::notifyFailed( + MaterializationResponsibility &MR) { + return Error::success(); +} + +Error GDBJITDebugInfoRegistrationPlugin::notifyRemovingResources( + JITDylib &JD, ResourceKey K) { + return Error::success(); +} + +void GDBJITDebugInfoRegistrationPlugin::notifyTransferringResources( + JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) {} + +void GDBJITDebugInfoRegistrationPlugin::modifyPassConfig( + MaterializationResponsibility &MR, LinkGraph &LG, + PassConfiguration &PassConfig) { + + if (LG.getTargetTriple().getObjectFormat() == Triple::MachO) + modifyPassConfigForMachO(MR, LG, PassConfig); + else { + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unspported graph " + << LG.getName() << "(triple = " << LG.getTargetTriple().str() + << "\n"; + }); + } +} + +void GDBJITDebugInfoRegistrationPlugin::modifyPassConfigForMachO( + MaterializationResponsibility &MR, jitlink::LinkGraph &LG, + jitlink::PassConfiguration &PassConfig) { + + switch (LG.getTargetTriple().getArch()) { + case Triple::x86_64: + case Triple::aarch64: + // Supported, continue. + assert(LG.getPointerSize() == 8 && "Graph has incorrect pointer size"); + assert(LG.getEndianness() == llvm::endianness::little && + "Graph has incorrect endianness"); + break; + default: + // Unsupported. + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unsupported " + << "MachO graph " << LG.getName() + << "(triple = " << LG.getTargetTriple().str() + << ", pointer size = " << LG.getPointerSize() << ", endianness = " + << (LG.getEndianness() == llvm::endianness::big ? "big" : "little") + << ")\n"; + }); + return; + } + + // Scan for debug sections. If we find one then install passes. + bool HasDebugSections = false; + for (auto &Sec : LG.sections()) + if (MachODebugObjectSynthesizerBase::isDebugSection(Sec)) { + HasDebugSections = true; + break; + } + + if (HasDebugSections) { + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() + << " contains debug info. Installing debugger support passes.\n"; + }); + + auto MDOS = std::make_shared<MachODebugObjectSynthesizer<MachO64LE>>( + MR.getTargetJITDylib().getExecutionSession(), LG, RegisterActionAddr); + PassConfig.PrePrunePasses.push_back( + [=](LinkGraph &G) { return MDOS->preserveDebugSections(); }); + PassConfig.PostPrunePasses.push_back( + [=](LinkGraph &G) { return MDOS->startSynthesis(); }); + PassConfig.PostFixupPasses.push_back( + [=](LinkGraph &G) { return MDOS->completeSynthesisAndRegister(); }); + } else { + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() + << " contains no debug info. Skipping.\n"; + }); + } +} + +} // namespace orc +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp new file mode 100644 index 000000000000..2df5aef733fb --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp @@ -0,0 +1,22 @@ +//===--------- LLJITUtilsCBindings.cpp - Advanced LLJIT features ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm-c/LLJIT.h" +#include "llvm-c/LLJITUtils.h" + +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" + +using namespace llvm; +using namespace llvm::orc; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJIT, LLVMOrcLLJITRef) + +LLVMErrorRef LLVMOrcLLJITEnableDebugSupport(LLVMOrcLLJITRef J) { + return wrap(llvm::orc::enableDebuggerSupport(*unwrap(J))); +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp new file mode 100644 index 000000000000..fffecfc97814 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp @@ -0,0 +1,303 @@ +//===----- PerfSupportPlugin.cpp --- Utils for perf support -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Handles support for registering code with perf +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h" + +#include "llvm/ExecutionEngine/JITLink/x86_64.h" +#include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" +#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::jitlink; + +namespace { + +// Creates an EH frame header prepared for a 32-bit relative relocation +// to the start of the .eh_frame section. Absolute injects a 64-bit absolute +// address space offset 4 bytes from the start instead of 4 bytes +Expected<std::string> createX64EHFrameHeader(Section &EHFrame, + llvm::endianness endianness, + bool absolute) { + uint8_t Version = 1; + uint8_t EhFramePtrEnc = 0; + if (absolute) { + EhFramePtrEnc |= dwarf::DW_EH_PE_sdata8 | dwarf::DW_EH_PE_absptr; + } else { + EhFramePtrEnc |= dwarf::DW_EH_PE_sdata4 | dwarf::DW_EH_PE_datarel; + } + uint8_t FDECountEnc = dwarf::DW_EH_PE_omit; + uint8_t TableEnc = dwarf::DW_EH_PE_omit; + // X86_64_64 relocation to the start of the .eh_frame section + uint32_t EHFrameRelocation = 0; + // uint32_t FDECount = 0; + // Skip the FDE binary search table + // We'd have to reprocess the CIEs to get this information, + // which seems like more trouble than it's worth + // TODO consider implementing this. + // binary search table goes here + + size_t HeaderSize = + (sizeof(Version) + sizeof(EhFramePtrEnc) + sizeof(FDECountEnc) + + sizeof(TableEnc) + + (absolute ? sizeof(uint64_t) : sizeof(EHFrameRelocation))); + std::string HeaderContent(HeaderSize, '\0'); + BinaryStreamWriter Writer( + MutableArrayRef<uint8_t>( + reinterpret_cast<uint8_t *>(HeaderContent.data()), HeaderSize), + endianness); + if (auto Err = Writer.writeInteger(Version)) + return std::move(Err); + if (auto Err = Writer.writeInteger(EhFramePtrEnc)) + return std::move(Err); + if (auto Err = Writer.writeInteger(FDECountEnc)) + return std::move(Err); + if (auto Err = Writer.writeInteger(TableEnc)) + return std::move(Err); + if (absolute) { + uint64_t EHFrameAddr = SectionRange(EHFrame).getStart().getValue(); + if (auto Err = Writer.writeInteger(EHFrameAddr)) + return std::move(Err); + } else { + if (auto Err = Writer.writeInteger(EHFrameRelocation)) + return std::move(Err); + } + return HeaderContent; +} + +constexpr StringRef RegisterPerfStartSymbolName = + "llvm_orc_registerJITLoaderPerfStart"; +constexpr StringRef RegisterPerfEndSymbolName = + "llvm_orc_registerJITLoaderPerfEnd"; +constexpr StringRef RegisterPerfImplSymbolName = + "llvm_orc_registerJITLoaderPerfImpl"; + +static PerfJITCodeLoadRecord +getCodeLoadRecord(const Symbol &Sym, std::atomic<uint64_t> &CodeIndex) { + PerfJITCodeLoadRecord Record; + auto Name = Sym.getName(); + auto Addr = Sym.getAddress(); + auto Size = Sym.getSize(); + Record.Prefix.Id = PerfJITRecordType::JIT_CODE_LOAD; + // Runtime sets PID + Record.Pid = 0; + // Runtime sets TID + Record.Tid = 0; + Record.Vma = Addr.getValue(); + Record.CodeAddr = Addr.getValue(); + Record.CodeSize = Size; + Record.CodeIndex = CodeIndex++; + Record.Name = Name.str(); + // Initialize last, once all the other fields are filled + Record.Prefix.TotalSize = + (2 * sizeof(uint32_t) // id, total_size + + sizeof(uint64_t) // timestamp + + 2 * sizeof(uint32_t) // pid, tid + + 4 * sizeof(uint64_t) // vma, code_addr, code_size, code_index + + Name.size() + 1 // symbol name + + Record.CodeSize // code + ); + return Record; +} + +static std::optional<PerfJITDebugInfoRecord> +getDebugInfoRecord(const Symbol &Sym, DWARFContext &DC) { + auto &Section = Sym.getBlock().getSection(); + auto Addr = Sym.getAddress(); + auto Size = Sym.getSize(); + auto SAddr = object::SectionedAddress{Addr.getValue(), Section.getOrdinal()}; + LLVM_DEBUG(dbgs() << "Getting debug info for symbol " << Sym.getName() + << " at address " << Addr.getValue() << " with size " + << Size << "\n" + << "Section ordinal: " << Section.getOrdinal() << "\n"); + auto LInfo = DC.getLineInfoForAddressRange( + SAddr, Size, DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath); + if (LInfo.empty()) { + // No line info available + LLVM_DEBUG(dbgs() << "No line info available\n"); + return std::nullopt; + } + PerfJITDebugInfoRecord Record; + Record.Prefix.Id = PerfJITRecordType::JIT_CODE_DEBUG_INFO; + Record.CodeAddr = Addr.getValue(); + for (const auto &Entry : LInfo) { + auto Addr = Entry.first; + // The function re-created by perf is preceded by a elf + // header. Need to adjust for that, otherwise the results are + // wrong. + Addr += 0x40; + Record.Entries.push_back({Addr, Entry.second.Line, + Entry.second.Discriminator, + Entry.second.FileName}); + } + size_t EntriesBytes = (2 // record header + + 2 // record fields + ) * + sizeof(uint64_t); + for (const auto &Entry : Record.Entries) { + EntriesBytes += + sizeof(uint64_t) + 2 * sizeof(uint32_t); // Addr, Line/Discrim + EntriesBytes += Entry.Name.size() + 1; // Name + } + Record.Prefix.TotalSize = EntriesBytes; + LLVM_DEBUG(dbgs() << "Created debug info record\n" + << "Total size: " << Record.Prefix.TotalSize << "\n" + << "Nr entries: " << Record.Entries.size() << "\n"); + return Record; +} + +static Expected<PerfJITCodeUnwindingInfoRecord> +getUnwindingRecord(LinkGraph &G) { + PerfJITCodeUnwindingInfoRecord Record; + Record.Prefix.Id = PerfJITRecordType::JIT_CODE_UNWINDING_INFO; + Record.Prefix.TotalSize = 0; + auto Eh_frame = G.findSectionByName(".eh_frame"); + if (!Eh_frame) { + LLVM_DEBUG(dbgs() << "No .eh_frame section found\n"); + return Record; + } + if (!G.getTargetTriple().isOSBinFormatELF()) { + LLVM_DEBUG(dbgs() << "Not an ELF file, will not emit unwinding info\n"); + return Record; + } + auto SR = SectionRange(*Eh_frame); + auto EHFrameSize = SR.getSize(); + auto Eh_frame_hdr = G.findSectionByName(".eh_frame_hdr"); + if (!Eh_frame_hdr) { + if (G.getTargetTriple().getArch() == Triple::x86_64) { + auto Hdr = createX64EHFrameHeader(*Eh_frame, G.getEndianness(), true); + if (!Hdr) + return Hdr.takeError(); + Record.EHFrameHdr = std::move(*Hdr); + } else { + LLVM_DEBUG(dbgs() << "No .eh_frame_hdr section found\n"); + return Record; + } + Record.EHFrameHdrAddr = 0; + Record.EHFrameHdrSize = Record.EHFrameHdr.size(); + Record.UnwindDataSize = EHFrameSize + Record.EHFrameHdrSize; + Record.MappedSize = 0; // Because the EHFrame header was not mapped + } else { + auto SR = SectionRange(*Eh_frame_hdr); + Record.EHFrameHdrAddr = SR.getStart().getValue(); + Record.EHFrameHdrSize = SR.getSize(); + Record.UnwindDataSize = EHFrameSize + Record.EHFrameHdrSize; + Record.MappedSize = Record.UnwindDataSize; + } + Record.EHFrameAddr = SR.getStart().getValue(); + Record.Prefix.TotalSize = + (2 * sizeof(uint32_t) // id, total_size + + sizeof(uint64_t) // timestamp + + + 3 * sizeof(uint64_t) // unwind_data_size, eh_frame_hdr_size, mapped_size + + Record.UnwindDataSize // eh_frame_hdr, eh_frame + ); + LLVM_DEBUG(dbgs() << "Created unwind record\n" + << "Total size: " << Record.Prefix.TotalSize << "\n" + << "Unwind size: " << Record.UnwindDataSize << "\n" + << "EHFrame size: " << EHFrameSize << "\n" + << "EHFrameHdr size: " << Record.EHFrameHdrSize << "\n"); + return Record; +} + +static PerfJITRecordBatch getRecords(ExecutionSession &ES, LinkGraph &G, + std::atomic<uint64_t> &CodeIndex, + bool EmitDebugInfo, bool EmitUnwindInfo) { + std::unique_ptr<DWARFContext> DC; + StringMap<std::unique_ptr<MemoryBuffer>> DCBacking; + if (EmitDebugInfo) { + auto EDC = createDWARFContext(G); + if (!EDC) { + ES.reportError(EDC.takeError()); + EmitDebugInfo = false; + } else { + DC = std::move(EDC->first); + DCBacking = std::move(EDC->second); + } + } + PerfJITRecordBatch Batch; + for (auto Sym : G.defined_symbols()) { + if (!Sym->hasName() || !Sym->isCallable()) + continue; + if (EmitDebugInfo) { + auto DebugInfo = getDebugInfoRecord(*Sym, *DC); + if (DebugInfo) + Batch.DebugInfoRecords.push_back(std::move(*DebugInfo)); + } + Batch.CodeLoadRecords.push_back(getCodeLoadRecord(*Sym, CodeIndex)); + } + if (EmitUnwindInfo) { + auto UWR = getUnwindingRecord(G); + if (!UWR) { + ES.reportError(UWR.takeError()); + } else { + Batch.UnwindingRecord = std::move(*UWR); + } + } else { + Batch.UnwindingRecord.Prefix.TotalSize = 0; + } + return Batch; +} +} // namespace + +PerfSupportPlugin::PerfSupportPlugin(ExecutorProcessControl &EPC, + ExecutorAddr RegisterPerfStartAddr, + ExecutorAddr RegisterPerfEndAddr, + ExecutorAddr RegisterPerfImplAddr, + bool EmitDebugInfo, bool EmitUnwindInfo) + : EPC(EPC), RegisterPerfStartAddr(RegisterPerfStartAddr), + RegisterPerfEndAddr(RegisterPerfEndAddr), + RegisterPerfImplAddr(RegisterPerfImplAddr), CodeIndex(0), + EmitDebugInfo(EmitDebugInfo), EmitUnwindInfo(EmitUnwindInfo) { + cantFail(EPC.callSPSWrapper<void()>(RegisterPerfStartAddr)); +} +PerfSupportPlugin::~PerfSupportPlugin() { + cantFail(EPC.callSPSWrapper<void()>(RegisterPerfEndAddr)); +} + +void PerfSupportPlugin::modifyPassConfig(MaterializationResponsibility &MR, + LinkGraph &G, + PassConfiguration &Config) { + Config.PostFixupPasses.push_back([this](LinkGraph &G) { + auto Batch = getRecords(EPC.getExecutionSession(), G, CodeIndex, + EmitDebugInfo, EmitUnwindInfo); + G.allocActions().push_back( + {cantFail(shared::WrapperFunctionCall::Create< + shared::SPSArgList<shared::SPSPerfJITRecordBatch>>( + RegisterPerfImplAddr, Batch)), + {}}); + return Error::success(); + }); +} + +Expected<std::unique_ptr<PerfSupportPlugin>> +PerfSupportPlugin::Create(ExecutorProcessControl &EPC, JITDylib &JD, + bool EmitDebugInfo, bool EmitUnwindInfo) { + if (!EPC.getTargetTriple().isOSBinFormatELF()) { + return make_error<StringError>( + "Perf support only available for ELF LinkGraphs!", + inconvertibleErrorCode()); + } + auto &ES = EPC.getExecutionSession(); + ExecutorAddr StartAddr, EndAddr, ImplAddr; + if (auto Err = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder({&JD}), + {{ES.intern(RegisterPerfStartSymbolName), &StartAddr}, + {ES.intern(RegisterPerfEndSymbolName), &EndAddr}, + {ES.intern(RegisterPerfImplSymbolName), &ImplAddr}})) + return std::move(Err); + return std::make_unique<PerfSupportPlugin>(EPC, StartAddr, EndAddr, ImplAddr, + EmitDebugInfo, EmitUnwindInfo); +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp index 1bb4ecdff299..2b6c4b9e7f43 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp @@ -11,6 +11,7 @@ #include "llvm/BinaryFormat/ELF.h" #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" #include "llvm/ExecutionEngine/JITLink/aarch64.h" +#include "llvm/ExecutionEngine/JITLink/ppc64.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" @@ -39,21 +40,31 @@ public: void materialize(std::unique_ptr<MaterializationResponsibility> R) override { unsigned PointerSize; - support::endianness Endianness; + llvm::endianness Endianness; jitlink::Edge::Kind EdgeKind; const auto &TT = ENP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::x86_64: PointerSize = 8; - Endianness = support::endianness::little; + Endianness = llvm::endianness::little; EdgeKind = jitlink::x86_64::Pointer64; break; case Triple::aarch64: PointerSize = 8; - Endianness = support::endianness::little; + Endianness = llvm::endianness::little; EdgeKind = jitlink::aarch64::Pointer64; break; + case Triple::ppc64: + PointerSize = 8; + Endianness = llvm::endianness::big; + EdgeKind = jitlink::ppc64::Pointer64; + break; + case Triple::ppc64le: + PointerSize = 8; + Endianness = llvm::endianness::little; + EdgeKind = jitlink::ppc64::Pointer64; + break; default: llvm_unreachable("Unrecognized architecture"); } @@ -238,6 +249,9 @@ bool ELFNixPlatform::supportedTarget(const Triple &TT) { switch (TT.getArch()) { case Triple::x86_64: case Triple::aarch64: + // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported + // right now. + case Triple::ppc64le: return true; default: return false; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp index 56cd982cd5e1..f15315260ab0 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp @@ -9,67 +9,40 @@ #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" #include "llvm/ExecutionEngine/Orc/Core.h" -#include "llvm/Support/BinaryStreamWriter.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" using namespace llvm::orc::shared; namespace llvm { namespace orc { -Expected<std::unique_ptr<EPCEHFrameRegistrar>> EPCEHFrameRegistrar::Create( - ExecutionSession &ES, - std::optional<ExecutorAddr> RegistrationFunctionsDylib) { - // FIXME: Proper mangling here -- we really need to decouple linker mangling - // from DataLayout. - - // Find the addresses of the registration/deregistration functions in the - // executor process. - auto &EPC = ES.getExecutorProcessControl(); - - if (!RegistrationFunctionsDylib) { - if (auto D = EPC.loadDylib(nullptr)) - RegistrationFunctionsDylib = *D; - else - return D.takeError(); - } - - std::string RegisterWrapperName, DeregisterWrapperName; - if (EPC.getTargetTriple().isOSBinFormatMachO()) { - RegisterWrapperName += '_'; - DeregisterWrapperName += '_'; - } - RegisterWrapperName += "llvm_orc_registerEHFrameSectionWrapper"; - DeregisterWrapperName += "llvm_orc_deregisterEHFrameSectionWrapper"; - - SymbolLookupSet RegistrationSymbols; - RegistrationSymbols.add(EPC.intern(RegisterWrapperName)); - RegistrationSymbols.add(EPC.intern(DeregisterWrapperName)); - - auto Result = - EPC.lookupSymbols({{*RegistrationFunctionsDylib, RegistrationSymbols}}); - if (!Result) - return Result.takeError(); - - assert(Result->size() == 1 && "Unexpected number of dylibs in result"); - assert((*Result)[0].size() == 2 && - "Unexpected number of addresses in result"); - - auto RegisterEHFrameWrapperFnAddr = (*Result)[0][0]; - auto DeregisterEHFrameWrapperFnAddr = (*Result)[0][1]; - - return std::make_unique<EPCEHFrameRegistrar>(ES, RegisterEHFrameWrapperFnAddr, - DeregisterEHFrameWrapperFnAddr); +Expected<std::unique_ptr<EPCEHFrameRegistrar>> +EPCEHFrameRegistrar::Create(ExecutionSession &ES) { + + // Lookup addresseses of the registration/deregistration functions in the + // bootstrap map. + ExecutorAddr RegisterEHFrameSectionWrapper; + ExecutorAddr DeregisterEHFrameSectionWrapper; + if (auto Err = ES.getExecutorProcessControl().getBootstrapSymbols( + {{RegisterEHFrameSectionWrapper, + rt::RegisterEHFrameSectionWrapperName}, + {DeregisterEHFrameSectionWrapper, + rt::DeregisterEHFrameSectionWrapperName}})) + return std::move(Err); + + return std::make_unique<EPCEHFrameRegistrar>( + ES, RegisterEHFrameSectionWrapper, DeregisterEHFrameSectionWrapper); } Error EPCEHFrameRegistrar::registerEHFrames(ExecutorAddrRange EHFrameSection) { return ES.callSPSWrapper<void(SPSExecutorAddrRange)>( - RegisterEHFrameWrapperFnAddr, EHFrameSection); + RegisterEHFrameSectionWrapper, EHFrameSection); } Error EPCEHFrameRegistrar::deregisterEHFrames( ExecutorAddrRange EHFrameSection) { return ES.callSPSWrapper<void(SPSExecutorAddrRange)>( - DeregisterEHFrameWrapperFnAddr, EHFrameSection); + DeregisterEHFrameSectionWrapper, EHFrameSection); } } // end namespace orc diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index fb685e6c3727..8d5608cc4d4c 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -103,8 +103,8 @@ bool StaticInitGVIterator::isStaticInitGlobal(GlobalValue &GV) { // FIXME: These section checks are too strict: We should match first and // second word split by comma. if (GV.hasSection() && - (GV.getSection().startswith("__DATA,__objc_classlist") || - GV.getSection().startswith("__DATA,__objc_selrefs"))) + (GV.getSection().starts_with("__DATA,__objc_classlist") || + GV.getSection().starts_with("__DATA,__objc_selrefs"))) return true; } @@ -284,7 +284,7 @@ StaticLibraryDefinitionGenerator::Load( // If this is a universal binary then search for a slice matching the given // Triple. - if (auto *UB = cast<object::MachOUniversalBinary>(B->getBinary())) { + if (auto *UB = dyn_cast<object::MachOUniversalBinary>(B->getBinary())) { const auto &TT = L.getExecutionSession().getTargetTriple(); @@ -347,7 +347,7 @@ StaticLibraryDefinitionGenerator::Create( // If this is a universal binary then search for a slice matching the given // Triple. - if (auto *UB = cast<object::MachOUniversalBinary>(B->get())) { + if (auto *UB = dyn_cast<object::MachOUniversalBinary>(B->get())) { const auto &TT = L.getExecutionSession().getTargetTriple(); @@ -503,7 +503,7 @@ Error DLLImportDefinitionGenerator::tryToGenerate( DenseMap<StringRef, SymbolLookupFlags> ToLookUpSymbols; for (auto &KV : Symbols) { StringRef Deinterned = *KV.first; - if (Deinterned.startswith(getImpPrefix())) + if (Deinterned.starts_with(getImpPrefix())) Deinterned = Deinterned.drop_front(StringRef(getImpPrefix()).size()); // Don't degrade the required state if (ToLookUpSymbols.count(Deinterned) && @@ -538,11 +538,11 @@ DLLImportDefinitionGenerator::getTargetPointerSize(const Triple &TT) { } } -Expected<support::endianness> +Expected<llvm::endianness> DLLImportDefinitionGenerator::getTargetEndianness(const Triple &TT) { switch (TT.getArch()) { case Triple::x86_64: - return support::endianness::little; + return llvm::endianness::little; default: return make_error<StringError>( "architecture unsupported by DLLImportDefinitionGenerator", @@ -553,7 +553,7 @@ DLLImportDefinitionGenerator::getTargetEndianness(const Triple &TT) { Expected<std::unique_ptr<jitlink::LinkGraph>> DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) { Triple TT = ES.getTargetTriple(); - auto PointerSize = getTargetEndianness(TT); + auto PointerSize = getTargetPointerSize(TT); if (!PointerSize) return PointerSize.takeError(); auto Endianness = getTargetEndianness(TT); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index b8b013f8a7a9..ad27deff38d9 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -9,6 +9,8 @@ #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Process.h" @@ -27,7 +29,8 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D, Triple TargetTriple, unsigned PageSize, std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) - : ExecutorProcessControl(std::move(SSP), std::move(D)) { + : ExecutorProcessControl(std::move(SSP), std::move(D)), + InProcessMemoryAccess(TargetTriple.isArch64Bit()) { OwnedMemMgr = std::move(MemMgr); if (!OwnedMemMgr) @@ -42,6 +45,11 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( ExecutorAddr::fromPtr(this)}; if (this->TargetTriple.isOSBinFormatMachO()) GlobalManglingPrefix = '_'; + + this->BootstrapSymbols[rt::RegisterEHFrameSectionWrapperName] = + ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper); + this->BootstrapSymbols[rt::DeregisterEHFrameSectionWrapperName] = + ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper); } Expected<std::unique_ptr<SelfExecutorProcessControl>> @@ -139,41 +147,54 @@ Error SelfExecutorProcessControl::disconnect() { return Error::success(); } -void SelfExecutorProcessControl::writeUInt8sAsync( - ArrayRef<tpctypes::UInt8Write> Ws, WriteResultFn OnWriteComplete) { +void InProcessMemoryAccess::writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws, + WriteResultFn OnWriteComplete) { for (auto &W : Ws) *W.Addr.toPtr<uint8_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeUInt16sAsync( +void InProcessMemoryAccess::writeUInt16sAsync( ArrayRef<tpctypes::UInt16Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) *W.Addr.toPtr<uint16_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeUInt32sAsync( +void InProcessMemoryAccess::writeUInt32sAsync( ArrayRef<tpctypes::UInt32Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) *W.Addr.toPtr<uint32_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeUInt64sAsync( +void InProcessMemoryAccess::writeUInt64sAsync( ArrayRef<tpctypes::UInt64Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) *W.Addr.toPtr<uint64_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeBuffersAsync( +void InProcessMemoryAccess::writeBuffersAsync( ArrayRef<tpctypes::BufferWrite> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) memcpy(W.Addr.toPtr<char *>(), W.Buffer.data(), W.Buffer.size()); OnWriteComplete(Error::success()); } +void InProcessMemoryAccess::writePointersAsync( + ArrayRef<tpctypes::PointerWrite> Ws, WriteResultFn OnWriteComplete) { + if (IsArch64Bit) { + for (auto &W : Ws) + *W.Addr.toPtr<uint64_t *>() = W.Value.getValue(); + } else { + for (auto &W : Ws) + *W.Addr.toPtr<uint32_t *>() = static_cast<uint32_t>(W.Value.getValue()); + } + + OnWriteComplete(Error::success()); +} + shared::CWrapperFunctionResult SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager( void *Ctx, const void *FnTag, const char *Data, size_t Size) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index a0d81cdf2086..f9efff148df9 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -244,8 +244,7 @@ Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) { Constant *AddrIntVal = ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue()); Constant *AddrPtrVal = - ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, - PointerType::get(&FT, 0)); + ConstantExpr::getIntToPtr(AddrIntVal, PointerType::get(&FT, 0)); return AddrPtrVal; } @@ -286,7 +285,7 @@ std::vector<GlobalValue *> SymbolLinkagePromoter::operator()(Module &M) { // Rename if necessary. if (!GV.hasName()) GV.setName("__orc_anon." + Twine(NextId++)); - else if (GV.getName().startswith("\01L")) + else if (GV.getName().starts_with("\01L")) GV.setName("__" + GV.getName().substr(1) + "." + Twine(NextId++)); else if (GV.hasLocalLinkage()) GV.setName("__orc_lcl." + GV.getName() + "." + Twine(NextId++)); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index b66f52f1ec5d..17a96dee1000 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -126,16 +126,16 @@ void JITTargetMachineBuilderPrinter::print(raw_ostream &OS) const { OS << "\n" << Indent << " Optimization Level = "; switch (JTMB.OptLevel) { - case CodeGenOpt::None: + case CodeGenOptLevel::None: OS << "None"; break; - case CodeGenOpt::Less: + case CodeGenOptLevel::Less: OS << "Less"; break; - case CodeGenOpt::Default: + case CodeGenOptLevel::Default: OS << "Default"; break; - case CodeGenOpt::Aggressive: + case CodeGenOptLevel::Aggressive: OS << "Aggressive"; break; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index 7c7c2f000368..a19e17029810 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -10,8 +10,6 @@ #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/Orc/COFFPlatform.h" -#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" -#include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" @@ -29,8 +27,6 @@ #include "llvm/IR/Module.h" #include "llvm/Support/DynamicLibrary.h" -#include <map> - #define DEBUG_TYPE "orc" using namespace llvm; @@ -88,65 +84,6 @@ Function *addHelperAndWrapper(Module &M, StringRef WrapperName, return WrapperFn; } -class ORCPlatformSupport : public LLJIT::PlatformSupport { -public: - ORCPlatformSupport(orc::LLJIT &J) : J(J) {} - - Error initialize(orc::JITDylib &JD) override { - using llvm::orc::shared::SPSExecutorAddr; - using llvm::orc::shared::SPSString; - using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t); - enum dlopen_mode : int32_t { - ORC_RT_RTLD_LAZY = 0x1, - ORC_RT_RTLD_NOW = 0x2, - ORC_RT_RTLD_LOCAL = 0x4, - ORC_RT_RTLD_GLOBAL = 0x8 - }; - - auto &ES = J.getExecutionSession(); - auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo( - [](const JITDylibSearchOrder &SO) { return SO; }); - - if (auto WrapperAddr = - ES.lookup(MainSearchOrder, - J.mangleAndIntern("__orc_rt_jit_dlopen_wrapper"))) { - return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(), - DSOHandles[&JD], JD.getName(), - int32_t(ORC_RT_RTLD_LAZY)); - } else - return WrapperAddr.takeError(); - } - - Error deinitialize(orc::JITDylib &JD) override { - using llvm::orc::shared::SPSExecutorAddr; - using SPSDLCloseSig = int32_t(SPSExecutorAddr); - - auto &ES = J.getExecutionSession(); - auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo( - [](const JITDylibSearchOrder &SO) { return SO; }); - - if (auto WrapperAddr = - ES.lookup(MainSearchOrder, - J.mangleAndIntern("__orc_rt_jit_dlclose_wrapper"))) { - int32_t result; - auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>( - WrapperAddr->getAddress(), result, DSOHandles[&JD]); - if (E) - return E; - else if (result) - return make_error<StringError>("dlclose failed", - inconvertibleErrorCode()); - DSOHandles.erase(&JD); - } else - return WrapperAddr.takeError(); - return Error::success(); - } - -private: - orc::LLJIT &J; - DenseMap<orc::JITDylib *, orc::ExecutorAddr> DSOHandles; -}; - class GenericLLVMIRPlatformSupport; /// orc::Platform component of Generic LLVM IR Platform support. @@ -276,11 +213,11 @@ public: // will trigger a lookup to materialize the module) and the InitFunctions // map (which holds the names of the symbols to execute). for (auto &KV : MU.getSymbols()) - if ((*KV.first).startswith(InitFunctionPrefix)) { + if ((*KV.first).starts_with(InitFunctionPrefix)) { InitSymbols[&JD].add(KV.first, SymbolLookupFlags::WeaklyReferencedSymbol); InitFunctions[&JD].add(KV.first); - } else if ((*KV.first).startswith(DeInitFunctionPrefix)) { + } else if ((*KV.first).starts_with(DeInitFunctionPrefix)) { DeInitFunctions[&JD].add(KV.first); } } @@ -660,6 +597,54 @@ public: namespace llvm { namespace orc { +Error ORCPlatformSupport::initialize(orc::JITDylib &JD) { + using llvm::orc::shared::SPSExecutorAddr; + using llvm::orc::shared::SPSString; + using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t); + enum dlopen_mode : int32_t { + ORC_RT_RTLD_LAZY = 0x1, + ORC_RT_RTLD_NOW = 0x2, + ORC_RT_RTLD_LOCAL = 0x4, + ORC_RT_RTLD_GLOBAL = 0x8 + }; + + auto &ES = J.getExecutionSession(); + auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo( + [](const JITDylibSearchOrder &SO) { return SO; }); + + if (auto WrapperAddr = ES.lookup( + MainSearchOrder, J.mangleAndIntern("__orc_rt_jit_dlopen_wrapper"))) { + return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(), + DSOHandles[&JD], JD.getName(), + int32_t(ORC_RT_RTLD_LAZY)); + } else + return WrapperAddr.takeError(); +} + +Error ORCPlatformSupport::deinitialize(orc::JITDylib &JD) { + using llvm::orc::shared::SPSExecutorAddr; + using SPSDLCloseSig = int32_t(SPSExecutorAddr); + + auto &ES = J.getExecutionSession(); + auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo( + [](const JITDylibSearchOrder &SO) { return SO; }); + + if (auto WrapperAddr = ES.lookup( + MainSearchOrder, J.mangleAndIntern("__orc_rt_jit_dlclose_wrapper"))) { + int32_t result; + auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>( + WrapperAddr->getAddress(), result, DSOHandles[&JD]); + if (E) + return E; + else if (result) + return make_error<StringError>("dlclose failed", + inconvertibleErrorCode()); + DSOHandles.erase(&JD); + } else + return WrapperAddr.takeError(); + return Error::success(); +} + void LLJIT::PlatformSupport::setInitTransform( LLJIT &J, IRTransformLayer::TransformFunction T) { J.InitHelperTransformLayer->setTransform(std::move(T)); @@ -752,6 +737,12 @@ Error LLJITBuilderState::prepareForConstruction() { case Triple::x86_64: UseJITLink = !TT.isOSBinFormatCOFF(); break; + case Triple::ppc64: + UseJITLink = TT.isPPC64ELFv2ABI(); + break; + case Triple::ppc64le: + UseJITLink = TT.isOSBinFormatELF(); + break; default: break; } @@ -775,25 +766,17 @@ Error LLJITBuilderState::prepareForConstruction() { // If we need a process JITDylib but no setup function has been given then // create a default one. - if (!SetupProcessSymbolsJITDylib && - (LinkProcessSymbolsByDefault || EnableDebuggerSupport)) { - - LLVM_DEBUG({ - dbgs() << "Creating default Process JD setup function (neeeded for"; - if (LinkProcessSymbolsByDefault) - dbgs() << " <link-process-syms-by-default>"; - if (EnableDebuggerSupport) - dbgs() << " <debugger-support>"; - dbgs() << ")\n"; - }); - - SetupProcessSymbolsJITDylib = [this](JITDylib &JD) -> Error { + if (!SetupProcessSymbolsJITDylib && LinkProcessSymbolsByDefault) { + LLVM_DEBUG(dbgs() << "Creating default Process JD setup function\n"); + SetupProcessSymbolsJITDylib = [this](LLJIT &J) -> Expected<JITDylibSP> { + auto &JD = + J.getExecutionSession().createBareJITDylib("<Process Symbols>"); auto G = orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( DL->getGlobalPrefix()); if (!G) return G.takeError(); JD.addGenerator(std::move(*G)); - return Error::success(); + return &JD; }; } @@ -998,50 +981,18 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) } if (S.SetupProcessSymbolsJITDylib) { - ProcessSymbols = &ES->createBareJITDylib("<Process Symbols>"); - if (auto Err2 = S.SetupProcessSymbolsJITDylib(*ProcessSymbols)) { - Err = std::move(Err2); + if (auto ProcSymsJD = S.SetupProcessSymbolsJITDylib(*this)) { + ProcessSymbols = ProcSymsJD->get(); + } else { + Err = ProcSymsJD.takeError(); return; } } - if (S.EnableDebuggerSupport) { - if (auto *OLL = dyn_cast<ObjectLinkingLayer>(ObjLinkingLayer.get())) { - switch (TT.getObjectFormat()) { - case Triple::ELF: { - auto Registrar = createJITLoaderGDBRegistrar(*ES); - if (!Registrar) { - Err = Registrar.takeError(); - return; - } - OLL->addPlugin(std::make_unique<DebugObjectManagerPlugin>( - *ES, std::move(*Registrar), true, true)); - break; - } - case Triple::MachO: { - assert(ProcessSymbols && "ProcessSymbols JD should be available when " - "EnableDebuggerSupport is set"); - auto DS = - GDBJITDebugInfoRegistrationPlugin::Create(*ES, *ProcessSymbols, TT); - if (!DS) { - Err = DS.takeError(); - return; - } - OLL->addPlugin(std::move(*DS)); - break; - } - default: - LLVM_DEBUG({ - dbgs() << "Cannot enable LLJIT debugger support: " - << Triple::getObjectFormatTypeName(TT.getObjectFormat()) - << " not supported.\n"; - }); - } - } else { - LLVM_DEBUG({ - dbgs() << "Cannot enable LLJIT debugger support: " - " debugger support is only available when using JITLink.\n"; - }); + if (S.PrePlatformSetup) { + if (auto Err2 = S.PrePlatformSetup(*this)) { + Err = std::move(Err2); + return; } } @@ -1131,7 +1082,7 @@ Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) { if (!ObjLinkingLayer) return make_error<StringError>( - "SetUpTargetPlatform requires ObjectLinkingLayer", + "ExecutorNativePlatform requires ObjectLinkingLayer", inconvertibleErrorCode()); std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index a3a766d602c1..9057300bf043 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -15,6 +15,7 @@ #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/ExecutionEngine/Orc/MachOBuilder.h" #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" @@ -34,6 +35,8 @@ using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>; using SPSMachOJITDylibDepInfoMap = SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>; +class SPSMachOExecutorSymbolFlags; + template <> class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOPlatform::MachOJITDylibDepInfo> { @@ -55,23 +58,54 @@ public: } }; +template <> +class SPSSerializationTraits<SPSMachOExecutorSymbolFlags, + MachOPlatform::MachOExecutorSymbolFlags> { +private: + using UT = std::underlying_type_t<MachOPlatform::MachOExecutorSymbolFlags>; + +public: + static size_t size(const MachOPlatform::MachOExecutorSymbolFlags &SF) { + return sizeof(UT); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOPlatform::MachOExecutorSymbolFlags &SF) { + return SPSArgList<UT>::serialize(OB, static_cast<UT>(SF)); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOPlatform::MachOExecutorSymbolFlags &SF) { + UT Tmp; + if (!SPSArgList<UT>::deserialize(IB, Tmp)) + return false; + SF = static_cast<MachOPlatform::MachOExecutorSymbolFlags>(Tmp); + return true; + } +}; + } // namespace shared } // namespace orc } // namespace llvm namespace { +using SPSRegisterSymbolsArgs = + SPSArgList<SPSExecutorAddr, + SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr, + SPSMachOExecutorSymbolFlags>>>; + std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP, std::string Name) { unsigned PointerSize; - support::endianness Endianness; + llvm::endianness Endianness; const auto &TT = MOP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::aarch64: case Triple::x86_64: PointerSize = 8; - Endianness = support::endianness::little; + Endianness = llvm::endianness::little; break; default: llvm_unreachable("Unrecognized architecture"); @@ -82,119 +116,32 @@ std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP, jitlink::getGenericEdgeKindName); } -// Generates a MachO header. -class MachOHeaderMaterializationUnit : public MaterializationUnit { -public: - MachOHeaderMaterializationUnit(MachOPlatform &MOP, - const SymbolStringPtr &HeaderStartSymbol) - : MaterializationUnit(createHeaderInterface(MOP, HeaderStartSymbol)), - MOP(MOP) {} - - StringRef getName() const override { return "MachOHeaderMU"; } - - void materialize(std::unique_ptr<MaterializationResponsibility> R) override { - auto G = createPlatformGraph(MOP, "<MachOHeaderMU>"); - addMachOHeader(*G, MOP, R->getInitializerSymbol()); - MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); - } - - void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} - - static void addMachOHeader(jitlink::LinkGraph &G, MachOPlatform &MOP, - const SymbolStringPtr &InitializerSymbol) { - auto &HeaderSection = G.createSection("__header", MemProt::Read); - auto &HeaderBlock = createHeaderBlock(G, HeaderSection); - - // Init symbol is header-start symbol. - G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol, - HeaderBlock.getSize(), jitlink::Linkage::Strong, - jitlink::Scope::Default, false, true); - for (auto &HS : AdditionalHeaderSymbols) - G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(), - jitlink::Linkage::Strong, jitlink::Scope::Default, - false, true); - } - -private: - struct HeaderSymbol { - const char *Name; - uint64_t Offset; - }; - - static constexpr HeaderSymbol AdditionalHeaderSymbols[] = { - {"___mh_executable_header", 0}}; - - static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, - jitlink::Section &HeaderSection) { - MachO::mach_header_64 Hdr; - Hdr.magic = MachO::MH_MAGIC_64; - switch (G.getTargetTriple().getArch()) { - case Triple::aarch64: - Hdr.cputype = MachO::CPU_TYPE_ARM64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; - break; - case Triple::x86_64: - Hdr.cputype = MachO::CPU_TYPE_X86_64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; - break; - default: - llvm_unreachable("Unrecognized architecture"); - } - Hdr.filetype = MachO::MH_DYLIB; // Custom file type? - Hdr.ncmds = 0; - Hdr.sizeofcmds = 0; - Hdr.flags = 0; - Hdr.reserved = 0; - - if (G.getEndianness() != support::endian::system_endianness()) - MachO::swapStruct(Hdr); - - auto HeaderContent = G.allocateContent( - ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); - - return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, - 0); - } - - static MaterializationUnit::Interface - createHeaderInterface(MachOPlatform &MOP, - const SymbolStringPtr &HeaderStartSymbol) { - SymbolFlagsMap HeaderSymbolFlags; - - HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; - for (auto &HS : AdditionalHeaderSymbols) - HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = - JITSymbolFlags::Exported; - - return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), - HeaderStartSymbol); - } - - MachOPlatform &MOP; -}; - -constexpr MachOHeaderMaterializationUnit::HeaderSymbol - MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[]; - // Creates a Bootstrap-Complete LinkGraph to run deferred actions. class MachOPlatformCompleteBootstrapMaterializationUnit : public MaterializationUnit { public: + using SymbolTableVector = + SmallVector<std::tuple<ExecutorAddr, ExecutorAddr, + MachOPlatform::MachOExecutorSymbolFlags>>; + MachOPlatformCompleteBootstrapMaterializationUnit( MachOPlatform &MOP, StringRef PlatformJDName, - SymbolStringPtr CompleteBootstrapSymbol, shared::AllocActions DeferredAAs, + SymbolStringPtr CompleteBootstrapSymbol, SymbolTableVector SymTab, + shared::AllocActions DeferredAAs, ExecutorAddr MachOHeaderAddr, ExecutorAddr PlatformBootstrap, ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib, ExecutorAddr DeregisterJITDylib, - ExecutorAddr MachOHeaderAddr) + ExecutorAddr RegisterObjectSymbolTable, + ExecutorAddr DeregisterObjectSymbolTable) : MaterializationUnit( {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}), MOP(MOP), PlatformJDName(PlatformJDName), CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)), - DeferredAAs(std::move(DeferredAAs)), - PlatformBootstrap(PlatformBootstrap), + SymTab(std::move(SymTab)), DeferredAAs(std::move(DeferredAAs)), + MachOHeaderAddr(MachOHeaderAddr), PlatformBootstrap(PlatformBootstrap), PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib), DeregisterJITDylib(DeregisterJITDylib), - MachOHeaderAddr(MachOHeaderAddr) {} + RegisterObjectSymbolTable(RegisterObjectSymbolTable), + DeregisterObjectSymbolTable(DeregisterObjectSymbolTable) {} StringRef getName() const override { return "MachOPlatformCompleteBootstrap"; @@ -211,7 +158,7 @@ public: Linkage::Strong, Scope::Hidden, false, true); // Reserve space for the stolen actions, plus two extras. - G->allocActions().reserve(DeferredAAs.size() + 2); + G->allocActions().reserve(DeferredAAs.size() + 3); // 1. Bootstrap the platform support code. G->allocActions().push_back( @@ -227,7 +174,14 @@ public: cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( DeregisterJITDylib, MachOHeaderAddr))}); - // 3. Add the deferred actions to the graph. + // 3. Register deferred symbols. + G->allocActions().push_back( + {cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( + RegisterObjectSymbolTable, MachOHeaderAddr, SymTab)), + cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( + DeregisterObjectSymbolTable, MachOHeaderAddr, SymTab))}); + + // 4. Add the deferred actions to the graph. std::move(DeferredAAs.begin(), DeferredAAs.end(), std::back_inserter(G->allocActions())); @@ -240,12 +194,15 @@ private: MachOPlatform &MOP; StringRef PlatformJDName; SymbolStringPtr CompleteBootstrapSymbol; + SymbolTableVector SymTab; shared::AllocActions DeferredAAs; + ExecutorAddr MachOHeaderAddr; ExecutorAddr PlatformBootstrap; ExecutorAddr PlatformShutdown; ExecutorAddr RegisterJITDylib; ExecutorAddr DeregisterJITDylib; - ExecutorAddr MachOHeaderAddr; + ExecutorAddr RegisterObjectSymbolTable; + ExecutorAddr DeregisterObjectSymbolTable; }; static StringRef ObjCRuntimeObjectSectionsData[] = { @@ -266,6 +223,33 @@ static StringRef ObjCRuntimeObjectSectionName = static StringRef ObjCImageInfoSymbolName = "__llvm_jitlink_macho_objc_imageinfo"; +struct ObjCImageInfoFlags { + uint16_t SwiftABIVersion; + uint16_t SwiftVersion; + bool HasCategoryClassProperties; + bool HasSignedObjCClassROs; + + static constexpr uint32_t SIGNED_CLASS_RO = (1 << 4); + static constexpr uint32_t HAS_CATEGORY_CLASS_PROPERTIES = (1 << 6); + + explicit ObjCImageInfoFlags(uint32_t RawFlags) { + HasSignedObjCClassROs = RawFlags & SIGNED_CLASS_RO; + HasCategoryClassProperties = RawFlags & HAS_CATEGORY_CLASS_PROPERTIES; + SwiftABIVersion = (RawFlags >> 8) & 0xFF; + SwiftVersion = (RawFlags >> 16) & 0xFFFF; + } + + uint32_t rawFlags() const { + uint32_t Result = 0; + if (HasCategoryClassProperties) + Result |= HAS_CATEGORY_CLASS_PROPERTIES; + if (HasSignedObjCClassROs) + Result |= SIGNED_CLASS_RO; + Result |= (SwiftABIVersion << 8); + Result |= (SwiftVersion << 16); + return Result; + } +}; } // end anonymous namespace namespace llvm { @@ -275,6 +259,7 @@ Expected<std::unique_ptr<MachOPlatform>> MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime, + MachOHeaderMUBuilder BuildMachOHeaderMU, std::optional<SymbolAliasMap> RuntimeAliases) { // If the target is not supported then bail out immediately. @@ -305,8 +290,9 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, // Create the instance. Error Err = Error::success(); - auto P = std::unique_ptr<MachOPlatform>(new MachOPlatform( - ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err)); + auto P = std::unique_ptr<MachOPlatform>( + new MachOPlatform(ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), + std::move(BuildMachOHeaderMU), Err)); if (Err) return std::move(Err); return std::move(P); @@ -315,6 +301,7 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, Expected<std::unique_ptr<MachOPlatform>> MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, const char *OrcRuntimePath, + MachOHeaderMUBuilder BuildMachOHeaderMU, std::optional<SymbolAliasMap> RuntimeAliases) { // Create a generator for the ORC runtime archive. @@ -325,12 +312,11 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, return Create(ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator), - std::move(RuntimeAliases)); + std::move(BuildMachOHeaderMU), std::move(RuntimeAliases)); } Error MachOPlatform::setupJITDylib(JITDylib &JD) { - if (auto Err = JD.define(std::make_unique<MachOHeaderMaterializationUnit>( - *this, MachOHeaderStartSymbol))) + if (auto Err = JD.define(BuildMachOHeaderMU(*this))) return Err; return ES.lookup({&JD}, MachOHeaderStartSymbol).takeError(); @@ -419,11 +405,36 @@ bool MachOPlatform::supportedTarget(const Triple &TT) { } } +jitlink::Edge::Kind MachOPlatform::getPointerEdgeKind(jitlink::LinkGraph &G) { + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + return jitlink::aarch64::Pointer64; + case Triple::x86_64: + return jitlink::x86_64::Pointer64; + default: + llvm_unreachable("Unsupported architecture"); + } +} + +MachOPlatform::MachOExecutorSymbolFlags +MachOPlatform::flagsForSymbol(jitlink::Symbol &Sym) { + MachOPlatform::MachOExecutorSymbolFlags Flags{}; + if (Sym.getLinkage() == jitlink::Linkage::Weak) + Flags |= MachOExecutorSymbolFlags::Weak; + + if (Sym.isCallable()) + Flags |= MachOExecutorSymbolFlags::Callable; + + return Flags; +} + MachOPlatform::MachOPlatform( ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, - std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err) - : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer) { + std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, + MachOHeaderMUBuilder BuildMachOHeaderMU, Error &Err) + : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer), + BuildMachOHeaderMU(std::move(BuildMachOHeaderMU)) { ErrorAsOutParameter _(&Err); ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this)); PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); @@ -442,11 +453,11 @@ MachOPlatform::MachOPlatform( // itself (to build the allocation actions that will call the registration // functions). Further complicating the situation (a) the graph containing // the registration functions is allowed to depend on other graphs (e.g. the - // graph containing the ORC runtime RTTI support) so we need to handle with - // an unknown set of dependencies during bootstrap, and (b) these graphs may + // graph containing the ORC runtime RTTI support) so we need to handle an + // unknown set of dependencies during bootstrap, and (b) these graphs may // be linked concurrently if the user has installed a concurrent dispatcher. // - // We satisfy these constraint by implementing a bootstrap phase during which + // We satisfy these constraints by implementing a bootstrap phase during which // allocation actions generated by MachOPlatform are appended to a list of // deferred allocation actions, rather than to the graphs themselves. At the // end of the bootstrap process the deferred actions are attached to a final @@ -486,8 +497,7 @@ MachOPlatform::MachOPlatform( // the support methods callable. The bootstrap is now complete. // Step (1) Add header materialization unit and request. - if ((Err = PlatformJD.define(std::make_unique<MachOHeaderMaterializationUnit>( - *this, MachOHeaderStartSymbol)))) + if ((Err = PlatformJD.define(this->BuildMachOHeaderMU(*this)))) return; if ((Err = ES.lookup(&PlatformJD, MachOHeaderStartSymbol).takeError())) return; @@ -498,6 +508,8 @@ MachOPlatform::MachOPlatform( SymbolLookupSet( {PlatformBootstrap.Name, PlatformShutdown.Name, RegisterJITDylib.Name, DeregisterJITDylib.Name, + RegisterObjectSymbolTable.Name, + DeregisterObjectSymbolTable.Name, RegisterObjectPlatformSections.Name, DeregisterObjectPlatformSections.Name, CreatePThreadKey.Name})) @@ -516,9 +528,11 @@ MachOPlatform::MachOPlatform( if ((Err = PlatformJD.define( std::make_unique<MachOPlatformCompleteBootstrapMaterializationUnit>( *this, PlatformJD.getName(), BootstrapCompleteSymbol, - std::move(BI.DeferredAAs), PlatformBootstrap.Addr, + std::move(BI.SymTab), std::move(BI.DeferredAAs), + BI.MachOHeaderAddr, PlatformBootstrap.Addr, PlatformShutdown.Addr, RegisterJITDylib.Addr, - DeregisterJITDylib.Addr, BI.MachOHeaderAddr)))) + DeregisterJITDylib.Addr, RegisterObjectSymbolTable.Addr, + DeregisterObjectSymbolTable.Addr)))) return; if ((Err = ES.lookup(makeJITDylibSearchOrder( &PlatformJD, JITDylibLookupFlags::MatchAllSymbols), @@ -540,11 +554,11 @@ Error MachOPlatform::associateRuntimeSupportFunctions() { ES.wrapAsyncWithSPS<PushInitializersSPSSig>( this, &MachOPlatform::rt_pushInitializers); - using LookupSymbolSPSSig = - SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString); - WFs[ES.intern("___orc_rt_macho_symbol_lookup_tag")] = - ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this, - &MachOPlatform::rt_lookupSymbol); + using PushSymbolsSPSSig = + SPSError(SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>); + WFs[ES.intern("___orc_rt_macho_push_symbols_tag")] = + ES.wrapAsyncWithSPS<PushSymbolsSPSSig>(this, + &MachOPlatform::rt_pushSymbols); return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs)); } @@ -665,11 +679,9 @@ void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, pushInitializersLoop(std::move(SendResult), JD); } -void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, - ExecutorAddr Handle, StringRef SymbolName) { - LLVM_DEBUG({ - dbgs() << "MachOPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"; - }); +void MachOPlatform::rt_pushSymbols( + PushSymbolsInSendResultFn SendResult, ExecutorAddr Handle, + const std::vector<std::pair<StringRef, bool>> &SymbolNames) { JITDylib *JD = nullptr; @@ -679,39 +691,37 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, if (I != HeaderAddrToJITDylib.end()) JD = I->second; } + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_pushSymbols("; + if (JD) + dbgs() << "\"" << JD->getName() << "\", [ "; + else + dbgs() << "<invalid handle " << Handle << ">, [ "; + for (auto &Name : SymbolNames) + dbgs() << "\"" << Name.first << "\" "; + dbgs() << "])\n"; + }); if (!JD) { - LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } - // Use functor class to work around XL build compiler issue on AIX. - class RtLookupNotifyComplete { - public: - RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult) - : SendResult(std::move(SendResult)) {} - void operator()(Expected<SymbolMap> Result) { - if (Result) { - assert(Result->size() == 1 && "Unexpected result map count"); - SendResult(Result->begin()->second.getAddress()); - } else { - SendResult(Result.takeError()); - } - } - - private: - SendSymbolAddressFn SendResult; - }; + SymbolLookupSet LS; + for (auto &[Name, Required] : SymbolNames) + LS.add(ES.intern(Name), Required + ? SymbolLookupFlags::RequiredSymbol + : SymbolLookupFlags::WeaklyReferencedSymbol); - // FIXME: Proper mangling. - auto MangledName = ("_" + SymbolName).str(); ES.lookup( LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, - SymbolLookupSet(ES.intern(MangledName)), SymbolState::Ready, - RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister); + std::move(LS), SymbolState::Ready, + [SendResult = std::move(SendResult)](Expected<SymbolMap> Result) mutable { + SendResult(Result.takeError()); + }, + NoDependenciesToRegister); } Expected<uint64_t> MachOPlatform::createPThreadKey() { @@ -781,6 +791,18 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( return fixTLVSectionsAndEdges(G, JD); }); + // Add symbol table prepare and register passes: These will add strings for + // all symbols to the c-strings section, and build a symbol table registration + // call. + auto JITSymTabInfo = std::make_shared<JITSymTabVector>(); + Config.PostPrunePasses.push_back([this, JITSymTabInfo](LinkGraph &G) { + return prepareSymbolTableRegistration(G, *JITSymTabInfo); + }); + Config.PostFixupPasses.push_back([this, &MR, JITSymTabInfo, + InBootstrapPhase](LinkGraph &G) { + return addSymbolTableRegistration(G, MR, *JITSymTabInfo, InBootstrapPhase); + }); + // Add a pass to register the final addresses of any special sections in the // object with the runtime. Config.PostAllocationPasses.push_back( @@ -826,6 +848,9 @@ Error MachOPlatform::MachOPlatformPlugin:: {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr}, {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr}, {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr}, + {*MP.RegisterObjectSymbolTable.Name, &MP.RegisterObjectSymbolTable.Addr}, + {*MP.DeregisterObjectSymbolTable.Name, + &MP.DeregisterObjectSymbolTable.Addr}, {*MP.RegisterObjectPlatformSections.Name, &MP.RegisterObjectPlatformSections.Addr}, {*MP.DeregisterObjectPlatformSections.Name, @@ -1029,15 +1054,19 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( " does not match first registered version", inconvertibleErrorCode()); if (ObjCImageInfoItr->second.Flags != Flags) - return make_error<StringError>("ObjC flags in " + G.getName() + - " do not match first registered flags", - inconvertibleErrorCode()); + if (Error E = mergeImageInfoFlags(G, MR, ObjCImageInfoItr->second, Flags)) + return E; // __objc_imageinfo is valid. Delete the block. for (auto *S : ObjCImageInfo->symbols()) G.removeDefinedSymbol(*S); G.removeBlock(ObjCImageInfoBlock); } else { + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Registered __objc_imageinfo for " + << MR.getTargetJITDylib().getName() << " in " << G.getName() + << "; flags = " << formatv("{0:x4}", Flags) << "\n"; + }); // We haven't registered an __objc_imageinfo section yet. Register and // move on. The section should already be marked no-dead-strip. G.addDefinedSymbol(ObjCImageInfoBlock, 0, ObjCImageInfoSymbolName, @@ -1047,12 +1076,66 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( {{MR.getExecutionSession().intern(ObjCImageInfoSymbolName), JITSymbolFlags()}})) return Err; - ObjCImageInfos[&MR.getTargetJITDylib()] = {Version, Flags}; + ObjCImageInfos[&MR.getTargetJITDylib()] = {Version, Flags, false}; } return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::mergeImageInfoFlags( + jitlink::LinkGraph &G, MaterializationResponsibility &MR, + ObjCImageInfo &Info, uint32_t NewFlags) { + if (Info.Flags == NewFlags) + return Error::success(); + + ObjCImageInfoFlags Old(Info.Flags); + ObjCImageInfoFlags New(NewFlags); + + // Check for incompatible flags. + if (Old.SwiftABIVersion && New.SwiftABIVersion && + Old.SwiftABIVersion != New.SwiftABIVersion) + return make_error<StringError>("Swift ABI version in " + G.getName() + + " does not match first registered flags", + inconvertibleErrorCode()); + + if (Old.HasCategoryClassProperties != New.HasCategoryClassProperties) + return make_error<StringError>("ObjC category class property support in " + + G.getName() + + " does not match first registered flags", + inconvertibleErrorCode()); + if (Old.HasSignedObjCClassROs != New.HasSignedObjCClassROs) + return make_error<StringError>("ObjC class_ro_t pointer signing in " + + G.getName() + + " does not match first registered flags", + inconvertibleErrorCode()); + + // If we cannot change the flags, ignore any remaining differences. Adding + // Swift or changing its version are unlikely to cause problems in practice. + if (Info.Finalized) + return Error::success(); + + // Use the minimum Swift version. + if (Old.SwiftVersion && New.SwiftVersion) + New.SwiftVersion = std::min(Old.SwiftVersion, New.SwiftVersion); + else if (Old.SwiftVersion) + New.SwiftVersion = Old.SwiftVersion; + // Add a Swift ABI version if it was pure objc before. + if (!New.SwiftABIVersion) + New.SwiftABIVersion = Old.SwiftABIVersion; + + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Merging __objc_imageinfo flags for " + << MR.getTargetJITDylib().getName() << " (was " + << formatv("{0:x4}", Old.rawFlags()) << ")" + << " with " << G.getName() << " (" << formatv("{0:x4}", NewFlags) + << ")" + << " -> " << formatv("{0:x4}", New.rawFlags()) << "\n"; + }); + + Info.Flags = New.rawFlags(); + return Error::success(); +} + Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( jitlink::LinkGraph &G, JITDylib &JD) { @@ -1250,15 +1333,6 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( UI->CompactUnwindSection); if (!MachOPlatformSecs.empty() || UnwindInfo) { - ExecutorAddr HeaderAddr; - { - std::lock_guard<std::mutex> Lock(MP.PlatformMutex); - auto I = MP.JITDylibToHeaderAddr.find(&JD); - assert(I != MP.JITDylibToHeaderAddr.end() && - "Missing header for JITDylib"); - HeaderAddr = I->second; - } - // Dump the scraped inits. LLVM_DEBUG({ dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; @@ -1276,6 +1350,15 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( ? G.allocActions() : MP.Bootstrap.load()->DeferredAAs; + ExecutorAddr HeaderAddr; + { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + auto I = MP.JITDylibToHeaderAddr.find(&JD); + assert(I != MP.JITDylibToHeaderAddr.end() && + "No header registered for JD"); + assert(I->second && "Null header registered for JD"); + HeaderAddr = I->second; + } allocActions.push_back( {cantFail( WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>( @@ -1374,17 +1457,7 @@ Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( strcpy(SD.Sec.segname, "__DATA"); SD.Sec.size = 8; SD.AddFixups = [&](size_t RecordOffset) { - jitlink::Edge::Kind PointerEdge = jitlink::Edge::Invalid; - switch (G.getTargetTriple().getArch()) { - case Triple::aarch64: - PointerEdge = jitlink::aarch64::Pointer64; - break; - case Triple::x86_64: - PointerEdge = jitlink::x86_64::Pointer64; - break; - default: - llvm_unreachable("Unsupported architecture"); - } + auto PointerEdge = getPointerEdgeKind(G); // Look for an existing __objc_imageinfo symbol. jitlink::Symbol *ObjCImageInfoSym = nullptr; @@ -1403,6 +1476,24 @@ Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( for (auto *Sym : G.defined_symbols()) if (Sym->hasName() && Sym->getName() == ObjCImageInfoSymbolName) { ObjCImageInfoSym = Sym; + std::optional<uint32_t> Flags; + { + std::lock_guard<std::mutex> Lock(PluginMutex); + auto It = ObjCImageInfos.find(&MR.getTargetJITDylib()); + if (It != ObjCImageInfos.end()) { + It->second.Finalized = true; + Flags = It->second.Flags; + } + } + + if (Flags) { + // We own the definition of __objc_image_info; write the final + // merged flags value. + auto Content = Sym->getBlock().getMutableContent(G); + assert(Content.size() == 8 && + "__objc_image_info size should have been verified already"); + support::endian::write32(&Content[4], *Flags, G.getEndianness()); + } break; } if (!ObjCImageInfoSym) @@ -1460,7 +1551,7 @@ Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( auto SecContent = SecBlock.getAlreadyMutableContent(); char *P = SecContent.data(); auto WriteMachOStruct = [&](auto S) { - if (G.getEndianness() != support::endian::system_endianness()) + if (G.getEndianness() != llvm::endianness::native) MachO::swapStruct(S); memcpy(P, &S, sizeof(S)); P += sizeof(S); @@ -1492,5 +1583,179 @@ Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::prepareSymbolTableRegistration( + jitlink::LinkGraph &G, JITSymTabVector &JITSymTabInfo) { + + auto *CStringSec = G.findSectionByName(MachOCStringSectionName); + if (!CStringSec) + CStringSec = &G.createSection(MachOCStringSectionName, + MemProt::Read | MemProt::Exec); + + // Make a map of existing strings so that we can re-use them: + DenseMap<StringRef, jitlink::Symbol *> ExistingStrings; + for (auto *Sym : CStringSec->symbols()) { + + // The LinkGraph builder should have created single strings blocks, and all + // plugins should have maintained this invariant. + auto Content = Sym->getBlock().getContent(); + ExistingStrings.insert( + std::make_pair(StringRef(Content.data(), Content.size()), Sym)); + } + + // Add all symbol names to the string section, and record the symbols for + // those names. + { + SmallVector<jitlink::Symbol *> SymsToProcess; + for (auto *Sym : G.defined_symbols()) + SymsToProcess.push_back(Sym); + + for (auto *Sym : SymsToProcess) { + if (!Sym->hasName()) + continue; + + auto I = ExistingStrings.find(Sym->getName()); + if (I == ExistingStrings.end()) { + auto &NameBlock = G.createMutableContentBlock( + *CStringSec, G.allocateCString(Sym->getName()), orc::ExecutorAddr(), + 1, 0); + auto &SymbolNameSym = G.addAnonymousSymbol( + NameBlock, 0, NameBlock.getSize(), false, true); + JITSymTabInfo.push_back({Sym, &SymbolNameSym}); + } else + JITSymTabInfo.push_back({Sym, I->second}); + } + } + + return Error::success(); +} + +Error MachOPlatform::MachOPlatformPlugin::addSymbolTableRegistration( + jitlink::LinkGraph &G, MaterializationResponsibility &MR, + JITSymTabVector &JITSymTabInfo, bool InBootstrapPhase) { + + ExecutorAddr HeaderAddr; + { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + auto I = MP.JITDylibToHeaderAddr.find(&MR.getTargetJITDylib()); + assert(I != MP.JITDylibToHeaderAddr.end() && "No header registered for JD"); + assert(I->second && "Null header registered for JD"); + HeaderAddr = I->second; + } + + SymbolTableVector LocalSymTab; + auto &SymTab = LLVM_LIKELY(!InBootstrapPhase) ? LocalSymTab + : MP.Bootstrap.load()->SymTab; + for (auto &[OriginalSymbol, NameSym] : JITSymTabInfo) + SymTab.push_back({NameSym->getAddress(), OriginalSymbol->getAddress(), + flagsForSymbol(*OriginalSymbol)}); + + // Bail out if we're in the bootstrap phase -- registration of thees symbols + // will be attached to the bootstrap graph. + if (LLVM_UNLIKELY(InBootstrapPhase)) + return Error::success(); + + shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase) + ? G.allocActions() + : MP.Bootstrap.load()->DeferredAAs; + allocActions.push_back( + {cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( + MP.RegisterObjectSymbolTable.Addr, HeaderAddr, SymTab)), + cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( + MP.DeregisterObjectSymbolTable.Addr, HeaderAddr, SymTab))}); + + return Error::success(); +} + +template <typename MachOTraits> +jitlink::Block &createTrivialHeaderBlock(MachOPlatform &MOP, + jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + auto HdrInfo = + getMachOHeaderInfoFromTriple(MOP.getExecutionSession().getTargetTriple()); + MachOBuilder<MachOTraits> B(HdrInfo.PageSize); + + B.Header.filetype = MachO::MH_DYLIB; + B.Header.cputype = HdrInfo.CPUType; + B.Header.cpusubtype = HdrInfo.CPUSubType; + + auto HeaderContent = G.allocateBuffer(B.layout()); + B.write(HeaderContent); + + return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, + 0); +} + +SimpleMachOHeaderMU::SimpleMachOHeaderMU(MachOPlatform &MOP, + SymbolStringPtr HeaderStartSymbol) + : MaterializationUnit( + createHeaderInterface(MOP, std::move(HeaderStartSymbol))), + MOP(MOP) {} + +void SimpleMachOHeaderMU::materialize( + std::unique_ptr<MaterializationResponsibility> R) { + auto G = createPlatformGraph(MOP, "<MachOHeaderMU>"); + addMachOHeader(R->getTargetJITDylib(), *G, R->getInitializerSymbol()); + MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); +} + +void SimpleMachOHeaderMU::discard(const JITDylib &JD, + const SymbolStringPtr &Sym) {} + +void SimpleMachOHeaderMU::addMachOHeader( + JITDylib &JD, jitlink::LinkGraph &G, + const SymbolStringPtr &InitializerSymbol) { + auto &HeaderSection = G.createSection("__header", MemProt::Read); + auto &HeaderBlock = createHeaderBlock(JD, G, HeaderSection); + + // Init symbol is header-start symbol. + G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol, HeaderBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, false, + true); + for (auto &HS : AdditionalHeaderSymbols) + G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, false, + true); +} + +jitlink::Block & +SimpleMachOHeaderMU::createHeaderBlock(JITDylib &JD, jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + switch (MOP.getExecutionSession().getTargetTriple().getArch()) { + case Triple::aarch64: + case Triple::x86_64: + return createTrivialHeaderBlock<MachO64LE>(MOP, G, HeaderSection); + default: + llvm_unreachable("Unsupported architecture"); + } +} + +MaterializationUnit::Interface SimpleMachOHeaderMU::createHeaderInterface( + MachOPlatform &MOP, const SymbolStringPtr &HeaderStartSymbol) { + SymbolFlagsMap HeaderSymbolFlags; + + HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; + for (auto &HS : AdditionalHeaderSymbols) + HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = + JITSymbolFlags::Exported; + + return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), + HeaderStartSymbol); +} + +MachOHeaderInfo getMachOHeaderInfoFromTriple(const Triple &TT) { + switch (TT.getArch()) { + case Triple::aarch64: + return {/* PageSize = */ 16 * 1024, + /* CPUType = */ MachO::CPU_TYPE_ARM64, + /* CPUSubType = */ MachO::CPU_SUBTYPE_ARM64_ALL}; + case Triple::x86_64: + return {/* PageSize = */ 4 * 1024, + /* CPUType = */ MachO::CPU_TYPE_X86_64, + /* CPUSubType = */ MachO::CPU_SUBTYPE_X86_64_ALL}; + default: + llvm_unreachable("Unrecognized architecture"); + } +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp index ca4950077ffe..9cfe547c84c3 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -322,8 +322,8 @@ void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); tpctypes::SharedMemorySegFinalizeRequest SegReq; - SegReq.RAG = {Segment.AG.getMemProt(), Segment.AG.getMemLifetimePolicy() == - MemLifetimePolicy::Finalize}; + SegReq.RAG = {Segment.AG.getMemProt(), + Segment.AG.getMemLifetime() == MemLifetime::Finalize}; SegReq.Addr = AI.MappingBase + Segment.Offset; SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp index 7c8fa63477d0..0286b0c93197 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp @@ -72,7 +72,7 @@ getMachOObjectFileSymbolInfo(ExecutionSession &ES, return SymFlags.takeError(); // Strip the 'exported' flag from MachO linker-private symbols. - if (Name->startswith("l")) + if (Name->starts_with("l")) *SymFlags &= ~JITSymbolFlags::Exported; I.SymbolFlags[ES.intern(*Name)] = std::move(*SymFlags); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index a29f3d1c3aec..3d77f82e6569 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -46,7 +46,7 @@ ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) { case Triple::armeb: case Triple::thumb: case Triple::thumbeb: - if (Sym.hasTargetFlags(aarch32::ThumbSymbol)) { + if (hasTargetFlags(Sym, aarch32::ThumbSymbol)) { // Set LSB to indicate thumb target assert(Sym.isCallable() && "Only callable symbols can have thumb flag"); assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear"); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp index a73aec6d98c6..72314cceedf3 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -27,42 +27,6 @@ class InProgressLookupState; class OrcV2CAPIHelper { public: - using PoolEntry = SymbolStringPtr::PoolEntry; - using PoolEntryPtr = SymbolStringPtr::PoolEntryPtr; - - // Move from SymbolStringPtr to PoolEntryPtr (no change in ref count). - static PoolEntryPtr moveFromSymbolStringPtr(SymbolStringPtr S) { - PoolEntryPtr Result = nullptr; - std::swap(Result, S.S); - return Result; - } - - // Move from a PoolEntryPtr to a SymbolStringPtr (no change in ref count). - static SymbolStringPtr moveToSymbolStringPtr(PoolEntryPtr P) { - SymbolStringPtr S; - S.S = P; - return S; - } - - // Copy a pool entry to a SymbolStringPtr (increments ref count). - static SymbolStringPtr copyToSymbolStringPtr(PoolEntryPtr P) { - return SymbolStringPtr(P); - } - - static PoolEntryPtr getRawPoolEntryPtr(const SymbolStringPtr &S) { - return S.S; - } - - static void retainPoolEntry(PoolEntryPtr P) { - SymbolStringPtr S(P); - S.S = nullptr; - } - - static void releasePoolEntry(PoolEntryPtr P) { - SymbolStringPtr S; - S.S = P; - } - static InProgressLookupState *extractLookupState(LookupState &LS) { return LS.IPLS.release(); } @@ -75,10 +39,16 @@ public: } // namespace orc } // namespace llvm +inline LLVMOrcSymbolStringPoolEntryRef wrap(SymbolStringPoolEntryUnsafe E) { + return reinterpret_cast<LLVMOrcSymbolStringPoolEntryRef>(E.rawPtr()); +} + +inline SymbolStringPoolEntryUnsafe unwrap(LLVMOrcSymbolStringPoolEntryRef E) { + return reinterpret_cast<SymbolStringPoolEntryUnsafe::PoolEntry *>(E); +} + DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ExecutionSession, LLVMOrcExecutionSessionRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SymbolStringPool, LLVMOrcSymbolStringPoolRef) -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcV2CAPIHelper::PoolEntry, - LLVMOrcSymbolStringPoolEntryRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(MaterializationUnit, LLVMOrcMaterializationUnitRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(MaterializationResponsibility, @@ -136,7 +106,7 @@ public: private: void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { - Discard(Ctx, wrap(&JD), wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); + Discard(Ctx, wrap(&JD), wrap(SymbolStringPoolEntryUnsafe::from(Name))); } std::string Name; @@ -184,7 +154,7 @@ static SymbolMap toSymbolMap(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs) { SymbolMap SM; for (size_t I = 0; I != NumPairs; ++I) { JITSymbolFlags Flags = toJITSymbolFlags(Syms[I].Sym.Flags); - SM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = { + SM[unwrap(Syms[I].Name).moveToSymbolStringPtr()] = { ExecutorAddr(Syms[I].Sym.Address), Flags}; } return SM; @@ -199,7 +169,7 @@ toSymbolDependenceMap(LLVMOrcCDependenceMapPairs Pairs, size_t NumPairs) { for (size_t J = 0; J != Pairs[I].Names.Length; ++J) { auto Sym = Pairs[I].Names.Symbols[J]; - Names.insert(OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Sym))); + Names.insert(unwrap(Sym).moveToSymbolStringPtr()); } SDM[JD] = Names; } @@ -309,7 +279,7 @@ public: CLookupSet.reserve(LookupSet.size()); for (auto &KV : LookupSet) { LLVMOrcSymbolStringPoolEntryRef Name = - ::wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(KV.first)); + ::wrap(SymbolStringPoolEntryUnsafe::from(KV.first)); LLVMOrcSymbolLookupFlags SLF = fromSymbolLookupFlags(KV.second); CLookupSet.push_back({Name, SLF}); } @@ -353,8 +323,7 @@ void LLVMOrcSymbolStringPoolClearDeadEntries(LLVMOrcSymbolStringPoolRef SSP) { LLVMOrcSymbolStringPoolEntryRef LLVMOrcExecutionSessionIntern(LLVMOrcExecutionSessionRef ES, const char *Name) { - return wrap( - OrcV2CAPIHelper::moveFromSymbolStringPtr(unwrap(ES)->intern(Name))); + return wrap(SymbolStringPoolEntryUnsafe::take(unwrap(ES)->intern(Name))); } void LLVMOrcExecutionSessionLookup( @@ -374,7 +343,7 @@ void LLVMOrcExecutionSessionLookup( SymbolLookupSet SLS; for (size_t I = 0; I != SymbolsSize; ++I) - SLS.add(OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Symbols[I].Name)), + SLS.add(unwrap(Symbols[I].Name).moveToSymbolStringPtr(), toSymbolLookupFlags(Symbols[I].LookupFlags)); unwrap(ES)->lookup( @@ -384,7 +353,7 @@ void LLVMOrcExecutionSessionLookup( SmallVector<LLVMOrcCSymbolMapPair> CResult; for (auto &KV : *Result) CResult.push_back(LLVMOrcCSymbolMapPair{ - wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(KV.first)), + wrap(SymbolStringPoolEntryUnsafe::from(KV.first)), fromExecutorSymbolDef(KV.second)}); HandleResult(LLVMErrorSuccess, CResult.data(), CResult.size(), Ctx); } else @@ -394,15 +363,15 @@ void LLVMOrcExecutionSessionLookup( } void LLVMOrcRetainSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { - OrcV2CAPIHelper::retainPoolEntry(unwrap(S)); + unwrap(S).retain(); } void LLVMOrcReleaseSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { - OrcV2CAPIHelper::releasePoolEntry(unwrap(S)); + unwrap(S).release(); } const char *LLVMOrcSymbolStringPoolEntryStr(LLVMOrcSymbolStringPoolEntryRef S) { - return unwrap(S)->getKey().data(); + return unwrap(S).rawPtr()->getKey().data(); } LLVMOrcResourceTrackerRef @@ -452,10 +421,10 @@ LLVMOrcMaterializationUnitRef LLVMOrcCreateCustomMaterializationUnit( LLVMOrcMaterializationUnitDestroyFunction Destroy) { SymbolFlagsMap SFM; for (size_t I = 0; I != NumSyms; ++I) - SFM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = + SFM[unwrap(Syms[I].Name).moveToSymbolStringPtr()] = toJITSymbolFlags(Syms[I].Flags); - auto IS = OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(InitSym)); + auto IS = unwrap(InitSym).moveToSymbolStringPtr(); return wrap(new OrcCAPIMaterializationUnit( Name, std::move(SFM), std::move(IS), Ctx, Materialize, Discard, Destroy)); @@ -476,9 +445,8 @@ LLVMOrcMaterializationUnitRef LLVMOrcLazyReexports( for (size_t I = 0; I != NumPairs; ++I) { auto pair = CallableAliases[I]; JITSymbolFlags Flags = toJITSymbolFlags(pair.Entry.Flags); - SymbolStringPtr Name = - OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(pair.Entry.Name)); - SAM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(pair.Name))] = + SymbolStringPtr Name = unwrap(pair.Entry.Name).moveToSymbolStringPtr(); + SAM[unwrap(pair.Name).moveToSymbolStringPtr()] = SymbolAliasMapEntry(Name, Flags); } @@ -511,7 +479,7 @@ LLVMOrcCSymbolFlagsMapPairs LLVMOrcMaterializationResponsibilityGetSymbols( safe_malloc(Symbols.size() * sizeof(LLVMOrcCSymbolFlagsMapPair))); size_t I = 0; for (auto const &pair : Symbols) { - auto Name = wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(pair.first)); + auto Name = wrap(SymbolStringPoolEntryUnsafe::from(pair.first)); auto Flags = pair.second; Result[I] = {Name, fromJITSymbolFlags(Flags)}; I++; @@ -528,7 +496,7 @@ LLVMOrcSymbolStringPoolEntryRef LLVMOrcMaterializationResponsibilityGetInitializerSymbol( LLVMOrcMaterializationResponsibilityRef MR) { auto Sym = unwrap(MR)->getInitializerSymbol(); - return wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Sym)); + return wrap(SymbolStringPoolEntryUnsafe::from(Sym)); } LLVMOrcSymbolStringPoolEntryRef * @@ -541,7 +509,7 @@ LLVMOrcMaterializationResponsibilityGetRequestedSymbols( Symbols.size() * sizeof(LLVMOrcSymbolStringPoolEntryRef))); size_t I = 0; for (auto &Name : Symbols) { - Result[I] = wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name)); + Result[I] = wrap(SymbolStringPoolEntryUnsafe::from(Name)); I++; } *NumSymbols = Symbols.size(); @@ -569,7 +537,7 @@ LLVMErrorRef LLVMOrcMaterializationResponsibilityDefineMaterializing( LLVMOrcCSymbolFlagsMapPairs Syms, size_t NumSyms) { SymbolFlagsMap SFM; for (size_t I = 0; I != NumSyms; ++I) - SFM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = + SFM[unwrap(Syms[I].Name).moveToSymbolStringPtr()] = toJITSymbolFlags(Syms[I].Flags); return wrap(unwrap(MR)->defineMaterializing(std::move(SFM))); @@ -588,7 +556,7 @@ LLVMErrorRef LLVMOrcMaterializationResponsibilityDelegate( LLVMOrcMaterializationResponsibilityRef *Result) { SymbolNameSet Syms; for (size_t I = 0; I != NumSymbols; I++) { - Syms.insert(OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Symbols[I]))); + Syms.insert(unwrap(Symbols[I]).moveToSymbolStringPtr()); } auto OtherMR = unwrap(MR)->delegate(Syms); @@ -605,7 +573,7 @@ void LLVMOrcMaterializationResponsibilityAddDependencies( LLVMOrcCDependenceMapPairs Dependencies, size_t NumPairs) { SymbolDependenceMap SDM = toSymbolDependenceMap(Dependencies, NumPairs); - auto Sym = OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Name)); + auto Sym = unwrap(Name).moveToSymbolStringPtr(); unwrap(MR)->addDependencies(Sym, SDM); } @@ -698,7 +666,7 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( DynamicLibrarySearchGenerator::SymbolPredicate Pred; if (Filter) Pred = [=](const SymbolStringPtr &Name) -> bool { - return Filter(FilterCtx, wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); + return Filter(FilterCtx, wrap(SymbolStringPoolEntryUnsafe::from(Name))); }; auto ProcessSymsGenerator = @@ -724,7 +692,7 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForPath( DynamicLibrarySearchGenerator::SymbolPredicate Pred; if (Filter) Pred = [=](const SymbolStringPtr &Name) -> bool { - return Filter(FilterCtx, wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); + return Filter(FilterCtx, wrap(SymbolStringPoolEntryUnsafe::from(Name))); }; auto LibrarySymsGenerator = @@ -992,7 +960,7 @@ char LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J) { LLVMOrcSymbolStringPoolEntryRef LLVMOrcLLJITMangleAndIntern(LLVMOrcLLJITRef J, const char *UnmangledName) { - return wrap(OrcV2CAPIHelper::moveFromSymbolStringPtr( + return wrap(SymbolStringPoolEntryUnsafe::take( unwrap(J)->mangleAndIntern(UnmangledName))); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 9ef333222028..f9630161b95e 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -233,7 +233,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(&Obj)) { auto &ES = getExecutionSession(); - // For all resolved symbols that are not already in the responsibilty set: + // For all resolved symbols that are not already in the responsibility set: // check whether the symbol is in a comdat section and if so mark it as // weak. for (auto &Sym : COFFObj->symbols()) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp index ecf5e2915773..a407fcab6ae3 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp @@ -19,6 +19,7 @@ StringRef MachODataCommonSectionName = "__DATA,__common"; StringRef MachODataDataSectionName = "__DATA,__data"; StringRef MachOEHFrameSectionName = "__TEXT,__eh_frame"; StringRef MachOCompactUnwindInfoSectionName = "__TEXT,__unwind_info"; +StringRef MachOCStringSectionName = "__TEXT,__cstring"; StringRef MachOModInitFuncSectionName = "__DATA,__mod_init_func"; StringRef MachOObjCCatListSectionName = "__DATA,__objc_catlist"; StringRef MachOObjCCatList2SectionName = "__DATA,__objc_catlist2"; @@ -56,7 +57,19 @@ StringRef MachOInitSectionNames[19] = { }; StringRef ELFEHFrameSectionName = ".eh_frame"; + StringRef ELFInitArrayFuncSectionName = ".init_array"; +StringRef ELFInitFuncSectionName = ".init"; +StringRef ELFFiniArrayFuncSectionName = ".fini_array"; +StringRef ELFFiniFuncSectionName = ".fini"; +StringRef ELFCtorArrayFuncSectionName = ".ctors"; +StringRef ELFDtorArrayFuncSectionName = ".dtors"; + +StringRef ELFInitSectionNames[3]{ + ELFInitArrayFuncSectionName, + ELFInitFuncSectionName, + ELFCtorArrayFuncSectionName, +}; StringRef ELFThreadBSSSectionName = ".tbss"; StringRef ELFThreadDataSectionName = ".tdata"; @@ -80,14 +93,16 @@ bool isMachOInitializerSection(StringRef QualifiedName) { } bool isELFInitializerSection(StringRef SecName) { - if (SecName.consume_front(ELFInitArrayFuncSectionName) && - (SecName.empty() || SecName[0] == '.')) - return true; + for (StringRef InitSection : ELFInitSectionNames) { + StringRef Name = SecName; + if (Name.consume_front(InitSection) && (Name.empty() || Name[0] == '.')) + return true; + } return false; } bool isCOFFInitializerSection(StringRef SecName) { - return SecName.startswith(".CRT"); + return SecName.starts_with(".CRT"); } } // namespace orc diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp index 86e31c52100e..ae39b1d1bfaa 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -51,9 +51,9 @@ const char *MemoryWriteBuffersWrapperName = "__llvm_orc_bootstrap_mem_write_buffers_wrapper"; const char *RegisterEHFrameSectionWrapperName = - "__llvm_orc_bootstrap_register_ehframe_section_wrapper"; + "llvm_orc_registerEHFrameSectionWrapper"; const char *DeregisterEHFrameSectionWrapperName = - "__llvm_orc_bootstrap_deregister_ehframe_section_wrapper"; + "llvm_orc_deregisterEHFrameSectionWrapper"; const char *RunAsMainWrapperName = "__llvm_orc_bootstrap_run_as_main_wrapper"; const char *RunAsVoidFunctionWrapperName = diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp index 0388725dfb63..8f42de91b5bb 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp @@ -10,7 +10,6 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" @@ -227,7 +226,7 @@ void SequenceBBQuery::traverseToExitBlock(const BasicBlock *AtBB, VisitedBlocks); } -// Get Block frequencies for blocks and take most frquently executed block, +// Get Block frequencies for blocks and take most frequently executed block, // walk towards the entry block from those blocks and discover the basic blocks // with call. SequenceBBQuery::BlockListTy diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp index d4cbd1970d8f..70b536d2feda 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Speculation.cpp @@ -67,7 +67,7 @@ void IRSpeculationLayer::emit(std::unique_ptr<MaterializationResponsibility> R, auto SpeculatorVTy = StructType::create(MContext, "Class.Speculator"); auto RuntimeCallTy = FunctionType::get( Type::getVoidTy(MContext), - {SpeculatorVTy->getPointerTo(), Type::getInt64Ty(MContext)}, false); + {PointerType::getUnqual(MContext), Type::getInt64Ty(MContext)}, false); auto RuntimeCall = Function::Create(RuntimeCallTy, Function::LinkageTypes::ExternalLinkage, "__orc_speculate_for", &M); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp index 3f70dbf60437..e8b0e240ac1f 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp @@ -194,9 +194,7 @@ Error ExecutorSharedMemoryMapperService::deinitialize( // Remove the allocation from the allocation list of its reservation for (auto &Reservation : Reservations) { - auto AllocationIt = - std::find(Reservation.second.Allocations.begin(), - Reservation.second.Allocations.end(), Base); + auto AllocationIt = llvm::find(Reservation.second.Allocations, Base); if (AllocationIt != Reservation.second.Allocations.end()) { Reservation.second.Allocations.erase(AllocationIt); break; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.cpp new file mode 100644 index 000000000000..5e0623102d33 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.cpp @@ -0,0 +1,457 @@ +//===------- JITLoaderPerf.cpp - Register profiler objects ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Register objects for access by profilers via the perf JIT interface. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h" + +#include "llvm/ExecutionEngine/Orc/Shared/PerfSharedStructs.h" + +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Threading.h" + +#include <mutex> +#include <optional> + +#ifdef __linux__ + +#include <sys/mman.h> // mmap() +#include <time.h> // clock_gettime(), time(), localtime_r() */ +#include <unistd.h> // for read(), close() + +#define DEBUG_TYPE "orc" + +// language identifier (XXX: should we generate something better from debug +// info?) +#define JIT_LANG "llvm-IR" +#define LLVM_PERF_JIT_MAGIC \ + ((uint32_t)'J' << 24 | (uint32_t)'i' << 16 | (uint32_t)'T' << 8 | \ + (uint32_t)'D') +#define LLVM_PERF_JIT_VERSION 1 + +using namespace llvm; +using namespace llvm::orc; + +struct PerfState { + // cache lookups + uint32_t Pid; + + // base directory for output data + std::string JitPath; + + // output data stream, closed via Dumpstream + int DumpFd = -1; + + // output data stream + std::unique_ptr<raw_fd_ostream> Dumpstream; + + // perf mmap marker + void *MarkerAddr = NULL; +}; + +// prevent concurrent dumps from messing up the output file +static std::mutex Mutex; +static std::optional<PerfState> State; + +struct RecHeader { + uint32_t Id; + uint32_t TotalSize; + uint64_t Timestamp; +}; + +struct DIR { + RecHeader Prefix; + uint64_t CodeAddr; + uint64_t NrEntry; +}; + +struct DIE { + uint64_t CodeAddr; + uint32_t Line; + uint32_t Discrim; +}; + +struct CLR { + RecHeader Prefix; + uint32_t Pid; + uint32_t Tid; + uint64_t Vma; + uint64_t CodeAddr; + uint64_t CodeSize; + uint64_t CodeIndex; +}; + +struct UWR { + RecHeader Prefix; + uint64_t UnwindDataSize; + uint64_t EhFrameHeaderSize; + uint64_t MappedSize; +}; + +static inline uint64_t timespec_to_ns(const struct timespec *TS) { + const uint64_t NanoSecPerSec = 1000000000; + return ((uint64_t)TS->tv_sec * NanoSecPerSec) + TS->tv_nsec; +} + +static inline uint64_t perf_get_timestamp() { + timespec TS; + if (clock_gettime(CLOCK_MONOTONIC, &TS)) + return 0; + + return timespec_to_ns(&TS); +} + +static void writeDebugRecord(const PerfJITDebugInfoRecord &DebugRecord) { + assert(State && "PerfState not initialized"); + LLVM_DEBUG(dbgs() << "Writing debug record with " + << DebugRecord.Entries.size() << " entries\n"); + [[maybe_unused]] size_t Written = 0; + DIR Dir{RecHeader{static_cast<uint32_t>(DebugRecord.Prefix.Id), + DebugRecord.Prefix.TotalSize, perf_get_timestamp()}, + DebugRecord.CodeAddr, DebugRecord.Entries.size()}; + State->Dumpstream->write(reinterpret_cast<const char *>(&Dir), sizeof(Dir)); + Written += sizeof(Dir); + for (auto &Die : DebugRecord.Entries) { + DIE d{Die.Addr, Die.Lineno, Die.Discrim}; + State->Dumpstream->write(reinterpret_cast<const char *>(&d), sizeof(d)); + State->Dumpstream->write(Die.Name.data(), Die.Name.size() + 1); + Written += sizeof(d) + Die.Name.size() + 1; + } + LLVM_DEBUG(dbgs() << "wrote " << Written << " bytes of debug info\n"); +} + +static void writeCodeRecord(const PerfJITCodeLoadRecord &CodeRecord) { + assert(State && "PerfState not initialized"); + uint32_t Tid = get_threadid(); + LLVM_DEBUG(dbgs() << "Writing code record with code size " + << CodeRecord.CodeSize << " and code index " + << CodeRecord.CodeIndex << "\n"); + CLR Clr{RecHeader{static_cast<uint32_t>(CodeRecord.Prefix.Id), + CodeRecord.Prefix.TotalSize, perf_get_timestamp()}, + State->Pid, + Tid, + CodeRecord.Vma, + CodeRecord.CodeAddr, + CodeRecord.CodeSize, + CodeRecord.CodeIndex}; + LLVM_DEBUG(dbgs() << "wrote " << sizeof(Clr) << " bytes of CLR, " + << CodeRecord.Name.size() + 1 << " bytes of name, " + << CodeRecord.CodeSize << " bytes of code\n"); + State->Dumpstream->write(reinterpret_cast<const char *>(&Clr), sizeof(Clr)); + State->Dumpstream->write(CodeRecord.Name.data(), CodeRecord.Name.size() + 1); + State->Dumpstream->write((const char *)CodeRecord.CodeAddr, + CodeRecord.CodeSize); +} + +static void +writeUnwindRecord(const PerfJITCodeUnwindingInfoRecord &UnwindRecord) { + assert(State && "PerfState not initialized"); + dbgs() << "Writing unwind record with unwind data size " + << UnwindRecord.UnwindDataSize << " and EH frame header size " + << UnwindRecord.EHFrameHdrSize << " and mapped size " + << UnwindRecord.MappedSize << "\n"; + UWR Uwr{RecHeader{static_cast<uint32_t>(UnwindRecord.Prefix.Id), + UnwindRecord.Prefix.TotalSize, perf_get_timestamp()}, + UnwindRecord.UnwindDataSize, UnwindRecord.EHFrameHdrSize, + UnwindRecord.MappedSize}; + LLVM_DEBUG(dbgs() << "wrote " << sizeof(Uwr) << " bytes of UWR, " + << UnwindRecord.EHFrameHdrSize + << " bytes of EH frame header, " + << UnwindRecord.UnwindDataSize - UnwindRecord.EHFrameHdrSize + << " bytes of EH frame\n"); + State->Dumpstream->write(reinterpret_cast<const char *>(&Uwr), sizeof(Uwr)); + if (UnwindRecord.EHFrameHdrAddr) + State->Dumpstream->write((const char *)UnwindRecord.EHFrameHdrAddr, + UnwindRecord.EHFrameHdrSize); + else + State->Dumpstream->write(UnwindRecord.EHFrameHdr.data(), + UnwindRecord.EHFrameHdrSize); + State->Dumpstream->write((const char *)UnwindRecord.EHFrameAddr, + UnwindRecord.UnwindDataSize - + UnwindRecord.EHFrameHdrSize); +} + +static Error registerJITLoaderPerfImpl(const PerfJITRecordBatch &Batch) { + if (!State) + return make_error<StringError>("PerfState not initialized", + inconvertibleErrorCode()); + + // Serialize the batch + std::lock_guard<std::mutex> Lock(Mutex); + if (Batch.UnwindingRecord.Prefix.TotalSize > 0) + writeUnwindRecord(Batch.UnwindingRecord); + + for (const auto &DebugInfo : Batch.DebugInfoRecords) + writeDebugRecord(DebugInfo); + + for (const auto &CodeLoad : Batch.CodeLoadRecords) + writeCodeRecord(CodeLoad); + + State->Dumpstream->flush(); + + return Error::success(); +} + +struct Header { + uint32_t Magic; // characters "JiTD" + uint32_t Version; // header version + uint32_t TotalSize; // total size of header + uint32_t ElfMach; // elf mach target + uint32_t Pad1; // reserved + uint32_t Pid; + uint64_t Timestamp; // timestamp + uint64_t Flags; // flags +}; + +static Error OpenMarker(PerfState &State) { + // We mmap the jitdump to create an MMAP RECORD in perf.data file. The mmap + // is captured either live (perf record running when we mmap) or in deferred + // mode, via /proc/PID/maps. The MMAP record is used as a marker of a jitdump + // file for more meta data info about the jitted code. Perf report/annotate + // detect this special filename and process the jitdump file. + // + // Mapping must be PROT_EXEC to ensure it is captured by perf record + // even when not using -d option. + State.MarkerAddr = + ::mmap(NULL, sys::Process::getPageSizeEstimate(), PROT_READ | PROT_EXEC, + MAP_PRIVATE, State.DumpFd, 0); + + if (State.MarkerAddr == MAP_FAILED) + return make_error<llvm::StringError>("could not mmap JIT marker", + inconvertibleErrorCode()); + + return Error::success(); +} + +void CloseMarker(PerfState &State) { + if (!State.MarkerAddr) + return; + + munmap(State.MarkerAddr, sys::Process::getPageSizeEstimate()); + State.MarkerAddr = nullptr; +} + +static Expected<Header> FillMachine(PerfState &State) { + Header Hdr; + Hdr.Magic = LLVM_PERF_JIT_MAGIC; + Hdr.Version = LLVM_PERF_JIT_VERSION; + Hdr.TotalSize = sizeof(Hdr); + Hdr.Pid = State.Pid; + Hdr.Timestamp = perf_get_timestamp(); + + char Id[16]; + struct { + uint16_t e_type; + uint16_t e_machine; + } Info; + + size_t RequiredMemory = sizeof(Id) + sizeof(Info); + + ErrorOr<std::unique_ptr<MemoryBuffer>> MB = + MemoryBuffer::getFileSlice("/proc/self/exe", RequiredMemory, 0); + + // This'll not guarantee that enough data was actually read from the + // underlying file. Instead the trailing part of the buffer would be + // zeroed. Given the ELF signature check below that seems ok though, + // it's unlikely that the file ends just after that, and the + // consequence would just be that perf wouldn't recognize the + // signature. + if (!MB) + return make_error<llvm::StringError>("could not open /proc/self/exe", + MB.getError()); + + memcpy(&Id, (*MB)->getBufferStart(), sizeof(Id)); + memcpy(&Info, (*MB)->getBufferStart() + sizeof(Id), sizeof(Info)); + + // check ELF signature + if (Id[0] != 0x7f || Id[1] != 'E' || Id[2] != 'L' || Id[3] != 'F') + return make_error<llvm::StringError>("invalid ELF signature", + inconvertibleErrorCode()); + + Hdr.ElfMach = Info.e_machine; + + return Hdr; +} + +static Error InitDebuggingDir(PerfState &State) { + time_t Time; + struct tm LocalTime; + char TimeBuffer[sizeof("YYYYMMDD")]; + SmallString<64> Path; + + // search for location to dump data to + if (const char *BaseDir = getenv("JITDUMPDIR")) + Path.append(BaseDir); + else if (!sys::path::home_directory(Path)) + Path = "."; + + // create debug directory + Path += "/.debug/jit/"; + if (auto EC = sys::fs::create_directories(Path)) { + std::string ErrStr; + raw_string_ostream ErrStream(ErrStr); + ErrStream << "could not create jit cache directory " << Path << ": " + << EC.message() << "\n"; + return make_error<StringError>(std::move(ErrStr), inconvertibleErrorCode()); + } + + // create unique directory for dump data related to this process + time(&Time); + localtime_r(&Time, &LocalTime); + strftime(TimeBuffer, sizeof(TimeBuffer), "%Y%m%d", &LocalTime); + Path += JIT_LANG "-jit-"; + Path += TimeBuffer; + + SmallString<128> UniqueDebugDir; + + using sys::fs::createUniqueDirectory; + if (auto EC = createUniqueDirectory(Path, UniqueDebugDir)) { + std::string ErrStr; + raw_string_ostream ErrStream(ErrStr); + ErrStream << "could not create unique jit cache directory " + << UniqueDebugDir << ": " << EC.message() << "\n"; + return make_error<StringError>(std::move(ErrStr), inconvertibleErrorCode()); + } + + State.JitPath = std::string(UniqueDebugDir.str()); + + return Error::success(); +} + +static Error registerJITLoaderPerfStartImpl() { + PerfState Tentative; + Tentative.Pid = sys::Process::getProcessId(); + // check if clock-source is supported + if (!perf_get_timestamp()) + return make_error<StringError>("kernel does not support CLOCK_MONOTONIC", + inconvertibleErrorCode()); + + if (auto Err = InitDebuggingDir(Tentative)) + return Err; + + std::string Filename; + raw_string_ostream FilenameBuf(Filename); + FilenameBuf << Tentative.JitPath << "/jit-" << Tentative.Pid << ".dump"; + + // Need to open ourselves, because we need to hand the FD to OpenMarker() and + // raw_fd_ostream doesn't expose the FD. + using sys::fs::openFileForWrite; + if (auto EC = openFileForReadWrite(FilenameBuf.str(), Tentative.DumpFd, + sys::fs::CD_CreateNew, sys::fs::OF_None)) { + std::string ErrStr; + raw_string_ostream ErrStream(ErrStr); + ErrStream << "could not open JIT dump file " << FilenameBuf.str() << ": " + << EC.message() << "\n"; + return make_error<StringError>(std::move(ErrStr), inconvertibleErrorCode()); + } + + Tentative.Dumpstream = + std::make_unique<raw_fd_ostream>(Tentative.DumpFd, true); + + auto Header = FillMachine(Tentative); + if (!Header) + return Header.takeError(); + + // signal this process emits JIT information + if (auto Err = OpenMarker(Tentative)) + return Err; + + Tentative.Dumpstream->write(reinterpret_cast<const char *>(&Header.get()), + sizeof(*Header)); + + // Everything initialized, can do profiling now. + if (Tentative.Dumpstream->has_error()) + return make_error<StringError>("could not write JIT dump header", + inconvertibleErrorCode()); + + State = std::move(Tentative); + return Error::success(); +} + +static Error registerJITLoaderPerfEndImpl() { + if (!State) + return make_error<StringError>("PerfState not initialized", + inconvertibleErrorCode()); + + RecHeader Close; + Close.Id = static_cast<uint32_t>(PerfJITRecordType::JIT_CODE_CLOSE); + Close.TotalSize = sizeof(Close); + Close.Timestamp = perf_get_timestamp(); + State->Dumpstream->write(reinterpret_cast<const char *>(&Close), + sizeof(Close)); + if (State->MarkerAddr) + CloseMarker(*State); + + State.reset(); + return Error::success(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction<SPSError(SPSPerfJITRecordBatch)>::handle( + Data, Size, registerJITLoaderPerfImpl) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfStart(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction<SPSError()>::handle(Data, Size, + registerJITLoaderPerfStartImpl) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfEnd(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction<SPSError()>::handle(Data, Size, + registerJITLoaderPerfEndImpl) + .release(); +} + +#else + +using namespace llvm; +using namespace llvm::orc; + +static Error badOS() { + using namespace llvm; + return llvm::make_error<StringError>( + "unsupported OS (perf support is only available on linux!)", + inconvertibleErrorCode()); +} + +static Error badOSBatch(PerfJITRecordBatch &Batch) { return badOS(); } + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfImpl(const char *Data, uint64_t Size) { + using namespace shared; + return WrapperFunction<SPSError(SPSPerfJITRecordBatch)>::handle(Data, Size, + badOSBatch) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfStart(const char *Data, uint64_t Size) { + using namespace shared; + return WrapperFunction<SPSError()>::handle(Data, Size, badOS).release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfEnd(const char *Data, uint64_t Size) { + using namespace shared; + return WrapperFunction<SPSError()>::handle(Data, Size, badOS).release(); +} + +#endif diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp index 67bc379f9821..a585767bf474 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp @@ -8,7 +8,9 @@ #include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Process.h" #include "llvm/TargetParser/Host.h" @@ -206,6 +208,10 @@ Error SimpleRemoteEPCServer::sendSetupMessage( "Dispatch function name should not be set"); EI.BootstrapSymbols[ExecutorSessionObjectName] = ExecutorAddr::fromPtr(this); EI.BootstrapSymbols[DispatchFnName] = ExecutorAddr::fromPtr(jitDispatchEntry); + EI.BootstrapSymbols[rt::RegisterEHFrameSectionWrapperName] = + ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper); + EI.BootstrapSymbols[rt::DeregisterEHFrameSectionWrapperName] = + ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper); using SPSSerialize = shared::SPSArgList<shared::SPSSimpleRemoteEPCExecutorInfo>; |
