diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ExecutionEngine')
53 files changed, 4886 insertions, 2317 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp index 7ed025fbb481..e15bce0d6c4b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp @@ -47,21 +47,28 @@ extern "C" { // We put information about the JITed function in this global, which the // debugger reads. Make sure to specify the version statically, because the // debugger checks the version before we can set it during runtime. - struct jit_descriptor __jit_debug_descriptor = { 1, 0, nullptr, nullptr }; + extern struct jit_descriptor __jit_debug_descriptor; // Debuggers puts a breakpoint in this function. - LLVM_ATTRIBUTE_NOINLINE void __jit_debug_register_code() { - // The noinline and the asm prevent calls to this function from being - // optimized out. -#if !defined(_MSC_VER) - asm volatile("":::"memory"); -#endif - } - + extern "C" void __jit_debug_register_code(); } namespace { +// FIXME: lli aims to provide both, RuntimeDyld and JITLink, as the dynamic +// loaders for it's JIT implementations. And they both offer debugging via the +// GDB JIT interface, which builds on the two well-known symbol names below. +// As these symbols must be unique accross the linked executable, we can only +// define them in one of the libraries and make the other depend on it. +// OrcTargetProcess is a minimal stub for embedding a JIT client in remote +// executors. For the moment it seems reasonable to have the definition there +// and let ExecutionEngine depend on it, until we find a better solution. +// +LLVM_ATTRIBUTE_USED void requiredSymbolDefinitionsFromOrcTargetProcess() { + errs() << (void *)&__jit_debug_register_code + << (void *)&__jit_debug_descriptor; +} + struct RegisteredObjectInfo { RegisteredObjectInfo() {} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp index 62e1ea6e0f0a..770fc9349083 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/Execution.cpp @@ -1017,7 +1017,7 @@ void Interpreter::SwitchToNewBasicBlock(BasicBlock *Dest, ExecutionContext &SF){ void Interpreter::visitAllocaInst(AllocaInst &I) { ExecutionContext &SF = ECStack.back(); - Type *Ty = I.getType()->getElementType(); // Type to be allocated + Type *Ty = I.getAllocatedType(); // Type to be allocated // Get the number of elements being allocated by the array... unsigned NumElements = diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index 3aa77557862e..c3ba5ebb36fb 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -130,6 +130,7 @@ static ffi_type *ffiTypeFor(Type *Ty) { case 32: return &ffi_type_sint32; case 64: return &ffi_type_sint64; } + llvm_unreachable("Unhandled integer type bitwidth"); case Type::FloatTyID: return &ffi_type_float; case Type::DoubleTyID: return &ffi_type_double; case Type::PointerTyID: return &ffi_type_pointer; @@ -166,6 +167,7 @@ static void *ffiValueFor(Type *Ty, const GenericValue &AV, return ArgDataPtr; } } + llvm_unreachable("Unhandled integer type bitwidth"); case Type::FloatTyID: { float *FloatPtr = (float *) ArgDataPtr; *FloatPtr = AV.FloatVal; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h deleted file mode 100644 index 82258a35a675..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h +++ /dev/null @@ -1,107 +0,0 @@ -//===--- BasicGOTAndStubsBuilder.h - Generic GOT/Stub creation --*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// A base for simple GOT and stub creation. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H -#define LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H - -#include "llvm/ExecutionEngine/JITLink/JITLink.h" - -#define DEBUG_TYPE "jitlink" - -namespace llvm { -namespace jitlink { - -template <typename BuilderImpl> class BasicGOTAndStubsBuilder { -public: - BasicGOTAndStubsBuilder(LinkGraph &G) : G(G) {} - - void run() { - // We're going to be adding new blocks, but we don't want to iterate over - // the newly added ones, so just copy the existing blocks out. - std::vector<Block *> Blocks(G.blocks().begin(), G.blocks().end()); - - LLVM_DEBUG(dbgs() << "Creating GOT entries and stubs:\n"); - - for (auto *B : Blocks) - for (auto &E : B->edges()) - if (impl().isGOTEdge(E)) { - LLVM_DEBUG({ - dbgs() << " Updating GOT edge "; - printEdge(dbgs(), *B, E, "<target GOT>"); - dbgs() << "\n"; - }); - impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget())); - } else if (impl().isExternalBranchEdge(E)) { - LLVM_DEBUG({ - dbgs() << " Updating external branch edge "; - printEdge(dbgs(), *B, E, "<target PC-rel>"); - dbgs() << "\n"; - }); - impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget())); - } - } - -protected: - Symbol &getGOTEntrySymbol(Symbol &Target) { - assert(Target.hasName() && "GOT edge cannot point to anonymous target"); - - auto GOTEntryI = GOTEntries.find(Target.getName()); - - // Build the entry if it doesn't exist. - if (GOTEntryI == GOTEntries.end()) { - auto &GOTEntry = impl().createGOTEntry(Target); - LLVM_DEBUG({ - dbgs() << " Created GOT entry for " << Target.getName() << ": " - << GOTEntry << "\n"; - }); - GOTEntryI = - GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; - } - - assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol"); - LLVM_DEBUG( - { dbgs() << " Using GOT entry " << *GOTEntryI->second << "\n"; }); - return *GOTEntryI->second; - } - - Symbol &getStubSymbol(Symbol &Target) { - assert(Target.hasName() && - "External branch edge can not point to an anonymous target"); - auto StubI = Stubs.find(Target.getName()); - - if (StubI == Stubs.end()) { - auto &StubSymbol = impl().createStub(Target); - LLVM_DEBUG({ - dbgs() << " Created stub for " << Target.getName() << ": " - << StubSymbol << "\n"; - }); - StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first; - } - - assert(StubI != Stubs.end() && "Count not get stub symbol"); - LLVM_DEBUG({ dbgs() << " Using stub " << *StubI->second << "\n"; }); - return *StubI->second; - } - - LinkGraph &G; - -private: - BuilderImpl &impl() { return static_cast<BuilderImpl &>(*this); } - - DenseMap<StringRef, Symbol *> GOTEntries; - DenseMap<StringRef, Symbol *> Stubs; -}; - -} // end namespace jitlink -} // end namespace llvm - -#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h new file mode 100644 index 000000000000..8ae3bc2bf61d --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/DefineExternalSectionStartAndEndSymbols.h @@ -0,0 +1,116 @@ +//===--------- DefineExternalSectionStartAndEndSymbols.h --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utility class for recognizing external section start and end symbols and +// transforming them into defined symbols for the start and end blocks of the +// associated Section. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_DEFINEEXTERNALSECTIONSTARTANDENDSYMBOLS_H +#define LLVM_EXECUTIONENGINE_JITLINK_DEFINEEXTERNALSECTIONSTARTANDENDSYMBOLS_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +struct SectionRangeSymbolDesc { + SectionRangeSymbolDesc() = default; + SectionRangeSymbolDesc(Section &Sec, bool IsStart) + : Sec(&Sec), IsStart(IsStart) {} + Section *Sec = nullptr; + bool IsStart = false; +}; + +/// Pass implementation for the createDefineExternalSectionStartAndEndSymbols +/// function. +template <typename SymbolIdentifierFunction> +class DefineExternalSectionStartAndEndSymbols { +public: + DefineExternalSectionStartAndEndSymbols(SymbolIdentifierFunction F) + : F(std::move(F)) {} + + Error operator()(LinkGraph &G) { + + // This pass will affect the external symbols set, so copy them out into a + // vector and iterate over that. + std::vector<Symbol *> Externals(G.external_symbols().begin(), + G.external_symbols().end()); + + for (auto *Sym : Externals) { + SectionRangeSymbolDesc D = F(G, *Sym); + if (D.Sec) { + auto &SR = getSectionRange(*D.Sec); + if (D.IsStart) { + if (SR.empty()) + G.makeAbsolute(*Sym, 0); + else + G.makeDefined(*Sym, *SR.getFirstBlock(), 0, 0, Linkage::Strong, + Scope::Local, false); + } else { + if (SR.empty()) + G.makeAbsolute(*Sym, 0); + else + G.makeDefined(*Sym, *SR.getLastBlock(), + SR.getLastBlock()->getSize(), 0, Linkage::Strong, + Scope::Local, false); + } + } + } + return Error::success(); + } + +private: + SectionRange &getSectionRange(Section &Sec) { + auto I = SectionRanges.find(&Sec); + if (I == SectionRanges.end()) + I = SectionRanges.insert(std::make_pair(&Sec, SectionRange(Sec))).first; + return I->second; + } + + DenseMap<Section *, SectionRange> SectionRanges; + SymbolIdentifierFunction F; +}; + +/// Returns a JITLink pass (as a function class) that uses the given symbol +/// identification function to identify external section start and end symbols +/// (and their associated Section*s) and transform the identified externals +/// into defined symbols pointing to the start of the first block in the +/// section and the end of the last (start and end symbols for empty sections +/// will be transformed into absolute symbols at address 0). +/// +/// The identification function should be callable as +/// +/// SectionRangeSymbolDesc (LinkGraph &G, Symbol &Sym) +/// +/// If Sym is not a section range start or end symbol then a default +/// constructed SectionRangeSymbolDesc should be returned. If Sym is a start +/// symbol then SectionRangeSymbolDesc(Sec, true), where Sec is a reference to +/// the target Section. If Sym is an end symbol then +/// SectionRangeSymbolDesc(Sec, false) should be returned. +/// +/// This pass should be run in the PostAllocationPass pipeline, at which point +/// all blocks should have been assigned their final addresses. +template <typename SymbolIdentifierFunction> +DefineExternalSectionStartAndEndSymbols<SymbolIdentifierFunction> +createDefineExternalSectionStartAndEndSymbolsPass( + SymbolIdentifierFunction &&F) { + return DefineExternalSectionStartAndEndSymbols<SymbolIdentifierFunction>( + std::forward<SymbolIdentifierFunction>(F)); +} + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE + +#endif // LLVM_EXECUTIONENGINE_JITLINK_DEFINEEXTERNALSECTIONSTARTANDENDSYMBOLS_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 3602601287f4..c85e80b52e5a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -81,7 +81,9 @@ Error EHFrameSplitter::processBlock(LinkGraph &G, Block &B, return Error::success(); } - BinaryStreamReader BlockReader(B.getContent(), G.getEndianness()); + BinaryStreamReader BlockReader( + StringRef(B.getContent().data(), B.getContent().size()), + G.getEndianness()); while (true) { uint64_t RecordStartOffset = BlockReader.getOffset(); @@ -203,7 +205,9 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { } CIEInfosMap CIEInfos; - BinaryStreamReader BlockReader(B.getContent(), PC.G.getEndianness()); + BinaryStreamReader BlockReader( + StringRef(B.getContent().data(), B.getContent().size()), + PC.G.getEndianness()); while (!BlockReader.empty()) { size_t RecordStartOffset = BlockReader.getOffset(); @@ -267,8 +271,10 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, LLVM_DEBUG(dbgs() << " Record is CIE\n"); - auto RecordContent = B.getContent().substr(RecordOffset, RecordLength); - BinaryStreamReader RecordReader(RecordContent, PC.G.getEndianness()); + auto RecordContent = B.getContent().slice(RecordOffset, RecordLength); + BinaryStreamReader RecordReader( + StringRef(RecordContent.data(), RecordContent.size()), + PC.G.getEndianness()); // Skip past the CIE delta field: we've already processed this far. RecordReader.setOffset(CIEDeltaFieldOffset + 4); @@ -397,8 +403,10 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, JITTargetAddress RecordAddress = B.getAddress() + RecordOffset; - auto RecordContent = B.getContent().substr(RecordOffset, RecordLength); - BinaryStreamReader RecordReader(RecordContent, PC.G.getEndianness()); + auto RecordContent = B.getContent().slice(RecordOffset, RecordLength); + BinaryStreamReader RecordReader( + StringRef(RecordContent.data(), RecordContent.size()), + PC.G.getEndianness()); // Skip past the CIE delta field: we've already read this far. RecordReader.setOffset(CIEDeltaFieldOffset + 4); @@ -730,6 +738,28 @@ Expected<Symbol &> EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, return PC.G.addAnonymousSymbol(*B, Addr - B->getAddress(), 0, false, false); } +char EHFrameNullTerminator::NullTerminatorBlockContent[4] = {0, 0, 0, 0}; + +EHFrameNullTerminator::EHFrameNullTerminator(StringRef EHFrameSectionName) + : EHFrameSectionName(EHFrameSectionName) {} + +Error EHFrameNullTerminator::operator()(LinkGraph &G) { + auto *EHFrame = G.findSectionByName(EHFrameSectionName); + + if (!EHFrame) + return Error::success(); + + LLVM_DEBUG({ + dbgs() << "EHFrameNullTerminator adding null terminator to " + << EHFrameSectionName << "\n"; + }); + + auto &NullTerminatorBlock = G.createContentBlock( + *EHFrame, NullTerminatorBlockContent, 0xfffffffffffffffc, 1, 0); + G.addAnonymousSymbol(NullTerminatorBlock, 0, 4, false, true); + return Error::success(); +} + EHFrameRegistrar::~EHFrameRegistrar() {} Error InProcessEHFrameRegistrar::registerEHFrames( @@ -751,7 +781,7 @@ createEHFrameRecorderPass(const Triple &TT, StoreFrameRangeFunction StoreRangeAddress) { const char *EHFrameSectionName = nullptr; if (TT.getObjectFormat() == Triple::MachO) - EHFrameSectionName = "__eh_frame"; + EHFrameSectionName = "__TEXT,__eh_frame"; else EHFrameSectionName = ".eh_frame"; @@ -768,8 +798,9 @@ createEHFrameRecorderPass(const Triple &TT, Size = R.getSize(); } if (Addr == 0 && Size != 0) - return make_error<JITLinkError>("__eh_frame section can not have zero " - "address with non-zero size"); + return make_error<JITLinkError>( + StringRef(EHFrameSectionName) + + " section can not have zero address with non-zero size"); StoreFrameRange(Addr, Size); return Error::success(); }; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index 5e68e72ba18d..b4c4b0f7b097 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -116,6 +116,17 @@ private: Edge::Kind NegDelta32; }; +/// Add a 32-bit null-terminator to the end of the eh-frame section. +class EHFrameNullTerminator { +public: + EHFrameNullTerminator(StringRef EHFrameSectionName); + Error operator()(LinkGraph &G); + +private: + static char NullTerminatorBlockContent[]; + StringRef EHFrameSectionName; +}; + } // end namespace jitlink } // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp index 27eb7d576e2d..252e44fe4a74 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp @@ -14,6 +14,7 @@ #include "llvm/ExecutionEngine/JITLink/ELF.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/ELF_riscv.h" #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" #include "llvm/Object/ELF.h" #include "llvm/Support/Endian.h" @@ -64,8 +65,10 @@ createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer) { return TargetMachineArch.takeError(); switch (*TargetMachineArch) { + case ELF::EM_RISCV: + return createLinkGraphFromELFObject_riscv(ObjectBuffer); case ELF::EM_X86_64: - return createLinkGraphFromELFObject_x86_64(std::move(ObjectBuffer)); + return createLinkGraphFromELFObject_x86_64(ObjectBuffer); default: return make_error<JITLinkError>( "Unsupported target machine architecture in ELF object " + @@ -76,6 +79,10 @@ createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer) { void link_ELF(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) { switch (G->getTargetTriple().getArch()) { + case Triple::riscv32: + case Triple::riscv64: + link_ELF_riscv(std::move(G), std::move(Ctx)); + return; case Triple::x86_64: link_ELF_x86_64(std::move(G), std::move(Ctx)); return; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.cpp new file mode 100644 index 000000000000..2194a4fbf1f4 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.cpp @@ -0,0 +1,33 @@ +//=----------- ELFLinkGraphBuilder.cpp - ELF LinkGraph builder ------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic ELF LinkGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "ELFLinkGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +static const char *DWSecNames[] = { +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ + ELF_NAME, +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION +}; + +namespace llvm { +namespace jitlink { + +StringRef ELFLinkGraphBuilderBase::CommonSectionName(".common"); +ArrayRef<const char *> ELFLinkGraphBuilderBase::DwarfSectionNames = DWSecNames; + +ELFLinkGraphBuilderBase::~ELFLinkGraphBuilderBase() {} + +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h new file mode 100644 index 000000000000..2b2a1a8db4c1 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h @@ -0,0 +1,429 @@ +//===------- ELFLinkGraphBuilder.h - ELF LinkGraph builder ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic ELF LinkGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_ELFLINKGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_ELFLINKGRAPHBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Object/ELF.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +/// Common link-graph building code shared between all ELFFiles. +class ELFLinkGraphBuilderBase { +public: + ELFLinkGraphBuilderBase(std::unique_ptr<LinkGraph> G) : G(std::move(G)) {} + virtual ~ELFLinkGraphBuilderBase(); + +protected: + static bool isDwarfSection(StringRef SectionName) { + return llvm::is_contained(DwarfSectionNames, SectionName); + } + + Section &getCommonSection() { + if (!CommonSection) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + CommonSection = &G->createSection(CommonSectionName, Prot); + } + return *CommonSection; + } + + std::unique_ptr<LinkGraph> G; + +private: + static StringRef CommonSectionName; + static ArrayRef<const char *> DwarfSectionNames; + + Section *CommonSection = nullptr; +}; + +/// Ling-graph building code that's specific to the given ELFT, but common +/// across all architectures. +template <typename ELFT> +class ELFLinkGraphBuilder : public ELFLinkGraphBuilderBase { + using ELFFile = object::ELFFile<ELFT>; + +public: + ELFLinkGraphBuilder(const object::ELFFile<ELFT> &Obj, Triple TT, + StringRef FileName, + LinkGraph::GetEdgeKindNameFunction GetEdgeKindName); + + /// Attempt to construct and return the LinkGraph. + Expected<std::unique_ptr<LinkGraph>> buildGraph(); + + /// Call to derived class to handle relocations. These require + /// architecture specific knowledge to map to JITLink edge kinds. + virtual Error addRelocations() = 0; + +protected: + using ELFSectionIndex = unsigned; + using ELFSymbolIndex = unsigned; + + bool isRelocatable() const { + return Obj.getHeader().e_type == llvm::ELF::ET_REL; + } + + void setGraphSection(ELFSectionIndex SecIndex, Section &Sec) { + assert(!GraphSections.count(SecIndex) && "Duplicate section at index"); + GraphSections[SecIndex] = &Sec; + } + + Section *getGraphSection(ELFSectionIndex SecIndex) { + auto I = GraphSections.find(SecIndex); + if (I == GraphSections.end()) + return nullptr; + return I->second; + } + + void setGraphSymbol(ELFSymbolIndex SymIndex, Symbol &Sym) { + assert(!GraphSymbols.count(SymIndex) && "Duplicate symbol at index"); + GraphSymbols[SymIndex] = &Sym; + } + + Symbol *getGraphSymbol(ELFSymbolIndex SymIndex) { + auto I = GraphSymbols.find(SymIndex); + if (I == GraphSymbols.end()) + return nullptr; + return I->second; + } + + Expected<std::pair<Linkage, Scope>> + getSymbolLinkageAndScope(const typename ELFT::Sym &Sym, StringRef Name); + + Error prepare(); + Error graphifySections(); + Error graphifySymbols(); + + const ELFFile &Obj; + + typename ELFFile::Elf_Shdr_Range Sections; + const typename ELFFile::Elf_Shdr *SymTabSec = nullptr; + StringRef SectionStringTab; + + // Maps ELF section indexes to LinkGraph Sections. + // Only SHF_ALLOC sections will have graph sections. + DenseMap<ELFSectionIndex, Section *> GraphSections; + DenseMap<ELFSymbolIndex, Symbol *> GraphSymbols; +}; + +template <typename ELFT> +ELFLinkGraphBuilder<ELFT>::ELFLinkGraphBuilder( + const ELFFile &Obj, Triple TT, StringRef FileName, + LinkGraph::GetEdgeKindNameFunction GetEdgeKindName) + : ELFLinkGraphBuilderBase(std::make_unique<LinkGraph>( + FileName.str(), Triple(std::move(TT)), ELFT::Is64Bits ? 8 : 4, + support::endianness(ELFT::TargetEndianness), + std::move(GetEdgeKindName))), + Obj(Obj) { + LLVM_DEBUG( + { dbgs() << "Created ELFLinkGraphBuilder for \"" << FileName << "\""; }); +} + +template <typename ELFT> +Expected<std::unique_ptr<LinkGraph>> ELFLinkGraphBuilder<ELFT>::buildGraph() { + if (!isRelocatable()) + return make_error<JITLinkError>("Object is not a relocatable ELF file"); + + if (auto Err = prepare()) + return std::move(Err); + + if (auto Err = graphifySections()) + return std::move(Err); + + if (auto Err = graphifySymbols()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +template <typename ELFT> +Expected<std::pair<Linkage, Scope>> +ELFLinkGraphBuilder<ELFT>::getSymbolLinkageAndScope( + const typename ELFT::Sym &Sym, StringRef Name) { + Linkage L = Linkage::Strong; + Scope S = Scope::Default; + + switch (Sym.getBinding()) { + case ELF::STB_LOCAL: + S = Scope::Local; + break; + case ELF::STB_GLOBAL: + // Nothing to do here. + break; + case ELF::STB_WEAK: + L = Linkage::Weak; + break; + default: + return make_error<StringError>("Unrecognized symbol binding for " + Name, + inconvertibleErrorCode()); + } + + switch (Sym.getVisibility()) { + case ELF::STV_DEFAULT: + case ELF::STV_PROTECTED: + // FIXME: Make STV_DEFAULT symbols pre-emptible? This probably needs + // Orc support. + // Otherwise nothing to do here. + break; + case ELF::STV_HIDDEN: + // Default scope -> Hidden scope. No effect on local scope. + if (S == Scope::Default) + S = Scope::Hidden; + break; + case ELF::STV_INTERNAL: + return make_error<StringError>("Unrecognized symbol visibility for " + Name, + inconvertibleErrorCode()); + } + + return std::make_pair(L, S); +} + +template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::prepare() { + LLVM_DEBUG(dbgs() << " Preparing to build...\n"); + + // Get the sections array. + if (auto SectionsOrErr = Obj.sections()) + Sections = *SectionsOrErr; + else + return SectionsOrErr.takeError(); + + // Get the section string table. + if (auto SectionStringTabOrErr = Obj.getSectionStringTable(Sections)) + SectionStringTab = *SectionStringTabOrErr; + else + return SectionStringTabOrErr.takeError(); + + // Get the SHT_SYMTAB section. + for (auto &Sec : Sections) + if (Sec.sh_type == ELF::SHT_SYMTAB) { + if (!SymTabSec) + SymTabSec = &Sec; + else + return make_error<JITLinkError>("Multiple SHT_SYMTAB sections in " + + G->getName()); + } + + return Error::success(); +} + +template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySections() { + LLVM_DEBUG(dbgs() << " Creating graph sections...\n"); + + // For each section... + for (ELFSectionIndex SecIndex = 0; SecIndex != Sections.size(); ++SecIndex) { + + auto &Sec = Sections[SecIndex]; + + // Start by getting the section name. + auto Name = Obj.getSectionName(Sec, SectionStringTab); + if (!Name) + return Name.takeError(); + + // If the name indicates that it's a debug section then skip it: We don't + // support those yet. + if (isDwarfSection(*Name)) { + LLVM_DEBUG({ + dbgs() << " " << SecIndex << ": \"" << *Name + << "\" is a debug section: " + "No graph section will be created.\n"; + }); + continue; + } + + // Skip non-SHF_ALLOC sections + if (!(Sec.sh_flags & ELF::SHF_ALLOC)) { + LLVM_DEBUG({ + dbgs() << " " << SecIndex << ": \"" << *Name + << "\" is not an SHF_ALLOC section: " + "No graph section will be created.\n"; + }); + continue; + } + + LLVM_DEBUG({ + dbgs() << " " << SecIndex << ": Creating section for \"" << *Name + << "\"\n"; + }); + + // Get the section's memory protection flags. + sys::Memory::ProtectionFlags Prot; + if (Sec.sh_flags & ELF::SHF_EXECINSTR) + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + // For now we just use this to skip the "undefined" section, probably need + // to revist. + if (Sec.sh_size == 0) + continue; + + auto &GraphSec = G->createSection(*Name, Prot); + if (Sec.sh_type != ELF::SHT_NOBITS) { + auto Data = Obj.template getSectionContentsAsArray<char>(Sec); + if (!Data) + return Data.takeError(); + + G->createContentBlock(GraphSec, *Data, Sec.sh_addr, Sec.sh_addralign, 0); + } else + G->createZeroFillBlock(GraphSec, Sec.sh_size, Sec.sh_addr, + Sec.sh_addralign, 0); + + setGraphSection(SecIndex, GraphSec); + } + + return Error::success(); +} + +template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { + LLVM_DEBUG(dbgs() << " Creating graph symbols...\n"); + + // No SYMTAB -- Bail out early. + if (!SymTabSec) + return Error::success(); + + // Get the section content as a Symbols array. + auto Symbols = Obj.symbols(SymTabSec); + if (!Symbols) + return Symbols.takeError(); + + // Get the string table for this section. + auto StringTab = Obj.getStringTableForSymtab(*SymTabSec, Sections); + if (!StringTab) + return StringTab.takeError(); + + LLVM_DEBUG({ + StringRef SymTabName; + + if (auto SymTabNameOrErr = Obj.getSectionName(*SymTabSec, SectionStringTab)) + SymTabName = *SymTabNameOrErr; + else { + dbgs() << "Could not get ELF SHT_SYMTAB section name for logging: " + << toString(SymTabNameOrErr.takeError()) << "\n"; + SymTabName = "<SHT_SYMTAB section with invalid name>"; + } + + dbgs() << " Adding symbols from symtab section \"" << SymTabName + << "\"\n"; + }); + + for (ELFSymbolIndex SymIndex = 0; SymIndex != Symbols->size(); ++SymIndex) { + auto &Sym = (*Symbols)[SymIndex]; + + // Check symbol type. + switch (Sym.getType()) { + case ELF::STT_FILE: + LLVM_DEBUG({ + if (auto Name = Sym.getName(*StringTab)) + dbgs() << " " << SymIndex << ": Skipping STT_FILE symbol \"" + << *Name << "\"\n"; + else { + dbgs() << "Could not get STT_FILE symbol name: " + << toString(Name.takeError()) << "\n"; + dbgs() << " " << SymIndex + << ": Skipping STT_FILE symbol with invalid name\n"; + } + }); + continue; + break; + } + + // Get the symbol name. + auto Name = Sym.getName(*StringTab); + if (!Name) + return Name.takeError(); + + // Handle common symbols specially. + if (Sym.isCommon()) { + Symbol &GSym = + G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, + Sym.st_size, Sym.getValue(), false); + setGraphSymbol(SymIndex, GSym); + continue; + } + + // Map Visibility and Binding to Scope and Linkage: + Linkage L; + Scope S; + + if (auto LSOrErr = getSymbolLinkageAndScope(Sym, *Name)) + std::tie(L, S) = *LSOrErr; + else + return LSOrErr.takeError(); + + if (Sym.isDefined() && + (Sym.getType() == ELF::STT_NOTYPE || Sym.getType() == ELF::STT_FUNC || + Sym.getType() == ELF::STT_OBJECT || + Sym.getType() == ELF::STT_SECTION)) { + + // FIXME: Handle extended tables. + if (auto *GraphSec = getGraphSection(Sym.st_shndx)) { + Block *B = nullptr; + { + auto Blocks = GraphSec->blocks(); + assert(Blocks.begin() != Blocks.end() && "No blocks for section"); + assert(std::next(Blocks.begin()) == Blocks.end() && + "Multiple blocks for section"); + B = *Blocks.begin(); + } + + LLVM_DEBUG({ + dbgs() << " " << SymIndex + << ": Creating defined graph symbol for ELF symbol \"" << *Name + << "\"\n"; + }); + + if (Sym.getType() == ELF::STT_SECTION) + *Name = GraphSec->getName(); + + auto &GSym = + G->addDefinedSymbol(*B, Sym.getValue(), *Name, Sym.st_size, L, S, + Sym.getType() == ELF::STT_FUNC, false); + setGraphSymbol(SymIndex, GSym); + } + } else if (Sym.isUndefined() && Sym.isExternal()) { + LLVM_DEBUG({ + dbgs() << " " << SymIndex + << ": Creating external graph symbol for ELF symbol \"" << *Name + << "\"\n"; + }); + auto &GSym = G->addExternalSymbol(*Name, Sym.st_size, L); + setGraphSymbol(SymIndex, GSym); + } else { + LLVM_DEBUG({ + dbgs() << " " << SymIndex + << ": Not creating graph symbol for ELF symbol \"" << *Name + << "\" with unrecognized type\n"; + }); + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE + +#endif // LIB_EXECUTIONENGINE_JITLINK_ELFLINKGRAPHBUILDER_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp new file mode 100644 index 000000000000..d0e65ef1c3ac --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp @@ -0,0 +1,315 @@ +//===------- ELF_riscv.cpp -JIT linker implementation for ELF/riscv -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ELF/riscv jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ELF_riscv.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITLink/riscv.h" +#include "llvm/Object/ELF.h" +#include "llvm/Object/ELFObjectFile.h" + +#include "ELFLinkGraphBuilder.h" +#include "JITLinkGeneric.h" + +#define DEBUG_TYPE "jitlink" +using namespace llvm; + +namespace llvm { +namespace jitlink { + +static Expected<const Edge &> getRISCVPCRelHi20(const Edge &E) { + using namespace riscv; + assert((E.getKind() == R_RISCV_PCREL_LO12_I || + E.getKind() == R_RISCV_PCREL_LO12_S) && + "Can only have high relocation for R_RISCV_PCREL_LO12_I or " + "R_RISCV_PCREL_LO12_S"); + + const Symbol &Sym = E.getTarget(); + const Block &B = Sym.getBlock(); + JITTargetAddress Offset = Sym.getOffset(); + + struct Comp { + bool operator()(const Edge &Lhs, JITTargetAddress Offset) { + return Lhs.getOffset() < Offset; + } + bool operator()(JITTargetAddress Offset, const Edge &Rhs) { + return Offset < Rhs.getOffset(); + } + }; + + auto Bound = + std::equal_range(B.edges().begin(), B.edges().end(), Offset, Comp{}); + + for (auto It = Bound.first; It != Bound.second; ++It) { + if (It->getKind() == R_RISCV_PCREL_HI20) + return *It; + } + + return make_error<JITLinkError>( + "No HI20 PCREL relocation type be found for LO12 PCREL relocation type"); +} + +static uint32_t extractBits(uint64_t Num, unsigned High, unsigned Low) { + return (Num & ((1ULL << (High + 1)) - 1)) >> Low; +} + +class ELFJITLinker_riscv : public JITLinker<ELFJITLinker_riscv> { + friend class JITLinker<ELFJITLinker_riscv>; + +public: + ELFJITLinker_riscv(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} + +private: + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { + using namespace riscv; + using namespace llvm::support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + switch (E.getKind()) { + case R_RISCV_HI20: { + int64_t Value = E.getTarget().getAddress() + E.getAddend(); + int32_t Hi = (Value + 0x800) & 0xFFFFF000; + uint32_t RawInstr = *(little32_t *)FixupPtr; + *(little32_t *)FixupPtr = (RawInstr & 0xFFF) | static_cast<uint32_t>(Hi); + break; + } + case R_RISCV_LO12_I: { + int64_t Value = E.getTarget().getAddress() + E.getAddend(); + int32_t Lo = Value & 0xFFF; + uint32_t RawInstr = *(little32_t *)FixupPtr; + *(little32_t *)FixupPtr = + (RawInstr & 0xFFFFF) | (static_cast<uint32_t>(Lo & 0xFFF) << 20); + break; + } + case R_RISCV_CALL: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + int32_t Hi = (Value + 0x800) & 0xFFFFF000; + int32_t Lo = Value & 0xFFF; + uint32_t RawInstrAuipc = *(little32_t *)FixupPtr; + uint32_t RawInstrJalr = *(little32_t *)(FixupPtr + 4); + *(little32_t *)FixupPtr = RawInstrAuipc | static_cast<uint32_t>(Hi); + *(little32_t *)(FixupPtr + 4) = + RawInstrJalr | (static_cast<uint32_t>(Lo) << 20); + break; + } + case R_RISCV_PCREL_HI20: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + int32_t Hi = (Value + 0x800) & 0xFFFFF000; + uint32_t RawInstr = *(little32_t *)FixupPtr; + *(little32_t *)FixupPtr = (RawInstr & 0xFFF) | static_cast<uint32_t>(Hi); + break; + } + case R_RISCV_PCREL_LO12_I: { + auto RelHI20 = getRISCVPCRelHi20(E); + if (!RelHI20) + return RelHI20.takeError(); + int64_t Value = RelHI20->getTarget().getAddress() + + RelHI20->getAddend() - E.getTarget().getAddress(); + int64_t Lo = Value & 0xFFF; + uint32_t RawInstr = *(little32_t *)FixupPtr; + *(little32_t *)FixupPtr = + (RawInstr & 0xFFFFF) | (static_cast<uint32_t>(Lo & 0xFFF) << 20); + break; + } + case R_RISCV_PCREL_LO12_S: { + auto RelHI20 = getRISCVPCRelHi20(E); + int64_t Value = RelHI20->getTarget().getAddress() + + RelHI20->getAddend() - E.getTarget().getAddress(); + int64_t Lo = Value & 0xFFF; + uint32_t Imm31_25 = extractBits(Lo, 11, 5) << 25; + uint32_t Imm11_7 = extractBits(Lo, 4, 0) << 7; + uint32_t RawInstr = *(little32_t *)FixupPtr; + + *(little32_t *)FixupPtr = (RawInstr & 0x1FFF07F) | Imm31_25 | Imm11_7; + break; + } + } + return Error::success(); + } +}; + +template <typename ELFT> +class ELFLinkGraphBuilder_riscv : public ELFLinkGraphBuilder<ELFT> { +private: + static Expected<riscv::EdgeKind_riscv> + getRelocationKind(const uint32_t Type) { + using namespace riscv; + switch (Type) { + case ELF::R_RISCV_32: + return EdgeKind_riscv::R_RISCV_32; + case ELF::R_RISCV_64: + return EdgeKind_riscv::R_RISCV_64; + case ELF::R_RISCV_HI20: + return EdgeKind_riscv::R_RISCV_HI20; + case ELF::R_RISCV_LO12_I: + return EdgeKind_riscv::R_RISCV_LO12_I; + case ELF::R_RISCV_CALL: + return EdgeKind_riscv::R_RISCV_CALL; + case ELF::R_RISCV_PCREL_HI20: + return EdgeKind_riscv::R_RISCV_PCREL_HI20; + case ELF::R_RISCV_PCREL_LO12_I: + return EdgeKind_riscv::R_RISCV_PCREL_LO12_I; + case ELF::R_RISCV_PCREL_LO12_S: + return EdgeKind_riscv::R_RISCV_PCREL_LO12_S; + } + + return make_error<JITLinkError>("Unsupported riscv relocation:" + + formatv("{0:d}", Type)); + } + + Error addRelocations() override { + using Base = ELFLinkGraphBuilder<ELFT>; + LLVM_DEBUG(dbgs() << "Adding relocations\n"); + + // TODO a partern is forming of iterate some sections but only give me + // ones I am interested, I should abstract that concept some where + for (auto &SecRef : Base::Sections) { + if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL) + continue; + auto RelSectName = Base::Obj.getSectionName(SecRef); + if (!RelSectName) + return RelSectName.takeError(); + + LLVM_DEBUG({ + dbgs() << "Adding relocations from section " << *RelSectName << "\n"; + }); + + auto UpdateSection = Base::Obj.getSection(SecRef.sh_info); + if (!UpdateSection) + return UpdateSection.takeError(); + + auto UpdateSectionName = Base::Obj.getSectionName(**UpdateSection); + if (!UpdateSectionName) + return UpdateSectionName.takeError(); + // Don't process relocations for debug sections. + if (Base::isDwarfSection(*UpdateSectionName)) { + LLVM_DEBUG({ + dbgs() << " Target is dwarf section " << *UpdateSectionName + << ". Skipping.\n"; + }); + continue; + } else + LLVM_DEBUG({ + dbgs() << " For target section " << *UpdateSectionName << "\n"; + }); + + auto *JITSection = Base::G->findSectionByName(*UpdateSectionName); + if (!JITSection) + return make_error<llvm::StringError>( + "Refencing a section that wasn't added to graph" + + *UpdateSectionName, + llvm::inconvertibleErrorCode()); + + auto Relocations = Base::Obj.relas(SecRef); + if (!Relocations) + return Relocations.takeError(); + + for (const auto &Rela : *Relocations) { + auto Type = Rela.getType(false); + + LLVM_DEBUG({ + dbgs() << "Relocation Type: " << Type << "\n" + << "Name: " << Base::Obj.getRelocationTypeName(Type) << "\n"; + }); + + auto SymbolIndex = Rela.getSymbol(false); + auto Symbol = Base::Obj.getRelocationSymbol(Rela, Base::SymTabSec); + if (!Symbol) + return Symbol.takeError(); + + auto BlockToFix = *(JITSection->blocks().begin()); + auto *TargetSymbol = Base::getGraphSymbol(SymbolIndex); + + if (!TargetSymbol) { + return make_error<llvm::StringError>( + "Could not find symbol at given index, did you add it to " + "JITSymbolTable? index: " + + std::to_string(SymbolIndex) + ", shndx: " + + std::to_string((*Symbol)->st_shndx) + " Size of table: " + + std::to_string(Base::GraphSymbols.size()), + llvm::inconvertibleErrorCode()); + } + int64_t Addend = Rela.r_addend; + JITTargetAddress FixupAddress = + (*UpdateSection)->sh_addr + Rela.r_offset; + + LLVM_DEBUG({ + dbgs() << "Processing relocation at " + << format("0x%016" PRIx64, FixupAddress) << "\n"; + }); + auto Kind = getRelocationKind(Type); + if (!Kind) + return Kind.takeError(); + + BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + *TargetSymbol, Addend); + } + } + return Error::success(); + } + +public: + ELFLinkGraphBuilder_riscv(StringRef FileName, + const object::ELFFile<ELFT> &Obj, const Triple T) + : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, + riscv::getEdgeKindName) {} +}; + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer) { + LLVM_DEBUG({ + dbgs() << "Building jitlink graph for new input " + << ObjectBuffer.getBufferIdentifier() << "...\n"; + }); + + auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); + if (!ELFObj) + return ELFObj.takeError(); + + if ((*ELFObj)->getArch() == Triple::riscv64) { + auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); + return ELFLinkGraphBuilder_riscv<object::ELF64LE>( + (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), + (*ELFObj)->makeTriple()) + .buildGraph(); + } else { + assert((*ELFObj)->getArch() == Triple::riscv32 && + "Invalid triple for RISCV ELF object file"); + auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj); + return ELFLinkGraphBuilder_riscv<object::ELF32LE>( + (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), + (*ELFObj)->makeTriple()) + .buildGraph(); + } +} + +void link_ELF_riscv(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; + const Triple &TT = G->getTargetTriple(); + if (Ctx->shouldAddDefaultTargetPasses(TT)) { + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + } + if (auto Err = Ctx->modifyPassConfig(*G, Config)) + return Ctx->notifyFailed(std::move(Err)); + + ELFJITLinker_riscv::link(std::move(Ctx), std::move(G), std::move(Config)); +} + +} // namespace jitlink +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 2a6b3eb19ded..a5aed6d25200 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -12,12 +12,15 @@ #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/Endian.h" -#include "BasicGOTAndStubsBuilder.h" +#include "DefineExternalSectionStartAndEndSymbols.h" #include "EHFrameSupportImpl.h" +#include "ELFLinkGraphBuilder.h" #include "JITLinkGeneric.h" +#include "PerGraphGOTAndPLTStubsBuilder.h" #define DEBUG_TYPE "jitlink" @@ -27,17 +30,29 @@ using namespace llvm::jitlink::ELF_x86_64_Edges; namespace { -class ELF_x86_64_GOTAndStubsBuilder - : public BasicGOTAndStubsBuilder<ELF_x86_64_GOTAndStubsBuilder> { +constexpr StringRef ELFGOTSectionName = "$__GOT"; +constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_"; + +class PerGraphGOTAndPLTStubsBuilder_ELF_x86_64 + : public PerGraphGOTAndPLTStubsBuilder< + PerGraphGOTAndPLTStubsBuilder_ELF_x86_64> { public: static const uint8_t NullGOTEntryContent[8]; static const uint8_t StubContent[6]; - ELF_x86_64_GOTAndStubsBuilder(LinkGraph &G) - : BasicGOTAndStubsBuilder<ELF_x86_64_GOTAndStubsBuilder>(G) {} + using PerGraphGOTAndPLTStubsBuilder< + PerGraphGOTAndPLTStubsBuilder_ELF_x86_64>::PerGraphGOTAndPLTStubsBuilder; + + bool isGOTEdgeToFix(Edge &E) const { + if (E.getKind() == GOTOFF64) { + // We need to make sure that the GOT section exists, but don't otherwise + // need to fix up this edge. + getGOTSection(); + return false; + } - bool isGOTEdge(Edge &E) const { - return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; + return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad || + E.getKind() == PCRel64GOT || E.getKind() == GOT64; } Symbol &createGOTEntry(Symbol &Target) { @@ -48,14 +63,25 @@ public: } void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && - "Not a GOT edge?"); - // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is - // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to - // check for GOT optimization opportunities in the + // If this is a PCRel32GOT/PCRel64GOT then change it to an ordinary + // PCRel32/PCRel64. If it is a PCRel32GOTLoad then leave it as-is for now: + // We will use the kind to check for GOT optimization opportunities in the // optimizeMachO_x86_64_GOTAndStubs pass below. - if (E.getKind() == PCRel32GOT) + // If it's a GOT64 leave it as is. + switch (E.getKind()) { + case PCRel32GOT: E.setKind(PCRel32); + break; + case PCRel64GOT: + E.setKind(PCRel64); + break; + case GOT64: + break; + case PCRel32GOTLoad: + break; + default: + llvm_unreachable("Unexpected GOT edge kind"); + } E.setTarget(GOTEntry); // Leave the edge addend as-is. @@ -65,16 +91,16 @@ public: return E.getKind() == Branch32 && !E.getTarget().isDefined(); } - Symbol &createStub(Symbol &Target) { + Symbol &createPLTStub(Symbol &Target) { auto &StubContentBlock = G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + auto &GOTEntrySymbol = getGOTEntry(Target); StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, -4); return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); } - void fixExternalBranchEdge(Edge &E, Symbol &Stub) { + void fixPLTEdge(Edge &E, Symbol &Stub) { assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); // Set the edge kind to Branch32ToStub. We will use this to check for stub @@ -85,13 +111,13 @@ public: } private: - Section &getGOTSection() { + Section &getGOTSection() const { if (!GOTSection) - GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ); + GOTSection = &G.createSection(ELFGOTSectionName, sys::Memory::MF_READ); return *GOTSection; } - Section &getStubsSection() { + Section &getStubsSection() const { if (!StubsSection) { auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( sys::Memory::MF_READ | sys::Memory::MF_EXEC); @@ -100,35 +126,26 @@ private: return *StubsSection; } - StringRef getGOTEntryBlockContent() { - return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), - sizeof(NullGOTEntryContent)); + ArrayRef<char> getGOTEntryBlockContent() { + return {reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)}; } - StringRef getStubBlockContent() { - return StringRef(reinterpret_cast<const char *>(StubContent), - sizeof(StubContent)); + ArrayRef<char> getStubBlockContent() { + return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)}; } - Section *GOTSection = nullptr; - Section *StubsSection = nullptr; -}; - -const char *const DwarfSectionNames[] = { -#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ - ELF_NAME, -#include "llvm/BinaryFormat/Dwarf.def" -#undef HANDLE_DWARF_SECTION + mutable Section *GOTSection = nullptr; + mutable Section *StubsSection = nullptr; }; } // namespace -const uint8_t ELF_x86_64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t ELF_x86_64_GOTAndStubsBuilder::StubContent[6] = { +const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::NullGOTEntryContent[8] = + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent[6] = { 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; -static const char *CommonSectionName = "__common"; static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) { LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); @@ -171,9 +188,10 @@ static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) { } } else if (E.getKind() == Branch32ToStub) { auto &StubBlock = E.getTarget().getBlock(); - assert(StubBlock.getSize() == - sizeof(ELF_x86_64_GOTAndStubsBuilder::StubContent) && - "Stub block should be stub sized"); + assert( + StubBlock.getSize() == + sizeof(PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::StubContent) && + "Stub block should be stub sized"); assert(StubBlock.edges_size() == 1 && "Stub block should only have one outgoing edge"); @@ -203,37 +221,13 @@ static Error optimizeELF_x86_64_GOTAndStubs(LinkGraph &G) { return Error::success(); } -static bool isDwarfSection(StringRef SectionName) { - for (auto &DwarfSectionName : DwarfSectionNames) - if (SectionName == DwarfSectionName) - return true; - return false; -} - namespace llvm { namespace jitlink { // This should become a template as the ELFFile is so a lot of this could become // generic -class ELFLinkGraphBuilder_x86_64 { - +class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> { private: - Section *CommonSection = nullptr; - // TODO hack to get this working - // Find a better way - using SymbolTable = object::ELFFile<object::ELF64LE>::Elf_Shdr; - // For now we just assume - using SymbolMap = std::map<int32_t, Symbol *>; - SymbolMap JITSymbolTable; - - Section &getCommonSection() { - if (!CommonSection) { - auto Prot = static_cast<sys::Memory::ProtectionFlags>( - sys::Memory::MF_READ | sys::Memory::MF_WRITE); - CommonSection = &G->createSection(CommonSectionName, Prot); - } - return *CommonSection; - } static Expected<ELF_x86_64_Edges::ELFX86RelocationKind> getRelocationKind(const uint32_t Type) { @@ -241,6 +235,7 @@ private: case ELF::R_X86_64_PC32: return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32; case ELF::R_X86_64_PC64: + case ELF::R_X86_64_GOTPC64: return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64; case ELF::R_X86_64_64: return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64; @@ -248,6 +243,12 @@ private: case ELF::R_X86_64_GOTPCRELX: case ELF::R_X86_64_REX_GOTPCRELX: return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad; + case ELF::R_X86_64_GOTPCREL64: + return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT; + case ELF::R_X86_64_GOT64: + return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64; + case ELF::R_X86_64_GOTOFF64: + return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64; case ELF::R_X86_64_PLT32: return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32; } @@ -255,145 +256,11 @@ private: formatv("{0:d}", Type)); } - std::unique_ptr<LinkGraph> G; - // This could be a template - const object::ELFFile<object::ELF64LE> &Obj; - object::ELFFile<object::ELF64LE>::Elf_Shdr_Range sections; - SymbolTable SymTab; - - bool isRelocatable() { return Obj.getHeader().e_type == llvm::ELF::ET_REL; } - - support::endianness - getEndianness(const object::ELFFile<object::ELF64LE> &Obj) { - return Obj.isLE() ? support::little : support::big; - } - - // This could also just become part of a template - unsigned getPointerSize(const object::ELFFile<object::ELF64LE> &Obj) { - return Obj.getHeader().getFileClass() == ELF::ELFCLASS64 ? 8 : 4; - } - - // We don't technically need this right now - // But for now going to keep it as it helps me to debug things - - Error createNormalizedSymbols() { - LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n"); - - for (auto SecRef : sections) { - if (SecRef.sh_type != ELF::SHT_SYMTAB && - SecRef.sh_type != ELF::SHT_DYNSYM) - continue; - - auto Symbols = Obj.symbols(&SecRef); - // TODO: Currently I use this function to test things - // I also want to leave it to see if its common between MACH and elf - // so for now I just want to continue even if there is an error - if (errorToBool(Symbols.takeError())) - continue; - - auto StrTabSec = Obj.getSection(SecRef.sh_link); - if (!StrTabSec) - return StrTabSec.takeError(); - auto StringTable = Obj.getStringTable(**StrTabSec); - if (!StringTable) - return StringTable.takeError(); - - for (auto SymRef : *Symbols) { - Optional<StringRef> Name; - - if (auto NameOrErr = SymRef.getName(*StringTable)) - Name = *NameOrErr; - else - return NameOrErr.takeError(); - - LLVM_DEBUG({ - dbgs() << " value = " << formatv("{0:x16}", SymRef.getValue()) - << ", type = " << formatv("{0:x2}", SymRef.getType()) - << ", binding = " << formatv("{0:x2}", SymRef.getBinding()) - << ", size = " - << formatv("{0:x16}", static_cast<uint64_t>(SymRef.st_size)) - << ", info = " << formatv("{0:x2}", SymRef.st_info) - << " :" << (Name ? *Name : "<anonymous symbol>") << "\n"; - }); - } - } - return Error::success(); - } - - Error createNormalizedSections() { - LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); - for (auto &SecRef : sections) { - auto Name = Obj.getSectionName(SecRef); - if (!Name) - return Name.takeError(); - - // Skip Dwarf sections. - if (isDwarfSection(*Name)) { - LLVM_DEBUG({ - dbgs() << *Name - << " is a debug section: No graph section will be created.\n"; - }); - continue; - } - - sys::Memory::ProtectionFlags Prot; - if (SecRef.sh_flags & ELF::SHF_EXECINSTR) { - Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_EXEC); - } else { - Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_WRITE); - } - uint64_t Address = SecRef.sh_addr; - uint64_t Size = SecRef.sh_size; - uint64_t Flags = SecRef.sh_flags; - uint64_t Alignment = SecRef.sh_addralign; - const char *Data = nullptr; - // for now we just use this to skip the "undefined" section, probably need - // to revist - if (Size == 0) - continue; - - // FIXME: Use flags. - (void)Flags; - - LLVM_DEBUG({ - dbgs() << " " << *Name << ": " << formatv("{0:x16}", Address) << " -- " - << formatv("{0:x16}", Address + Size) << ", align: " << Alignment - << " Flags: " << formatv("{0:x}", Flags) << "\n"; - }); - - if (SecRef.sh_type != ELF::SHT_NOBITS) { - // .sections() already checks that the data is not beyond the end of - // file - auto contents = Obj.getSectionContentsAsArray<char>(SecRef); - if (!contents) - return contents.takeError(); - - Data = contents->data(); - // TODO protection flags. - // for now everything is - auto §ion = G->createSection(*Name, Prot); - // Do this here because we have it, but move it into graphify later - G->createContentBlock(section, StringRef(Data, Size), Address, - Alignment, 0); - if (SecRef.sh_type == ELF::SHT_SYMTAB) - // TODO: Dynamic? - SymTab = SecRef; - } else { - auto &Section = G->createSection(*Name, Prot); - G->createZeroFillBlock(Section, Size, Address, Alignment, 0); - } - } - - return Error::success(); - } - - Error addRelocations() { + Error addRelocations() override { LLVM_DEBUG(dbgs() << "Adding relocations\n"); // TODO a partern is forming of iterate some sections but only give me // ones I am interested, i should abstract that concept some where - for (auto &SecRef : sections) { + for (auto &SecRef : Sections) { if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL) continue; // TODO can the elf obj file do this for me? @@ -448,19 +315,20 @@ private: << "Name: " << Obj.getRelocationTypeName(Type) << "\n"; }); auto SymbolIndex = Rela.getSymbol(false); - auto Symbol = Obj.getRelocationSymbol(Rela, &SymTab); + auto Symbol = Obj.getRelocationSymbol(Rela, SymTabSec); if (!Symbol) return Symbol.takeError(); auto BlockToFix = *(JITSection->blocks().begin()); - auto *TargetSymbol = JITSymbolTable[SymbolIndex]; + auto *TargetSymbol = getGraphSymbol(SymbolIndex); if (!TargetSymbol) { return make_error<llvm::StringError>( "Could not find symbol at given index, did you add it to " - "JITSymbolTable? index: " + std::to_string(SymbolIndex) - + ", shndx: " + std::to_string((*Symbol)->st_shndx) + - " Size of table: " + std::to_string(JITSymbolTable.size()), + "JITSymbolTable? index: " + + std::to_string(SymbolIndex) + + ", shndx: " + std::to_string((*Symbol)->st_shndx) + + " Size of table: " + std::to_string(GraphSymbols.size()), llvm::inconvertibleErrorCode()); } uint64_t Addend = Rela.r_addend; @@ -489,203 +357,11 @@ private: return Error::success(); } - Error graphifyRegularSymbols() { - - // TODO: ELF supports beyond SHN_LORESERVE, - // need to perf test how a vector vs map handles those cases - - std::vector<std::vector<object::ELFFile<object::ELF64LE>::Elf_Shdr_Range *>> - SecIndexToSymbols; - - LLVM_DEBUG(dbgs() << "Creating graph symbols...\n"); - - for (auto SecRef : sections) { - - if (SecRef.sh_type != ELF::SHT_SYMTAB && - SecRef.sh_type != ELF::SHT_DYNSYM) - continue; - auto Symbols = Obj.symbols(&SecRef); - if (!Symbols) - return Symbols.takeError(); - - auto StrTabSec = Obj.getSection(SecRef.sh_link); - if (!StrTabSec) - return StrTabSec.takeError(); - auto StringTable = Obj.getStringTable(**StrTabSec); - if (!StringTable) - return StringTable.takeError(); - auto Name = Obj.getSectionName(SecRef); - if (!Name) - return Name.takeError(); - - LLVM_DEBUG(dbgs() << "Processing symbol section " << *Name << ":\n"); - - auto Section = G->findSectionByName(*Name); - if (!Section) - return make_error<llvm::StringError>("Could not find a section " + - *Name, - llvm::inconvertibleErrorCode()); - // we only have one for now - auto blocks = Section->blocks(); - if (blocks.empty()) - return make_error<llvm::StringError>("Section has no block", - llvm::inconvertibleErrorCode()); - int SymbolIndex = -1; - for (auto SymRef : *Symbols) { - ++SymbolIndex; - auto Type = SymRef.getType(); - - if (Type == ELF::STT_FILE || SymbolIndex == 0) - continue; - // these should do it for now - // if(Type != ELF::STT_NOTYPE && - // Type != ELF::STT_OBJECT && - // Type != ELF::STT_FUNC && - // Type != ELF::STT_SECTION && - // Type != ELF::STT_COMMON) { - // continue; - // } - auto Name = SymRef.getName(*StringTable); - // I am not sure on If this is going to hold as an invariant. Revisit. - if (!Name) - return Name.takeError(); - - if (SymRef.isCommon()) { - // Symbols in SHN_COMMON refer to uninitialized data. The st_value - // field holds alignment constraints. - Symbol &S = - G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, - SymRef.st_size, SymRef.getValue(), false); - JITSymbolTable[SymbolIndex] = &S; - continue; - } - - // Map Visibility and Binding to Scope and Linkage: - Linkage L = Linkage::Strong; - Scope S = Scope::Default; - - switch (SymRef.getBinding()) { - case ELF::STB_LOCAL: - S = Scope::Local; - break; - case ELF::STB_GLOBAL: - // Nothing to do here. - break; - case ELF::STB_WEAK: - L = Linkage::Weak; - break; - default: - return make_error<StringError>("Unrecognized symbol binding for " + - *Name, - inconvertibleErrorCode()); - } - - switch (SymRef.getVisibility()) { - case ELF::STV_DEFAULT: - case ELF::STV_PROTECTED: - // FIXME: Make STV_DEFAULT symbols pre-emptible? This probably needs - // Orc support. - // Otherwise nothing to do here. - break; - case ELF::STV_HIDDEN: - // Default scope -> Hidden scope. No effect on local scope. - if (S == Scope::Default) - S = Scope::Hidden; - break; - case ELF::STV_INTERNAL: - return make_error<StringError>("Unrecognized symbol visibility for " + - *Name, - inconvertibleErrorCode()); - } - - if (SymRef.isDefined() && - (Type == ELF::STT_FUNC || Type == ELF::STT_OBJECT || - Type == ELF::STT_SECTION)) { - - auto DefinedSection = Obj.getSection(SymRef.st_shndx); - if (!DefinedSection) - return DefinedSection.takeError(); - auto sectName = Obj.getSectionName(**DefinedSection); - if (!sectName) - return Name.takeError(); - - // Skip debug section symbols. - if (isDwarfSection(*sectName)) - continue; - - auto JitSection = G->findSectionByName(*sectName); - if (!JitSection) - return make_error<llvm::StringError>( - "Could not find the JitSection " + *sectName, - llvm::inconvertibleErrorCode()); - auto bs = JitSection->blocks(); - if (bs.empty()) - return make_error<llvm::StringError>( - "Section has no block", llvm::inconvertibleErrorCode()); - - auto *B = *bs.begin(); - LLVM_DEBUG({ dbgs() << " " << *Name << " at index " << SymbolIndex << "\n"; }); - if (SymRef.getType() == ELF::STT_SECTION) - *Name = *sectName; - auto &Sym = G->addDefinedSymbol( - *B, SymRef.getValue(), *Name, SymRef.st_size, L, S, - SymRef.getType() == ELF::STT_FUNC, false); - JITSymbolTable[SymbolIndex] = &Sym; - } else if (SymRef.isUndefined() && SymRef.isExternal()) { - auto &Sym = G->addExternalSymbol(*Name, SymRef.st_size, L); - JITSymbolTable[SymbolIndex] = &Sym; - } else - LLVM_DEBUG({ - dbgs() - << "Not creating graph symbol for normalized symbol at index " - << SymbolIndex << ", \"" << *Name << "\"\n"; - }); - - // TODO: The following has to be implmented. - // leaving commented out to save time for future patchs - /* - G->addAbsoluteSymbol(*Name, SymRef.getValue(), SymRef.st_size, - Linkage::Strong, Scope::Default, false); - */ - } - } - return Error::success(); - } - public: ELFLinkGraphBuilder_x86_64(StringRef FileName, const object::ELFFile<object::ELF64LE> &Obj) - : G(std::make_unique<LinkGraph>(FileName.str(), - Triple("x86_64-unknown-linux"), - getPointerSize(Obj), getEndianness(Obj))), - Obj(Obj) {} - - Expected<std::unique_ptr<LinkGraph>> buildGraph() { - // Sanity check: we only operate on relocatable objects. - if (!isRelocatable()) - return make_error<JITLinkError>("Object is not a relocatable ELF"); - - auto Secs = Obj.sections(); - - if (!Secs) { - return Secs.takeError(); - } - sections = *Secs; - - if (auto Err = createNormalizedSections()) - return std::move(Err); - - if (auto Err = createNormalizedSymbols()) - return std::move(Err); - - if (auto Err = graphifyRegularSymbols()) - return std::move(Err); - - if (auto Err = addRelocations()) - return std::move(Err); - - return std::move(G); - } + : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName, + getELFX86RelocationKindName) {} }; class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> { @@ -695,27 +371,66 @@ public: ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) - : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { + getPassConfig().PostAllocationPasses.push_back( + [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); + } private: - StringRef getEdgeKindName(Edge::Kind R) const override { - return getELFX86RelocationKindName(R); - } + Symbol *GOTSymbol = nullptr; + + Error getOrCreateGOTSymbol(LinkGraph &G) { + auto DefineExternalGOTSymbolIfPresent = + createDefineExternalSectionStartAndEndSymbolsPass( + [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc { + if (Sym.getName() == ELFGOTSymbolName) + if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) { + GOTSymbol = &Sym; + return {*GOTSection, true}; + } + return {}; + }); + + // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an + // external. + if (auto Err = DefineExternalGOTSymbolIfPresent(G)) + return Err; - static Error targetOutOfRangeError(const Block &B, const Edge &E) { - std::string ErrMsg; - { - raw_string_ostream ErrStream(ErrMsg); - ErrStream << "Relocation target out of range: "; - printEdge(ErrStream, B, E, getELFX86RelocationKindName(E.getKind())); - ErrStream << "\n"; + // If we succeeded then we're done. + if (GOTSymbol) + return Error::success(); + + // Otherwise look for a GOT section: If it already has a start symbol we'll + // record it, otherwise we'll create our own. + // If there's a GOT section but we didn't find an external GOT symbol... + if (auto *GOTSection = G.findSectionByName(ELFGOTSectionName)) { + + // Check for an existing defined symbol. + for (auto *Sym : GOTSection->symbols()) + if (Sym->getName() == ELFGOTSymbolName) { + GOTSymbol = Sym; + return Error::success(); + } + + // If there's no defined symbol then create one. + SectionRange SR(*GOTSection); + if (SR.empty()) + GOTSymbol = &G.addAbsoluteSymbol(ELFGOTSymbolName, 0, 0, + Linkage::Strong, Scope::Local, true); + else + GOTSymbol = + &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0, + Linkage::Strong, Scope::Local, false, true); } - return make_error<JITLinkError>(std::move(ErrMsg)); + + return Error::success(); } - Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { using namespace ELF_x86_64_Edges; using namespace llvm::support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); char *FixupPtr = BlockWorkingMem + E.getOffset(); JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); switch (E.getKind()) { @@ -724,10 +439,15 @@ private: case ELFX86RelocationKind::PCRel32: case ELFX86RelocationKind::PCRel32GOTLoad: { int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; - if (Value < std::numeric_limits<int32_t>::min() || - Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(B, E); - *(little32_t *)FixupPtr = Value; + if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value))) + *(little32_t *)FixupPtr = Value; + else + return makeTargetOutOfRangeError(G, B, E); + break; + } + case ELFX86RelocationKind::PCRel64: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + *(little64_t *)FixupPtr = Value; break; } case ELFX86RelocationKind::Pointer64: { @@ -735,11 +455,51 @@ private: *(ulittle64_t *)FixupPtr = Value; break; } + case ELFX86RelocationKind::Delta32: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value))) + *(little32_t *)FixupPtr = Value; + else + return makeTargetOutOfRangeError(G, B, E); + break; + } case ELFX86RelocationKind::Delta64: { int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; *(little64_t *)FixupPtr = Value; break; } + case ELFX86RelocationKind::NegDelta32: { + int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + if (LLVM_LIKELY(x86_64::isInRangeForImmS32(Value))) + *(little32_t *)FixupPtr = Value; + else + return makeTargetOutOfRangeError(G, B, E); + break; + } + case ELFX86RelocationKind::NegDelta64: { + int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + *(little64_t *)FixupPtr = Value; + break; + } + case ELFX86RelocationKind::GOT64: + case ELFX86RelocationKind::GOTOFF64: { + // GOT64: Offset of GOT entry within GOT. + // GOTOFF64: Offset from GOT base to target. + // The expressions are the same in both cases, but in the GOT64 case the + // edge will have been fixed to point at the GOT entry, and in the + // GOTOFF64 case it will still point at the original target. + assert(GOTSymbol && "No GOT section symbol"); + int64_t Value = + E.getTarget().getAddress() - GOTSymbol->getAddress() + E.getAddend(); + *(little64_t *)FixupPtr = Value; + break; + } + default: + LLVM_DEBUG({ + dbgs() << "Bad edge: " << getELFX86RelocationKindName(E.getKind()) + << "\n"; + }); + llvm_unreachable("Unsupported relocation"); } return Error::success(); } @@ -762,6 +522,24 @@ createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { .buildGraph(); } +static SectionRangeSymbolDesc +identifyELFSectionStartAndEndSymbols(LinkGraph &G, Symbol &Sym) { + constexpr StringRef StartSymbolPrefix = "__start"; + constexpr StringRef EndSymbolPrefix = "__end"; + + auto SymName = Sym.getName(); + if (SymName.startswith(StartSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(StartSymbolPrefix.size()))) + return {*Sec, true}; + } else if (SymName.startswith(EndSymbolPrefix)) { + if (auto *Sec = + G.findSectionByName(SymName.drop_front(EndSymbolPrefix.size()))) + return {*Sec, false}; + } + return {}; +} + void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) { PassConfiguration Config; @@ -771,6 +549,7 @@ void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, Config.PrePrunePasses.push_back(EHFrameSplitter(".eh_frame")); Config.PrePrunePasses.push_back(EHFrameEdgeFixer( ".eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32)); + Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); // Construct a JITLinker and run the link function. // Add a mark-live pass. @@ -780,32 +559,65 @@ void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, Config.PrePrunePasses.push_back(markAllSymbolsLive); // Add an in-place GOT/Stubs pass. - Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { - ELF_x86_64_GOTAndStubsBuilder(G).run(); - return Error::success(); - }); + Config.PostPrunePasses.push_back( + PerGraphGOTAndPLTStubsBuilder_ELF_x86_64::asPass); + + // Resolve any external section start / end symbols. + Config.PostAllocationPasses.push_back( + createDefineExternalSectionStartAndEndSymbolsPass( + identifyELFSectionStartAndEndSymbols)); // Add GOT/Stubs optimizer pass. Config.PreFixupPasses.push_back(optimizeELF_x86_64_GOTAndStubs); } - if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config)) + if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); } -StringRef getELFX86RelocationKindName(Edge::Kind R) { +const char *getELFX86RelocationKindName(Edge::Kind R) { switch (R) { - case PCRel32: - return "PCRel32"; - case Pointer64: - return "Pointer64"; - case PCRel32GOTLoad: - return "PCRel32GOTLoad"; case Branch32: return "Branch32"; case Branch32ToStub: return "Branch32ToStub"; + case Pointer32: + return "Pointer32"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case PCRel32: + return "PCRel32"; + case PCRel32Minus1: + return "PCRel32Minus1"; + case PCRel32Minus2: + return "PCRel32Minus2"; + case PCRel32Minus4: + return "PCRel32Minus4"; + case PCRel32Anon: + return "PCRel32Anon"; + case PCRel32Minus1Anon: + return "PCRel32Minus1Anon"; + case PCRel32Minus2Anon: + return "PCRel32Minus2Anon"; + case PCRel32Minus4Anon: + return "PCRel32Minus4Anon"; + case PCRel32GOTLoad: + return "PCRel32GOTLoad"; + case PCRel32GOT: + return "PCRel32GOT"; + case PCRel32TLV: + return "PCRel32TLV"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; } return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index 93dfba9c759b..a4976f2f3d27 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -51,7 +51,7 @@ namespace jitlink { char JITLinkError::ID = 0; -void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg << "\n"; } +void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg; } std::error_code JITLinkError::convertToErrorCode() const { return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); @@ -92,8 +92,8 @@ const char *getScopeName(Scope S) { raw_ostream &operator<<(raw_ostream &OS, const Block &B) { return OS << formatv("{0:x16}", B.getAddress()) << " -- " - << formatv("{0:x16}", B.getAddress() + B.getSize()) << ": " - << "size = " << formatv("{0:x}", B.getSize()) << ", " + << formatv("{0:x8}", B.getAddress() + B.getSize()) << ": " + << "size = " << formatv("{0:x8}", B.getSize()) << ", " << (B.isZeroFill() ? "zero-fill" : "content") << ", align = " << B.getAlignment() << ", align-ofs = " << B.getAlignmentOffset() @@ -101,39 +101,14 @@ raw_ostream &operator<<(raw_ostream &OS, const Block &B) { } raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) { - OS << "<"; - if (Sym.getName().empty()) - OS << "*anon*"; - else - OS << Sym.getName(); - OS << ": flags = "; - switch (Sym.getLinkage()) { - case Linkage::Strong: - OS << 'S'; - break; - case Linkage::Weak: - OS << 'W'; - break; - } - switch (Sym.getScope()) { - case Scope::Default: - OS << 'D'; - break; - case Scope::Hidden: - OS << 'H'; - break; - case Scope::Local: - OS << 'L'; - break; - } - OS << (Sym.isLive() ? '+' : '-') - << ", size = " << formatv("{0:x}", Sym.getSize()) - << ", addr = " << formatv("{0:x16}", Sym.getAddress()) << " (" - << formatv("{0:x16}", Sym.getAddressable().getAddress()) << " + " - << formatv("{0:x}", Sym.getOffset()); - if (Sym.isDefined()) - OS << " " << Sym.getBlock().getSection().getName(); - OS << ")>"; + OS << formatv("{0:x16}", Sym.getAddress()) << " (" + << (Sym.isDefined() ? "block" : "addressable") << " + " + << formatv("{0:x8}", Sym.getOffset()) + << "): size: " << formatv("{0:x8}", Sym.getSize()) + << ", linkage: " << formatv("{0:6}", getLinkageName(Sym.getLinkage())) + << ", scope: " << formatv("{0:8}", getScopeName(Sym.getScope())) << ", " + << (Sym.isLive() ? "live" : "dead") << " - " + << (Sym.hasName() ? Sym.getName() : "<anonymous symbol>"); return OS; } @@ -193,12 +168,12 @@ Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex, ? createZeroFillBlock(B.getSection(), SplitIndex, B.getAddress(), B.getAlignment(), B.getAlignmentOffset()) : createContentBlock( - B.getSection(), B.getContent().substr(0, SplitIndex), + B.getSection(), B.getContent().slice(0, SplitIndex), B.getAddress(), B.getAlignment(), B.getAlignmentOffset()); // Modify B to cover [ SplitIndex, B.size() ). B.setAddress(B.getAddress() + SplitIndex); - B.setContent(B.getContent().substr(SplitIndex)); + B.setContent(B.getContent().slice(SplitIndex)); B.setAlignmentOffset((B.getAlignmentOffset() + SplitIndex) % B.getAlignment()); @@ -251,42 +226,103 @@ Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex, return NewBlock; } -void LinkGraph::dump(raw_ostream &OS, - std::function<StringRef(Edge::Kind)> EdgeKindToName) { - if (!EdgeKindToName) - EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; - - OS << "Symbols:\n"; - for (auto *Sym : defined_symbols()) { - OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym - << "\n"; - if (Sym->isDefined()) { - for (auto &E : Sym->getBlock().edges()) { - OS << " "; - StringRef EdgeName = (E.getKind() < Edge::FirstRelocation - ? getGenericEdgeKindName(E.getKind()) - : EdgeKindToName(E.getKind())); - - if (!EdgeName.empty()) - printEdge(OS, Sym->getBlock(), E, EdgeName); - else { - auto EdgeNumberString = std::to_string(E.getKind()); - printEdge(OS, Sym->getBlock(), E, EdgeNumberString); - } - OS << "\n"; +void LinkGraph::dump(raw_ostream &OS) { + DenseMap<Block *, std::vector<Symbol *>> BlockSymbols; + + // Map from blocks to the symbols pointing at them. + for (auto *Sym : defined_symbols()) + BlockSymbols[&Sym->getBlock()].push_back(Sym); + + // For each block, sort its symbols by something approximating + // relevance. + for (auto &KV : BlockSymbols) + llvm::sort(KV.second, [](const Symbol *LHS, const Symbol *RHS) { + if (LHS->getOffset() != RHS->getOffset()) + return LHS->getOffset() < RHS->getOffset(); + if (LHS->getLinkage() != RHS->getLinkage()) + return LHS->getLinkage() < RHS->getLinkage(); + if (LHS->getScope() != RHS->getScope()) + return LHS->getScope() < RHS->getScope(); + if (LHS->hasName()) { + if (!RHS->hasName()) + return true; + return LHS->getName() < RHS->getName(); } + return false; + }); + + for (auto &Sec : sections()) { + OS << "section " << Sec.getName() << ":\n\n"; + + std::vector<Block *> SortedBlocks; + llvm::copy(Sec.blocks(), std::back_inserter(SortedBlocks)); + llvm::sort(SortedBlocks, [](const Block *LHS, const Block *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + for (auto *B : SortedBlocks) { + OS << " block " << formatv("{0:x16}", B->getAddress()) + << " size = " << formatv("{0:x8}", B->getSize()) + << ", align = " << B->getAlignment() + << ", alignment-offset = " << B->getAlignmentOffset(); + if (B->isZeroFill()) + OS << ", zero-fill"; + OS << "\n"; + + auto BlockSymsI = BlockSymbols.find(B); + if (BlockSymsI != BlockSymbols.end()) { + OS << " symbols:\n"; + auto &Syms = BlockSymsI->second; + for (auto *Sym : Syms) + OS << " " << *Sym << "\n"; + } else + OS << " no symbols\n"; + + if (!B->edges_empty()) { + OS << " edges:\n"; + std::vector<Edge> SortedEdges; + llvm::copy(B->edges(), std::back_inserter(SortedEdges)); + llvm::sort(SortedEdges, [](const Edge &LHS, const Edge &RHS) { + return LHS.getOffset() < RHS.getOffset(); + }); + for (auto &E : SortedEdges) { + OS << " " << formatv("{0:x16}", B->getFixupAddress(E)) + << " (block + " << formatv("{0:x8}", E.getOffset()) + << "), addend = "; + if (E.getAddend() >= 0) + OS << formatv("+{0:x8}", E.getAddend()); + else + OS << formatv("-{0:x8}", -E.getAddend()); + OS << ", kind = " << getEdgeKindName(E.getKind()) << ", target = "; + if (E.getTarget().hasName()) + OS << E.getTarget().getName(); + else + OS << "addressable@" + << formatv("{0:x16}", E.getTarget().getAddress()) << "+" + << formatv("{0:x8}", E.getTarget().getOffset()); + OS << "\n"; + } + } else + OS << " no edges\n"; + OS << "\n"; } } OS << "Absolute symbols:\n"; - for (auto *Sym : absolute_symbols()) - OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym - << "\n"; - - OS << "External symbols:\n"; - for (auto *Sym : external_symbols()) - OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym - << "\n"; + if (!llvm::empty(absolute_symbols())) { + for (auto *Sym : absolute_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym + << "\n"; + } else + OS << " none\n"; + + OS << "\nExternal symbols:\n"; + if (!llvm::empty(external_symbols())) { + for (auto *Sym : external_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym + << "\n"; + } else + OS << " none\n"; } raw_ostream &operator<<(raw_ostream &OS, const SymbolLookupFlags &LF) { @@ -311,7 +347,7 @@ LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { return LinkGraphPassFunction(); } -Error JITLinkContext::modifyPassConfig(const Triple &TT, +Error JITLinkContext::modifyPassConfig(LinkGraph &G, PassConfiguration &Config) { return Error::success(); } @@ -322,14 +358,47 @@ Error markAllSymbolsLive(LinkGraph &G) { return Error::success(); } +Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B, + const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + Section &Sec = B.getSection(); + ErrStream << "In graph " << G.getName() << ", section " << Sec.getName() + << ": relocation target "; + if (E.getTarget().hasName()) + ErrStream << "\"" << E.getTarget().getName() << "\" "; + ErrStream << "at address " << formatv("{0:x}", E.getTarget().getAddress()); + ErrStream << " is out of range of " << G.getEdgeKindName(E.getKind()) + << " fixup at " << formatv("{0:x}", B.getFixupAddress(E)) << " ("; + + Symbol *BestSymbolForBlock = nullptr; + for (auto *Sym : Sec.symbols()) + if (&Sym->getBlock() == &B && Sym->hasName() && Sym->getOffset() == 0 && + (!BestSymbolForBlock || + Sym->getScope() < BestSymbolForBlock->getScope() || + Sym->getLinkage() < BestSymbolForBlock->getLinkage())) + BestSymbolForBlock = Sym; + + if (BestSymbolForBlock) + ErrStream << BestSymbolForBlock->getName() << ", "; + else + ErrStream << "<anonymous block> @ "; + + ErrStream << formatv("{0:x}", B.getAddress()) << " + " + << formatv("{0:x}", E.getOffset()) << ")"; + } + return make_error<JITLinkError>(std::move(ErrMsg)); +} + Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) { auto Magic = identify_magic(ObjectBuffer.getBuffer()); switch (Magic) { case file_magic::macho_object: - return createLinkGraphFromMachOObject(std::move(ObjectBuffer)); + return createLinkGraphFromMachOObject(ObjectBuffer); case file_magic::elf_relocatable: - return createLinkGraphFromELFObject(std::move(ObjectBuffer)); + return createLinkGraphFromELFObject(ObjectBuffer); default: return make_error<JITLinkError>("Unsupported file format"); }; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 7a5e014f223d..5b163ab6316d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -34,14 +34,14 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" pre-pruning:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); prune(*G); LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" post-pruning:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); // Run post-pruning passes. @@ -58,7 +58,7 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" before post-allocation passes:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); // Run post-allocation passes. @@ -66,14 +66,27 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { return Ctx->notifyFailed(std::move(Err)); // Notify client that the defined symbols have been assigned addresses. - LLVM_DEBUG( - { dbgs() << "Resolving symbols defined in " << G->getName() << "\n"; }); + LLVM_DEBUG(dbgs() << "Resolving symbols defined in " << G->getName() << "\n"); if (auto Err = Ctx->notifyResolved(*G)) return Ctx->notifyFailed(std::move(Err)); auto ExternalSymbols = getExternalSymbolNames(); + // If there are no external symbols then proceed immediately with phase 2. + if (ExternalSymbols.empty()) { + LLVM_DEBUG({ + dbgs() << "No external symbols for " << G->getName() + << ". Proceeding immediately with link phase 2.\n"; + }); + // FIXME: Once callee expressions are defined to be sequenced before + // argument expressions (c++17) we can simplify this. See below. + auto &TmpSelf = *Self; + TmpSelf.linkPhase2(std::move(Self), AsyncLookupResult(), std::move(Layout)); + return; + } + + // Otherwise look up the externals. LLVM_DEBUG({ dbgs() << "Issuing lookup for external symbols for " << G->getName() << " (may trigger materialization/linking of other graphs)...\n"; @@ -121,7 +134,7 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" before pre-fixup passes:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); if (auto Err = runPasses(Passes.PreFixupPasses)) @@ -129,7 +142,7 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); // Fix up block content. @@ -138,7 +151,7 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" after copy-and-fixup:\n"; - dumpGraph(dbgs()); + G->dump(dbgs()); }); if (auto Err = runPasses(Passes.PostFixupPasses)) @@ -391,7 +404,7 @@ void JITLinkerBase::copyBlockContentToWorkingMemory( memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size()); // Point the block's content to the fixed up buffer. - B->setContent(StringRef(BlockDataPtr, B->getContent().size())); + B->setMutableContent({BlockDataPtr, B->getContent().size()}); // Update block end pointer. LastBlockEnd = BlockDataPtr + B->getContent().size(); @@ -415,11 +428,6 @@ void JITLinkerBase::deallocateAndBailOut(Error Err) { Ctx->notifyFailed(joinErrors(std::move(Err), Alloc->deallocate())); } -void JITLinkerBase::dumpGraph(raw_ostream &OS) { - assert(G && "Graph is not set yet"); - G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); -} - void prune(LinkGraph &G) { std::vector<Symbol *> Worklist; DenseSet<Block *> VisitedBlocks; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index 1d28f5006b2b..6b815fe4fb31 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -51,20 +51,29 @@ protected: using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>; + // Returns the PassConfiguration for this instance. This can be used by + // JITLinkerBase implementations to add late passes that reference their + // own data structures (e.g. for ELF implementations to locate / construct + // a GOT start symbol prior to fixup). + PassConfiguration &getPassConfig() { return Passes; } + // Phase 1: // 1.1: Run pre-prune passes // 1.2: Prune graph // 1.3: Run post-prune passes // 1.4: Sort blocks into segments - // 1.5: Allocate segment memory - // 1.6: Identify externals and make an async call to resolve function + // 1.5: Allocate segment memory, update node vmaddrs to target vmaddrs + // 1.6: Run post-allocation passes + // 1.7: Notify context of final assigned symbol addresses + // 1.8: Identify external symbols and make an async call to resolve void linkPhase1(std::unique_ptr<JITLinkerBase> Self); // Phase 2: // 2.1: Apply resolution results - // 2.2: Fix up block contents - // 2.3: Call OnResolved callback - // 2.3: Make an async call to transfer and finalize memory. + // 2.2: Run pre-fixup passes + // 2.3: Fix up block contents + // 2.4: Run post-fixup passes + // 2.5: Make an async call to transfer and finalize memory. void linkPhase2(std::unique_ptr<JITLinkerBase> Self, Expected<AsyncLookupResult> LookupResult, SegmentLayoutMap Layout); @@ -73,9 +82,6 @@ protected: // 3.1: Call OnFinalized callback, handing off allocation. void linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err); - // For debug dumping of the link graph. - virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; - // Align a JITTargetAddress to conform with block alignment requirements. static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) { uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment(); @@ -106,8 +112,6 @@ private: JITLinkMemoryManager::Allocation &Alloc); void deallocateAndBailOut(Error Err); - void dumpGraph(raw_ostream &OS); - std::unique_ptr<JITLinkContext> Ctx; std::unique_ptr<LinkGraph> G; PassConfiguration Passes; @@ -155,8 +159,7 @@ private: continue; // Dispatch to LinkerImpl for fixup. - auto *BlockData = const_cast<char *>(B->getContent().data()); - if (auto Err = impl().applyFixup(*B, E, BlockData)) + if (auto Err = impl().applyFixup(G, *B, E)) return Err; } } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index fbbb29e9164a..36067ccf2753 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -113,10 +113,12 @@ InProcessMemoryManager::allocate(const JITLinkDylib *JD, uint64_t SegmentSize = alignTo(Seg.getContentSize() + Seg.getZeroFillSize(), sys::Process::getPageSizeEstimate()); + assert(SlabRemaining.allocatedSize() >= SegmentSize && + "Mapping exceeds allocation"); sys::MemoryBlock SegMem(SlabRemaining.base(), SegmentSize); SlabRemaining = sys::MemoryBlock((char *)SlabRemaining.base() + SegmentSize, - SegmentSize); + SlabRemaining.allocatedSize() - SegmentSize); // Zero out the zero-fill memory. memset(static_cast<char *>(SegMem.base()) + Seg.getContentSize(), 0, @@ -125,6 +127,7 @@ InProcessMemoryManager::allocate(const JITLinkDylib *JD, // Record the block for this segment. Blocks[KV.first] = std::move(SegMem); } + return std::unique_ptr<InProcessMemoryManager::Allocation>( new IPMMAlloc(std::move(Blocks))); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO.cpp index e9327df6da41..eda2b8811deb 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO.cpp @@ -64,9 +64,9 @@ createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer) { switch (CPUType) { case MachO::CPU_TYPE_ARM64: - return createLinkGraphFromMachOObject_arm64(std::move(ObjectBuffer)); + return createLinkGraphFromMachOObject_arm64(ObjectBuffer); case MachO::CPU_TYPE_X86_64: - return createLinkGraphFromMachOObject_x86_64(std::move(ObjectBuffer)); + return createLinkGraphFromMachOObject_x86_64(ObjectBuffer); } return make_error<JITLinkError>("MachO-64 CPU type not valid"); } else diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp index 4602154eb579..03a8b98dff18 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -45,12 +45,13 @@ Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() { return std::move(G); } -MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, - Triple TT) +MachOLinkGraphBuilder::MachOLinkGraphBuilder( + const object::MachOObjectFile &Obj, Triple TT, + LinkGraph::GetEdgeKindNameFunction GetEdgeKindName) : Obj(Obj), - G(std::make_unique<LinkGraph>(std::string(Obj.getFileName()), - std::move(TT), getPointerSize(Obj), - getEndianness(Obj))) {} + G(std::make_unique<LinkGraph>( + std::string(Obj.getFileName()), std::move(TT), getPointerSize(Obj), + getEndianness(Obj), std::move(GetEdgeKindName))) {} void MachOLinkGraphBuilder::addCustomSectionParser( StringRef SectionName, SectionParserFunction Parser) { @@ -84,6 +85,17 @@ bool MachOLinkGraphBuilder::isDebugSection(const NormalizedSection &NSec) { strcmp(NSec.SegName, "__DWARF") == 0); } +bool MachOLinkGraphBuilder::isZeroFillSection(const NormalizedSection &NSec) { + switch (NSec.Flags & MachO::SECTION_TYPE) { + case MachO::S_ZEROFILL: + case MachO::S_GB_ZEROFILL: + case MachO::S_THREAD_LOCAL_ZEROFILL: + return true; + default: + return false; + } +} + unsigned MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { return Obj.is64Bit() ? 8 : 4; @@ -115,10 +127,6 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { auto SecIndex = Obj.getSectionIndex(SecRef.getRawDataRefImpl()); - auto Name = SecRef.getName(); - if (!Name) - return Name.takeError(); - if (Obj.is64Bit()) { const MachO::section_64 &Sec64 = Obj.getSection64(SecRef.getRawDataRefImpl()); @@ -149,24 +157,20 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { } LLVM_DEBUG({ - dbgs() << " " << *Name << ": " << formatv("{0:x16}", NSec.Address) - << " -- " << formatv("{0:x16}", NSec.Address + NSec.Size) + dbgs() << " " << NSec.SegName << "," << NSec.SectName << ": " + << formatv("{0:x16}", NSec.Address) << " -- " + << formatv("{0:x16}", NSec.Address + NSec.Size) << ", align: " << NSec.Alignment << ", index: " << SecIndex << "\n"; }); // Get the section data if any. - { - unsigned SectionType = NSec.Flags & MachO::SECTION_TYPE; - if (SectionType != MachO::S_ZEROFILL && - SectionType != MachO::S_GB_ZEROFILL) { - - if (DataOffset + NSec.Size > Obj.getData().size()) - return make_error<JITLinkError>( - "Section data extends past end of file"); + if (!isZeroFillSection(NSec)) { + if (DataOffset + NSec.Size > Obj.getData().size()) + return make_error<JITLinkError>( + "Section data extends past end of file"); - NSec.Data = Obj.getData().data() + DataOffset; - } + NSec.Data = Obj.getData().data() + DataOffset; } // Get prot flags. @@ -180,11 +184,15 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | sys::Memory::MF_WRITE); - if (!isDebugSection(NSec)) - NSec.GraphSection = &G->createSection(*Name, Prot); - else + if (!isDebugSection(NSec)) { + auto FullyQualifiedName = + G->allocateString(StringRef(NSec.SegName) + "," + NSec.SectName); + NSec.GraphSection = &G->createSection( + StringRef(FullyQualifiedName.data(), FullyQualifiedName.size()), + Prot); + } else LLVM_DEBUG({ - dbgs() << " " << *Name + dbgs() << " " << NSec.SegName << "," << NSec.SectName << " is a debug section: No graph section will be created.\n"; }); @@ -316,8 +324,8 @@ void MachOLinkGraphBuilder::addSectionStartSymAndBlock( Section &GraphSec, uint64_t Address, const char *Data, uint64_t Size, uint32_t Alignment, bool IsLive) { Block &B = - Data ? G->createContentBlock(GraphSec, StringRef(Data, Size), Address, - Alignment, 0) + Data ? G->createContentBlock(GraphSec, ArrayRef<char>(Data, Size), + Address, Alignment, 0) : G->createZeroFillBlock(GraphSec, Size, Address, Alignment, 0); auto &Sym = G->addAnonymousSymbol(B, 0, Size, false, IsLive); assert(!AddrToCanonicalSymbol.count(Sym.getAddress()) && @@ -409,10 +417,16 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { << " as it has a custom parser.\n"; }); continue; + } else if ((NSec.Flags & MachO::SECTION_TYPE) == + MachO::S_CSTRING_LITERALS) { + if (auto Err = graphifyCStringSection( + NSec, std::move(SecIndexToSymbols[SecIndex]))) + return Err; + continue; } else LLVM_DEBUG({ - dbgs() << " Processing section " << NSec.GraphSection->getName() - << "...\n"; + dbgs() << " Graphifying regular section " + << NSec.GraphSection->getName() << "...\n"; }); bool SectionIsNoDeadStrip = NSec.Flags & MachO::S_ATTR_NO_DEAD_STRIP; @@ -508,8 +522,8 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { NSec.Data ? G->createContentBlock( *NSec.GraphSection, - StringRef(NSec.Data + BlockOffset, BlockSize), BlockStart, - NSec.Alignment, BlockStart % NSec.Alignment) + ArrayRef<char>(NSec.Data + BlockOffset, BlockSize), + BlockStart, NSec.Alignment, BlockStart % NSec.Alignment) : G->createZeroFillBlock(*NSec.GraphSection, BlockSize, BlockStart, NSec.Alignment, BlockStart % NSec.Alignment); @@ -523,34 +537,14 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { bool SymLive = (NSym.Desc & MachO::N_NO_DEAD_STRIP) || SectionIsNoDeadStrip; - LLVM_DEBUG({ - dbgs() << " " << formatv("{0:x16}", NSym.Value) << " -- " - << formatv("{0:x16}", SymEnd) << ": "; - if (!NSym.Name) - dbgs() << "<anonymous symbol>"; - else - dbgs() << NSym.Name; - if (SymLive) - dbgs() << " [no-dead-strip]"; - if (LastCanonicalAddr == NSym.Value) - dbgs() << " [non-canonical]"; - dbgs() << "\n"; - }); + auto &Sym = createStandardGraphSymbol(NSym, B, SymEnd - NSym.Value, + SectionIsText, SymLive, + LastCanonicalAddr != NSym.Value); - auto &Sym = - NSym.Name - ? G->addDefinedSymbol(B, NSym.Value - BlockStart, *NSym.Name, - SymEnd - NSym.Value, NSym.L, NSym.S, - SectionIsText, SymLive) - : G->addAnonymousSymbol(B, NSym.Value - BlockStart, - SymEnd - NSym.Value, SectionIsText, - SymLive); - NSym.GraphSymbol = &Sym; if (LastCanonicalAddr != Sym.getAddress()) { if (LastCanonicalAddr) SymEnd = *LastCanonicalAddr; LastCanonicalAddr = Sym.getAddress(); - setCanonicalSymbol(Sym); } } } @@ -559,6 +553,41 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { return Error::success(); } +Symbol &MachOLinkGraphBuilder::createStandardGraphSymbol(NormalizedSymbol &NSym, + Block &B, size_t Size, + bool IsText, + bool IsNoDeadStrip, + bool IsCanonical) { + + LLVM_DEBUG({ + dbgs() << " " << formatv("{0:x16}", NSym.Value) << " -- " + << formatv("{0:x16}", NSym.Value + Size) << ": "; + if (!NSym.Name) + dbgs() << "<anonymous symbol>"; + else + dbgs() << NSym.Name; + if (IsText) + dbgs() << " [text]"; + if (IsNoDeadStrip) + dbgs() << " [no-dead-strip]"; + if (!IsCanonical) + dbgs() << " [non-canonical]"; + dbgs() << "\n"; + }); + + auto &Sym = NSym.Name ? G->addDefinedSymbol(B, NSym.Value - B.getAddress(), + *NSym.Name, Size, NSym.L, NSym.S, + IsText, IsNoDeadStrip) + : G->addAnonymousSymbol(B, NSym.Value - B.getAddress(), + Size, IsText, IsNoDeadStrip); + NSym.GraphSymbol = &Sym; + + if (IsCanonical) + setCanonicalSymbol(Sym); + + return Sym; +} + Error MachOLinkGraphBuilder::graphifySectionsWithCustomParsers() { // Graphify special sections. for (auto &KV : IndexToSection) { @@ -579,5 +608,97 @@ Error MachOLinkGraphBuilder::graphifySectionsWithCustomParsers() { return Error::success(); } +Error MachOLinkGraphBuilder::graphifyCStringSection( + NormalizedSection &NSec, std::vector<NormalizedSymbol *> NSyms) { + + assert(NSec.GraphSection && "C string literal section missing graph section"); + assert(NSec.Data && "C string literal section has no data"); + + LLVM_DEBUG({ + dbgs() << " Graphifying C-string literal section " + << NSec.GraphSection->getName() << "\n"; + }); + + if (NSec.Data[NSec.Size - 1] != '\0') + return make_error<JITLinkError>("C string literal section " + + NSec.GraphSection->getName() + + " does not end with null terminator"); + + /// Sort into reverse order to use as a stack. + llvm::sort(NSyms, + [](const NormalizedSymbol *LHS, const NormalizedSymbol *RHS) { + if (LHS->Value != RHS->Value) + return LHS->Value > RHS->Value; + if (LHS->L != RHS->L) + return LHS->L > RHS->L; + if (LHS->S != RHS->S) + return LHS->S > RHS->S; + if (RHS->Name) { + if (!LHS->Name) + return true; + return *LHS->Name > *RHS->Name; + } + return false; + }); + + bool SectionIsNoDeadStrip = NSec.Flags & MachO::S_ATTR_NO_DEAD_STRIP; + bool SectionIsText = NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS; + JITTargetAddress BlockStart = 0; + + // Scan section for null characters. + for (size_t I = 0; I != NSec.Size; ++I) + if (NSec.Data[I] == '\0') { + JITTargetAddress BlockEnd = I + 1; + size_t BlockSize = BlockEnd - BlockStart; + // Create a block for this null terminated string. + auto &B = G->createContentBlock(*NSec.GraphSection, + {NSec.Data + BlockStart, BlockSize}, + NSec.Address + BlockStart, 1, 0); + + LLVM_DEBUG({ + dbgs() << " Created block " << formatv("{0:x}", B.getAddress()) + << " -- " << formatv("{0:x}", B.getAddress() + B.getSize()) + << " for \"" << StringRef(B.getContent().data()) << "\"\n"; + }); + + // If there's no symbol at the start of this block then create one. + if (NSyms.empty() || NSyms.back()->Value != B.getAddress()) { + auto &S = G->addAnonymousSymbol(B, 0, BlockSize, false, false); + setCanonicalSymbol(S); + LLVM_DEBUG({ + dbgs() << " Adding anonymous symbol for c-string block " + << formatv("{0:x16} -- {1:x16}", S.getAddress(), + S.getAddress() + BlockSize) + << "\n"; + }); + } + + // Process any remaining symbols that point into this block. + JITTargetAddress LastCanonicalAddr = B.getAddress() + BlockEnd; + while (!NSyms.empty() && + NSyms.back()->Value < (B.getAddress() + BlockSize)) { + auto &NSym = *NSyms.back(); + size_t SymSize = (B.getAddress() + BlockSize) - NSyms.back()->Value; + bool SymLive = + (NSym.Desc & MachO::N_NO_DEAD_STRIP) || SectionIsNoDeadStrip; + + bool IsCanonical = false; + if (LastCanonicalAddr != NSym.Value) { + IsCanonical = true; + LastCanonicalAddr = NSym.Value; + } + + createStandardGraphSymbol(NSym, B, SymSize, SectionIsText, SymLive, + IsCanonical); + + NSyms.pop_back(); + } + + BlockStart += BlockSize; + } + + return Error::success(); +} + } // end namespace jitlink } // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index 26e6859de91d..90b14c44ff8a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -81,7 +81,8 @@ protected: using SectionParserFunction = std::function<Error(NormalizedSection &S)>; - MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, Triple TT); + MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, Triple TT, + LinkGraph::GetEdgeKindNameFunction GetEdgeKindName); LinkGraph &getGraph() const { return *G; } @@ -158,6 +159,7 @@ protected: static bool isAltEntry(const NormalizedSymbol &NSym); static bool isDebugSection(const NormalizedSection &NSec); + static bool isZeroFillSection(const NormalizedSection &NSec); MachO::relocation_info getRelocationInfo(const object::relocation_iterator RelItr) { @@ -199,9 +201,21 @@ private: /// all defined symbols in sections without custom parsers. Error graphifyRegularSymbols(); + /// Create and return a graph symbol for the given normalized symbol. + /// + /// NSym's GraphSymbol member will be updated to point at the newly created + /// symbol. + Symbol &createStandardGraphSymbol(NormalizedSymbol &Sym, Block &B, + size_t Size, bool IsText, + bool IsNoDeadStrip, bool IsCanonical); + /// Create graph blocks and symbols for all sections. Error graphifySectionsWithCustomParsers(); + /// Graphify cstring section. + Error graphifyCStringSection(NormalizedSection &NSec, + std::vector<NormalizedSymbol *> NSyms); + // Put the BumpPtrAllocator first so that we don't free any of the underlying // memory until the Symbol/Addressable destructors have been run. BumpPtrAllocator Allocator; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 8366e9658539..169e20a1d1d3 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -12,8 +12,8 @@ #include "llvm/ExecutionEngine/JITLink/MachO_arm64.h" -#include "BasicGOTAndStubsBuilder.h" #include "MachOLinkGraphBuilder.h" +#include "PerGraphGOTAndPLTStubsBuilder.h" #define DEBUG_TYPE "jitlink" @@ -26,7 +26,8 @@ namespace { class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) - : MachOLinkGraphBuilder(Obj, Triple("arm64-apple-darwin")), + : MachOLinkGraphBuilder(Obj, Triple("arm64-apple-darwin"), + getMachOARM64RelocationKindName), NumSymbols(Obj.getSymtabLoadCommand().nsyms) {} private: @@ -271,7 +272,7 @@ private: if (*Kind != Branch26 && *Kind != Page21 && *Kind != PageOffset12) return make_error<JITLinkError>( "Invalid relocation pair: Addend + " + - getMachOARM64RelocationKindName(*Kind)); + StringRef(getMachOARM64RelocationKindName(*Kind))); LLVM_DEBUG({ dbgs() << " Addend: value = " << formatv("{0:x6}", Addend) @@ -404,13 +405,14 @@ private: unsigned NumSymbols = 0; }; -class MachO_arm64_GOTAndStubsBuilder - : public BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder> { +class PerGraphGOTAndPLTStubsBuilder_MachO_arm64 + : public PerGraphGOTAndPLTStubsBuilder< + PerGraphGOTAndPLTStubsBuilder_MachO_arm64> { public: - MachO_arm64_GOTAndStubsBuilder(LinkGraph &G) - : BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder>(G) {} + using PerGraphGOTAndPLTStubsBuilder< + PerGraphGOTAndPLTStubsBuilder_MachO_arm64>::PerGraphGOTAndPLTStubsBuilder; - bool isGOTEdge(Edge &E) const { + bool isGOTEdgeToFix(Edge &E) const { return E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 || E.getKind() == PointerToGOT; } @@ -437,16 +439,16 @@ public: return E.getKind() == Branch26 && !E.getTarget().isDefined(); } - Symbol &createStub(Symbol &Target) { + Symbol &createPLTStub(Symbol &Target) { auto &StubContentBlock = G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + auto &GOTEntrySymbol = getGOTEntry(Target); StubContentBlock.addEdge(LDRLiteral19, 0, GOTEntrySymbol, 0); return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false); } - void fixExternalBranchEdge(Edge &E, Symbol &Stub) { + void fixPLTEdge(Edge &E, Symbol &Stub) { assert(E.getKind() == Branch26 && "Not a Branch32 edge?"); assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); E.setTarget(Stub); @@ -468,14 +470,13 @@ private: return *StubsSection; } - StringRef getGOTEntryBlockContent() { - return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), - sizeof(NullGOTEntryContent)); + ArrayRef<char> getGOTEntryBlockContent() { + return {reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)}; } - StringRef getStubBlockContent() { - return StringRef(reinterpret_cast<const char *>(StubContent), - sizeof(StubContent)); + ArrayRef<char> getStubBlockContent() { + return {reinterpret_cast<const char *>(StubContent), sizeof(StubContent)}; } static const uint8_t NullGOTEntryContent[8]; @@ -484,9 +485,10 @@ private: Section *StubsSection = nullptr; }; -const uint8_t MachO_arm64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t MachO_arm64_GOTAndStubsBuilder::StubContent[8] = { +const uint8_t + PerGraphGOTAndPLTStubsBuilder_MachO_arm64::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t PerGraphGOTAndPLTStubsBuilder_MachO_arm64::StubContent[8] = { 0x10, 0x00, 0x00, 0x58, // LDR x16, <literal> 0x00, 0x02, 0x1f, 0xd6 // BR x16 }; @@ -506,20 +508,6 @@ public: : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: - StringRef getEdgeKindName(Edge::Kind R) const override { - return getMachOARM64RelocationKindName(R); - } - - static Error targetOutOfRangeError(const Block &B, const Edge &E) { - std::string ErrMsg; - { - raw_string_ostream ErrStream(ErrMsg); - ErrStream << "Relocation target out of range: "; - printEdge(ErrStream, B, E, getMachOARM64RelocationKindName(E.getKind())); - ErrStream << "\n"; - } - return make_error<JITLinkError>(std::move(ErrMsg)); - } static unsigned getPageOffset12Shift(uint32_t Instr) { constexpr uint32_t LoadStoreImm12Mask = 0x3b000000; @@ -537,9 +525,10 @@ private: return 0; } - Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { using namespace support; + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); char *FixupPtr = BlockWorkingMem + E.getOffset(); JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); @@ -554,7 +543,7 @@ private: "aligned"); if (Value < -(1 << 27) || Value > ((1 << 27) - 1)) - return targetOutOfRangeError(B, E); + return makeTargetOutOfRangeError(G, B, E); uint32_t RawInstr = *(little32_t *)FixupPtr; assert((RawInstr & 0x7fffffff) == 0x14000000 && @@ -567,7 +556,7 @@ private: case Pointer32: { uint64_t Value = E.getTarget().getAddress() + E.getAddend(); if (Value > std::numeric_limits<uint32_t>::max()) - return targetOutOfRangeError(B, E); + return makeTargetOutOfRangeError(G, B, E); *(ulittle32_t *)FixupPtr = Value; break; } @@ -588,7 +577,7 @@ private: int64_t PageDelta = TargetPage - PCPage; if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1)) - return targetOutOfRangeError(B, E); + return makeTargetOutOfRangeError(G, B, E); uint32_t RawInstr = *(ulittle32_t *)FixupPtr; assert((RawInstr & 0xffffffe0) == 0x90000000 && @@ -638,7 +627,7 @@ private: return make_error<JITLinkError>("LDR literal target is not 32-bit " "aligned"); if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1)) - return targetOutOfRangeError(B, E); + return makeTargetOutOfRangeError(G, B, E); uint32_t EncodedImm = (static_cast<uint32_t>(Delta) >> 2) << 5; uint32_t FixedInstr = RawInstr | EncodedImm; @@ -658,7 +647,7 @@ private: if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { if (Value < std::numeric_limits<int32_t>::min() || Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(B, E); + return makeTargetOutOfRangeError(G, B, E); *(little32_t *)FixupPtr = Value; } else *(little64_t *)FixupPtr = Value; @@ -695,20 +684,18 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G, Config.PrePrunePasses.push_back(markAllSymbolsLive); // Add an in-place GOT/Stubs pass. - Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { - MachO_arm64_GOTAndStubsBuilder(G).run(); - return Error::success(); - }); + Config.PostPrunePasses.push_back( + PerGraphGOTAndPLTStubsBuilder_MachO_arm64::asPass); } - if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config)) + if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); // Construct a JITLinker and run the link function. MachOJITLinker_arm64::link(std::move(Ctx), std::move(G), std::move(Config)); } -StringRef getMachOARM64RelocationKindName(Edge::Kind R) { +const char *getMachOARM64RelocationKindName(Edge::Kind R) { switch (R) { case Branch26: return "Branch26"; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index bde4a19e71ba..61d5c5e21ff1 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -11,77 +11,95 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" -#include "BasicGOTAndStubsBuilder.h" #include "MachOLinkGraphBuilder.h" +#include "PerGraphGOTAndPLTStubsBuilder.h" #define DEBUG_TYPE "jitlink" using namespace llvm; using namespace llvm::jitlink; -using namespace llvm::jitlink::MachO_x86_64_Edges; namespace { class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) - : MachOLinkGraphBuilder(Obj, Triple("x86_64-apple-darwin")) {} + : MachOLinkGraphBuilder(Obj, Triple("x86_64-apple-darwin"), + x86_64::getEdgeKindName) {} private: - static Expected<MachOX86RelocationKind> - getRelocationKind(const MachO::relocation_info &RI) { + enum MachONormalizedRelocationType : unsigned { + MachOBranch32, + MachOPointer32, + MachOPointer64, + MachOPointer64Anon, + MachOPCRel32, + MachOPCRel32Minus1, + MachOPCRel32Minus2, + MachOPCRel32Minus4, + MachOPCRel32Anon, + MachOPCRel32Minus1Anon, + MachOPCRel32Minus2Anon, + MachOPCRel32Minus4Anon, + MachOPCRel32GOTLoad, + MachOPCRel32GOT, + MachOPCRel32TLV, + MachOSubtractor32, + MachOSubtractor64, + }; + + static Expected<MachONormalizedRelocationType> + getRelocKind(const MachO::relocation_info &RI) { switch (RI.r_type) { case MachO::X86_64_RELOC_UNSIGNED: if (!RI.r_pcrel) { if (RI.r_length == 3) - return RI.r_extern ? Pointer64 : Pointer64Anon; + return RI.r_extern ? MachOPointer64 : MachOPointer64Anon; else if (RI.r_extern && RI.r_length == 2) - return Pointer32; + return MachOPointer32; } break; case MachO::X86_64_RELOC_SIGNED: if (RI.r_pcrel && RI.r_length == 2) - return RI.r_extern ? PCRel32 : PCRel32Anon; + return RI.r_extern ? MachOPCRel32 : MachOPCRel32Anon; break; case MachO::X86_64_RELOC_BRANCH: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return Branch32; + return MachOBranch32; break; case MachO::X86_64_RELOC_GOT_LOAD: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PCRel32GOTLoad; + return MachOPCRel32GOTLoad; break; case MachO::X86_64_RELOC_GOT: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PCRel32GOT; + return MachOPCRel32GOT; break; case MachO::X86_64_RELOC_SUBTRACTOR: - // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3. - // Initially represent SUBTRACTOR relocations with 'Delta<W>'. They may - // be turned into NegDelta<W> by parsePairRelocation. if (!RI.r_pcrel && RI.r_extern) { if (RI.r_length == 2) - return Delta32; + return MachOSubtractor32; else if (RI.r_length == 3) - return Delta64; + return MachOSubtractor64; } break; case MachO::X86_64_RELOC_SIGNED_1: if (RI.r_pcrel && RI.r_length == 2) - return RI.r_extern ? PCRel32Minus1 : PCRel32Minus1Anon; + return RI.r_extern ? MachOPCRel32Minus1 : MachOPCRel32Minus1Anon; break; case MachO::X86_64_RELOC_SIGNED_2: if (RI.r_pcrel && RI.r_length == 2) - return RI.r_extern ? PCRel32Minus2 : PCRel32Minus2Anon; + return RI.r_extern ? MachOPCRel32Minus2 : MachOPCRel32Minus2Anon; break; case MachO::X86_64_RELOC_SIGNED_4: if (RI.r_pcrel && RI.r_length == 2) - return RI.r_extern ? PCRel32Minus4 : PCRel32Minus4Anon; + return RI.r_extern ? MachOPCRel32Minus4 : MachOPCRel32Minus4Anon; break; case MachO::X86_64_RELOC_TLV: if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) - return PCRel32TLV; + return MachOPCRel32TLV; break; } @@ -95,20 +113,19 @@ private: ", length=" + formatv("{0:d}", RI.r_length)); } - using PairRelocInfo = std::tuple<MachOX86RelocationKind, Symbol *, uint64_t>; + using PairRelocInfo = std::tuple<Edge::Kind, Symbol *, uint64_t>; // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, // returns the edge kind and addend to be used. - Expected<PairRelocInfo> - parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind, - const MachO::relocation_info &SubRI, - JITTargetAddress FixupAddress, const char *FixupContent, - object::relocation_iterator &UnsignedRelItr, - object::relocation_iterator &RelEnd) { + Expected<PairRelocInfo> parsePairRelocation( + Block &BlockToFix, MachONormalizedRelocationType SubtractorKind, + const MachO::relocation_info &SubRI, JITTargetAddress FixupAddress, + const char *FixupContent, object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { using namespace support; - assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || - (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + assert(((SubtractorKind == MachOSubtractor32 && SubRI.r_length == 2) || + (SubtractorKind == MachOSubtractor64 && SubRI.r_length == 3)) && "Subtractor kind should match length"); assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); @@ -158,17 +175,18 @@ private: FixupValue -= ToSymbol->getAddress(); } - MachOX86RelocationKind DeltaKind; + Edge::Kind DeltaKind; Symbol *TargetSymbol; uint64_t Addend; if (&BlockToFix == &FromSymbol->getAddressable()) { TargetSymbol = ToSymbol; - DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + DeltaKind = (SubRI.r_length == 3) ? x86_64::Delta64 : x86_64::Delta32; Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); // FIXME: handle extern 'from'. } else if (&BlockToFix == &ToSymbol->getAddressable()) { TargetSymbol = FromSymbol; - DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + DeltaKind = + (SubRI.r_length == 3) ? x86_64::NegDelta64 : x86_64::NegDelta32; Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); } else { // BlockToFix was neither FromSymbol nor ToSymbol. @@ -218,11 +236,6 @@ private: MachO::relocation_info RI = getRelocationInfo(RelItr); - // Sanity check the relocation kind. - auto Kind = getRelocationKind(RI); - if (!Kind) - return Kind.takeError(); - // Find the address of the value to fix up. JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; @@ -251,111 +264,152 @@ private: const char *FixupContent = BlockToFix->getContent().data() + (FixupAddress - BlockToFix->getAddress()); + size_t FixupOffset = FixupAddress - BlockToFix->getAddress(); + // The target symbol and addend will be populated by the switch below. Symbol *TargetSymbol = nullptr; uint64_t Addend = 0; - switch (*Kind) { - case Branch32: - case PCRel32: - case PCRel32GOTLoad: - case PCRel32GOT: + // Sanity check the relocation kind. + auto MachORelocKind = getRelocKind(RI); + if (!MachORelocKind) + return MachORelocKind.takeError(); + + Edge::Kind Kind = Edge::Invalid; + + switch (*MachORelocKind) { + case MachOBranch32: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const little32_t *)FixupContent; + Kind = x86_64::BranchPCRel32; + break; + case MachOPCRel32: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const little32_t *)FixupContent - 4; + Kind = x86_64::Delta32; + break; + case MachOPCRel32GOTLoad: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const little32_t *)FixupContent; + Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; + if (FixupOffset < 3) + return make_error<JITLinkError>("GOTLD at invalid offset " + + formatv("{0}", FixupOffset)); break; - case Pointer32: + case MachOPCRel32GOT: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const little32_t *)FixupContent - 4; + Kind = x86_64::RequestGOTAndTransformToDelta32; + break; + case MachOPCRel32TLV: + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; + else + return TargetSymbolOrErr.takeError(); + Addend = *(const little32_t *)FixupContent; + Kind = x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable; + break; + case MachOPointer32: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent; + Kind = x86_64::Pointer32; break; - case Pointer64: + case MachOPointer64: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); Addend = *(const ulittle64_t *)FixupContent; + Kind = x86_64::Pointer64; break; - case Pointer64Anon: { + case MachOPointer64Anon: { JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) TargetSymbol = &*TargetSymbolOrErr; else return TargetSymbolOrErr.takeError(); Addend = TargetAddress - TargetSymbol->getAddress(); + Kind = x86_64::Pointer64; break; } - case PCRel32Minus1: - case PCRel32Minus2: - case PCRel32Minus4: + case MachOPCRel32Minus1: + case MachOPCRel32Minus2: + case MachOPCRel32Minus4: if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else return TargetSymbolOrErr.takeError(); - Addend = *(const little32_t *)FixupContent + - (1 << (*Kind - PCRel32Minus1)); + Addend = *(const little32_t *)FixupContent - 4; + Kind = x86_64::Delta32; break; - case PCRel32Anon: { + case MachOPCRel32Anon: { JITTargetAddress TargetAddress = FixupAddress + 4 + *(const little32_t *)FixupContent; if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) TargetSymbol = &*TargetSymbolOrErr; else return TargetSymbolOrErr.takeError(); - Addend = TargetAddress - TargetSymbol->getAddress(); + Addend = TargetAddress - TargetSymbol->getAddress() - 4; + Kind = x86_64::Delta32; break; } - case PCRel32Minus1Anon: - case PCRel32Minus2Anon: - case PCRel32Minus4Anon: { + case MachOPCRel32Minus1Anon: + case MachOPCRel32Minus2Anon: + case MachOPCRel32Minus4Anon: { JITTargetAddress Delta = - static_cast<JITTargetAddress>(1ULL << (*Kind - PCRel32Minus1Anon)); + 4 + static_cast<JITTargetAddress>( + 1ULL << (*MachORelocKind - MachOPCRel32Minus1Anon)); JITTargetAddress TargetAddress = - FixupAddress + 4 + Delta + *(const little32_t *)FixupContent; + FixupAddress + Delta + *(const little32_t *)FixupContent; if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) TargetSymbol = &*TargetSymbolOrErr; else return TargetSymbolOrErr.takeError(); - Addend = TargetAddress - TargetSymbol->getAddress(); + Addend = TargetAddress - TargetSymbol->getAddress() - Delta; + Kind = x86_64::Delta32; break; } - case Delta32: - case Delta64: { + case MachOSubtractor32: + case MachOSubtractor64: { // We use Delta32/Delta64 to represent SUBTRACTOR relocations. // parsePairRelocation handles the paired reloc, and returns the // edge kind to be used (either Delta32/Delta64, or // NegDelta32/NegDelta64, depending on the direction of the // subtraction) along with the addend. auto PairInfo = - parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, - FixupContent, ++RelItr, RelEnd); + parsePairRelocation(*BlockToFix, *MachORelocKind, RI, + FixupAddress, FixupContent, ++RelItr, RelEnd); if (!PairInfo) return PairInfo.takeError(); - std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + std::tie(Kind, TargetSymbol, Addend) = *PairInfo; assert(TargetSymbol && "No target symbol from parsePairRelocation?"); break; } - case PCRel32TLV: - return make_error<JITLinkError>( - "MachO TLV relocations not yet supported"); - default: - llvm_unreachable("Special relocation kind should not appear in " - "mach-o file"); } LLVM_DEBUG({ dbgs() << " "; - Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, + Edge GE(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); - printEdge(dbgs(), *BlockToFix, GE, - getMachOX86RelocationKindName(*Kind)); + printEdge(dbgs(), *BlockToFix, GE, x86_64::getEdgeKindName(Kind)); dbgs() << "\n"; }); - BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + BlockToFix->addEdge(Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); } } @@ -363,61 +417,59 @@ private: } }; -class MachO_x86_64_GOTAndStubsBuilder - : public BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder> { +class PerGraphGOTAndPLTStubsBuilder_MachO_x86_64 + : public PerGraphGOTAndPLTStubsBuilder< + PerGraphGOTAndPLTStubsBuilder_MachO_x86_64> { public: - static const uint8_t NullGOTEntryContent[8]; - static const uint8_t StubContent[6]; - MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G) - : BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder>(G) {} + using PerGraphGOTAndPLTStubsBuilder< + PerGraphGOTAndPLTStubsBuilder_MachO_x86_64>:: + PerGraphGOTAndPLTStubsBuilder; - bool isGOTEdge(Edge &E) const { - return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; + bool isGOTEdgeToFix(Edge &E) const { + return E.getKind() == x86_64::RequestGOTAndTransformToDelta32 || + E.getKind() == + x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; } Symbol &createGOTEntry(Symbol &Target) { - auto &GOTEntryBlock = G.createContentBlock( - getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); - GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); - return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); + return x86_64::createAnonymousPointer(G, getGOTSection(), &Target); } void fixGOTEdge(Edge &E, Symbol &GOTEntry) { - assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && - "Not a GOT edge?"); - // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is - // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to - // check for GOT optimization opportunities in the - // optimizeMachO_x86_64_GOTAndStubs pass below. - if (E.getKind() == PCRel32GOT) - E.setKind(PCRel32); - + // Fix the edge kind. + switch (E.getKind()) { + case x86_64::RequestGOTAndTransformToDelta32: + E.setKind(x86_64::Delta32); + break; + case x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable: + E.setKind(x86_64::PCRel32GOTLoadRelaxable); + break; + default: + llvm_unreachable("Not a GOT transform edge"); + } + // Fix the target, leave the addend as-is. E.setTarget(GOTEntry); - // Leave the edge addend as-is. } bool isExternalBranchEdge(Edge &E) { - return E.getKind() == Branch32 && !E.getTarget().isDefined(); + return E.getKind() == x86_64::BranchPCRel32 && E.getTarget().isExternal(); } - Symbol &createStub(Symbol &Target) { - auto &StubContentBlock = - G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); - // Re-use GOT entries for stub targets. - auto &GOTEntrySymbol = getGOTEntrySymbol(Target); - StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, 0); - return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); + Symbol &createPLTStub(Symbol &Target) { + return x86_64::createAnonymousPointerJumpStub(G, getStubsSection(), + getGOTEntry(Target)); } - void fixExternalBranchEdge(Edge &E, Symbol &Stub) { - assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); - assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + void fixPLTEdge(Edge &E, Symbol &Stub) { + assert(E.getKind() == x86_64::BranchPCRel32 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && + "BranchPCRel32 edge has unexpected addend value"); - // Set the edge kind to Branch32ToStub. We will use this to check for stub - // optimization opportunities in the optimizeMachO_x86_64_GOTAndStubs pass - // below. - E.setKind(Branch32ToStub); + // Set the edge kind to BranchPCRel32ToPtrJumpStubRelaxable. We will use + // this to check for stub optimization opportunities in the + // optimizeMachO_x86_64_GOTAndStubs pass below. + E.setKind(x86_64::BranchPCRel32ToPtrJumpStubRelaxable); E.setTarget(Stub); } @@ -437,24 +489,10 @@ private: return *StubsSection; } - StringRef getGOTEntryBlockContent() { - return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), - sizeof(NullGOTEntryContent)); - } - - StringRef getStubBlockContent() { - return StringRef(reinterpret_cast<const char *>(StubContent), - sizeof(StubContent)); - } - Section *GOTSection = nullptr; Section *StubsSection = nullptr; }; -const uint8_t MachO_x86_64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; -const uint8_t MachO_x86_64_GOTAndStubsBuilder::StubContent[6] = { - 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; } // namespace static Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) { @@ -462,13 +500,9 @@ static Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) { for (auto *B : G.blocks()) for (auto &E : B->edges()) - if (E.getKind() == PCRel32GOTLoad) { + if (E.getKind() == x86_64::PCRel32GOTLoadRelaxable) { assert(E.getOffset() >= 3 && "GOT edge occurs too early in block"); - // Switch the edge kind to PCRel32: Whether we change the edge target - // or not this will be the desired kind. - E.setKind(PCRel32); - // Optimize GOT references. auto &GOTBlock = E.getTarget().getBlock(); assert(GOTBlock.getSize() == G.getPointerSize() && @@ -491,25 +525,19 @@ static Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) { if (Displacement >= std::numeric_limits<int32_t>::min() && Displacement <= std::numeric_limits<int32_t>::max()) { E.setTarget(GOTTarget); - auto *BlockData = reinterpret_cast<uint8_t *>( - const_cast<char *>(B->getContent().data())); - BlockData[E.getOffset() - 2] = 0x8d; + E.setKind(x86_64::Delta32); + E.setAddend(E.getAddend() - 4); + char *BlockData = B->getMutableContent(G).data(); + BlockData[E.getOffset() - 2] = (char)0x8d; LLVM_DEBUG({ dbgs() << " Replaced GOT load wih LEA:\n "; - printEdge(dbgs(), *B, E, - getMachOX86RelocationKindName(E.getKind())); + printEdge(dbgs(), *B, E, x86_64::getEdgeKindName(E.getKind())); dbgs() << "\n"; }); } - } else if (E.getKind() == Branch32ToStub) { - - // Switch the edge kind to PCRel32: Whether we change the edge target - // or not this will be the desired kind. - E.setKind(Branch32); - + } else if (E.getKind() == x86_64::BranchPCRel32ToPtrJumpStubRelaxable) { auto &StubBlock = E.getTarget().getBlock(); - assert(StubBlock.getSize() == - sizeof(MachO_x86_64_GOTAndStubsBuilder::StubContent) && + assert(StubBlock.getSize() == sizeof(x86_64::PointerJumpStubContent) && "Stub block should be stub sized"); assert(StubBlock.edges_size() == 1 && "Stub block should only have one outgoing edge"); @@ -527,11 +555,11 @@ static Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) { int64_t Displacement = TargetAddr - EdgeAddr + 4; if (Displacement >= std::numeric_limits<int32_t>::min() && Displacement <= std::numeric_limits<int32_t>::max()) { + E.setKind(x86_64::BranchPCRel32); E.setTarget(GOTTarget); LLVM_DEBUG({ dbgs() << " Replaced stub branch with direct branch:\n "; - printEdge(dbgs(), *B, E, - getMachOX86RelocationKindName(E.getKind())); + printEdge(dbgs(), *B, E, x86_64::getEdgeKindName(E.getKind())); dbgs() << "\n"; }); } @@ -553,104 +581,9 @@ public: : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} private: - StringRef getEdgeKindName(Edge::Kind R) const override { - return getMachOX86RelocationKindName(R); - } - - static Error targetOutOfRangeError(const Block &B, const Edge &E) { - std::string ErrMsg; - { - raw_string_ostream ErrStream(ErrMsg); - ErrStream << "Relocation target out of range: "; - printEdge(ErrStream, B, E, getMachOX86RelocationKindName(E.getKind())); - ErrStream << "\n"; - } - return make_error<JITLinkError>(std::move(ErrMsg)); - } - - Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { - - using namespace support; - - char *FixupPtr = BlockWorkingMem + E.getOffset(); - JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); - - switch (E.getKind()) { - case Branch32: - case PCRel32: - case PCRel32Anon: { - int64_t Value = - E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); - if (Value < std::numeric_limits<int32_t>::min() || - Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(B, E); - *(little32_t *)FixupPtr = Value; - break; - } - case Pointer64: - case Pointer64Anon: { - uint64_t Value = E.getTarget().getAddress() + E.getAddend(); - *(ulittle64_t *)FixupPtr = Value; - break; - } - case PCRel32Minus1: - case PCRel32Minus2: - case PCRel32Minus4: { - int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1)); - int64_t Value = - E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); - if (Value < std::numeric_limits<int32_t>::min() || - Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(B, E); - *(little32_t *)FixupPtr = Value; - break; - } - case PCRel32Minus1Anon: - case PCRel32Minus2Anon: - case PCRel32Minus4Anon: { - int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1Anon)); - int64_t Value = - E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); - if (Value < std::numeric_limits<int32_t>::min() || - Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(B, E); - *(little32_t *)FixupPtr = Value; - break; - } - case Delta32: - case Delta64: - case NegDelta32: - case NegDelta64: { - int64_t Value; - if (E.getKind() == Delta32 || E.getKind() == Delta64) - Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); - else - Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); - - if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { - if (Value < std::numeric_limits<int32_t>::min() || - Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(B, E); - *(little32_t *)FixupPtr = Value; - } else - *(little64_t *)FixupPtr = Value; - break; - } - case Pointer32: { - uint64_t Value = E.getTarget().getAddress() + E.getAddend(); - if (Value > std::numeric_limits<uint32_t>::max()) - return targetOutOfRangeError(B, E); - *(ulittle32_t *)FixupPtr = Value; - break; - } - default: - llvm_unreachable("Unrecognized edge kind"); - } - - return Error::success(); + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { + return x86_64::applyFixup(G, B, E); } - - uint64_t NullValue = 0; }; Expected<std::unique_ptr<LinkGraph>> @@ -668,9 +601,8 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G, if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { // Add eh-frame passses. - Config.PrePrunePasses.push_back(EHFrameSplitter("__eh_frame")); - Config.PrePrunePasses.push_back(EHFrameEdgeFixer( - "__eh_frame", G->getPointerSize(), Delta64, Delta32, NegDelta32)); + Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_x86_64()); + Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_x86_64()); // Add a mark-live pass. if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) @@ -679,67 +611,27 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G, Config.PrePrunePasses.push_back(markAllSymbolsLive); // Add an in-place GOT/Stubs pass. - Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { - MachO_x86_64_GOTAndStubsBuilder(G).run(); - return Error::success(); - }); + Config.PostPrunePasses.push_back( + PerGraphGOTAndPLTStubsBuilder_MachO_x86_64::asPass); // Add GOT/Stubs optimizer pass. Config.PreFixupPasses.push_back(optimizeMachO_x86_64_GOTAndStubs); } - if (auto Err = Ctx->modifyPassConfig(G->getTargetTriple(), Config)) + if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); // Construct a JITLinker and run the link function. MachOJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); } -StringRef getMachOX86RelocationKindName(Edge::Kind R) { - switch (R) { - case Branch32: - return "Branch32"; - case Branch32ToStub: - return "Branch32ToStub"; - case Pointer32: - return "Pointer32"; - case Pointer64: - return "Pointer64"; - case Pointer64Anon: - return "Pointer64Anon"; - case PCRel32: - return "PCRel32"; - case PCRel32Minus1: - return "PCRel32Minus1"; - case PCRel32Minus2: - return "PCRel32Minus2"; - case PCRel32Minus4: - return "PCRel32Minus4"; - case PCRel32Anon: - return "PCRel32Anon"; - case PCRel32Minus1Anon: - return "PCRel32Minus1Anon"; - case PCRel32Minus2Anon: - return "PCRel32Minus2Anon"; - case PCRel32Minus4Anon: - return "PCRel32Minus4Anon"; - case PCRel32GOTLoad: - return "PCRel32GOTLoad"; - case PCRel32GOT: - return "PCRel32GOT"; - case PCRel32TLV: - return "PCRel32TLV"; - case Delta32: - return "Delta32"; - case Delta64: - return "Delta64"; - case NegDelta32: - return "NegDelta32"; - case NegDelta64: - return "NegDelta64"; - default: - return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); - } +LinkGraphPassFunction createEHFrameSplitterPass_MachO_x86_64() { + return EHFrameSplitter("__TEXT,__eh_frame"); +} + +LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_x86_64() { + return EHFrameEdgeFixer("__TEXT,__eh_frame", x86_64::PointerSize, + x86_64::Delta64, x86_64::Delta32, x86_64::NegDelta32); } } // end namespace jitlink diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h new file mode 100644 index 000000000000..6e9df9c75a65 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/PerGraphGOTAndPLTStubsBuilder.h @@ -0,0 +1,126 @@ +//===--------------- PerGraphGOTAndPLTStubBuilder.h -------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Construct GOT and PLT entries for each graph. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHGOTANDPLTSTUBSBUILDER_H +#define LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHGOTANDPLTSTUBSBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +/// Per-object GOT and PLT Stub builder. +/// +/// Constructs GOT entries and PLT stubs in every graph for referenced symbols. +/// Building these blocks in every graph is likely to lead to duplicate entries +/// in the JITLinkDylib, but allows graphs to be trivially removed independently +/// without affecting other graphs (since those other graphs will have their own +/// copies of any required entries). +template <typename BuilderImplT> +class PerGraphGOTAndPLTStubsBuilder { +public: + PerGraphGOTAndPLTStubsBuilder(LinkGraph &G) : G(G) {} + + static Error asPass(LinkGraph &G) { return BuilderImplT(G).run(); } + + Error run() { + LLVM_DEBUG(dbgs() << "Running Per-Graph GOT and Stubs builder:\n"); + + // We're going to be adding new blocks, but we don't want to iterate over + // the new ones, so build a worklist. + std::vector<Block *> Worklist(G.blocks().begin(), G.blocks().end()); + + for (auto *B : Worklist) + for (auto &E : B->edges()) { + if (impl().isGOTEdgeToFix(E)) { + LLVM_DEBUG({ + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) + << " edge at " << formatv("{0:x}", B->getFixupAddress(E)) + << " (" << formatv("{0:x}", B->getAddress()) << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + impl().fixGOTEdge(E, getGOTEntry(E.getTarget())); + } else if (impl().isExternalBranchEdge(E)) { + LLVM_DEBUG({ + dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) + << " edge at " << formatv("{0:x}", B->getFixupAddress(E)) + << " (" << formatv("{0:x}", B->getAddress()) << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"; + }); + impl().fixPLTEdge(E, getPLTStub(E.getTarget())); + } + } + + return Error::success(); + } + +protected: + Symbol &getGOTEntry(Symbol &Target) { + assert(Target.hasName() && "GOT edge cannot point to anonymous target"); + + auto GOTEntryI = GOTEntries.find(Target.getName()); + + // Build the entry if it doesn't exist. + if (GOTEntryI == GOTEntries.end()) { + auto &GOTEntry = impl().createGOTEntry(Target); + LLVM_DEBUG({ + dbgs() << " Created GOT entry for " << Target.getName() << ": " + << GOTEntry << "\n"; + }); + GOTEntryI = + GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; + } + + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol"); + LLVM_DEBUG( + { dbgs() << " Using GOT entry " << *GOTEntryI->second << "\n"; }); + return *GOTEntryI->second; + } + + Symbol &getPLTStub(Symbol &Target) { + assert(Target.hasName() && + "External branch edge can not point to an anonymous target"); + auto StubI = PLTStubs.find(Target.getName()); + + if (StubI == PLTStubs.end()) { + auto &StubSymbol = impl().createPLTStub(Target); + LLVM_DEBUG({ + dbgs() << " Created PLT stub for " << Target.getName() << ": " + << StubSymbol << "\n"; + }); + StubI = + PLTStubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first; + } + + assert(StubI != PLTStubs.end() && "Count not get stub symbol"); + LLVM_DEBUG({ dbgs() << " Using PLT stub " << *StubI->second << "\n"; }); + return *StubI->second; + } + + LinkGraph &G; + +private: + BuilderImplT &impl() { return static_cast<BuilderImplT &>(*this); } + + DenseMap<StringRef, Symbol *> GOTEntries; + DenseMap<StringRef, Symbol *> PLTStubs; +}; + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE + +#endif // LLVM_EXECUTIONENGINE_JITLINK_PERGRAPHGOTANDPLTSTUBSBUILDER_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/riscv.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/riscv.cpp new file mode 100644 index 000000000000..6b73ff95a3b0 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/riscv.cpp @@ -0,0 +1,44 @@ +//===------ riscv.cpp - Generic JITLink riscv edge kinds, utilities -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing riscv objects. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/riscv.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { +namespace riscv { + +const char *getEdgeKindName(Edge::Kind K) { + switch (K) { + case R_RISCV_32: + return "R_RISCV_32"; + case R_RISCV_64: + return "R_RISCV_64"; + case R_RISCV_HI20: + return "R_RISCV_HI20"; + case R_RISCV_LO12_I: + return "R_RISCV_LO12_I"; + case R_RISCV_PCREL_HI20: + return "R_RISCV_PCREL_HI20"; + case R_RISCV_PCREL_LO12_I: + return "R_RISCV_PCREL_LO12_I"; + case R_RISCV_PCREL_LO12_S: + return "R_RISCV_PCREL_LO12_S"; + case R_RISCV_CALL: + return "R_RISCV_CALL"; + } + return getGenericEdgeKindName(K); +} +} // namespace riscv +} // namespace jitlink +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp new file mode 100644 index 000000000000..c951ed6d95be --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp @@ -0,0 +1,64 @@ +//===----- x86_64.cpp - Generic JITLink x86-64 edge kinds, utilities ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing x86-64 objects. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/x86_64.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { +namespace x86_64 { + +const char *getEdgeKindName(Edge::Kind K) { + switch (K) { + case Pointer64: + return "Pointer64"; + case Pointer32: + return "Pointer32"; + case Delta64: + return "Delta64"; + case Delta32: + return "Delta32"; + case NegDelta64: + return "NegDelta64"; + case NegDelta32: + return "NegDelta32"; + case BranchPCRel32: + return "BranchPCRel32"; + case BranchPCRel32ToPtrJumpStub: + return "BranchPCRel32ToPtrJumpStub"; + case BranchPCRel32ToPtrJumpStubRelaxable: + return "BranchPCRel32ToPtrJumpStubRelaxable"; + case RequestGOTAndTransformToDelta32: + return "RequestGOTAndTransformToDelta32"; + case PCRel32GOTLoadRelaxable: + return "PCRel32GOTLoadRelaxable"; + case RequestGOTAndTransformToPCRel32GOTLoadRelaxable: + return "RequestGOTAndTransformToPCRel32GOTLoadRelaxable"; + case PCRel32TLVPLoadRelaxable: + return "PCRel32TLVPLoadRelaxable"; + case RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable: + return "RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable"; + default: + return getGenericEdgeKindName(static_cast<Edge::Kind>(K)); + } +} + +const char NullPointerContent[PointerSize] = {0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}; + +const char PointerJumpStubContent[6] = { + static_cast<char>(0xFFu), 0x25, 0x00, 0x00, 0x00, 0x00}; + +} // end namespace x86_64 +} // end namespace jitlink +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index 68878f6729e9..5b73c0e2fbc8 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -7,12 +7,12 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h" - #include "llvm/ADT/Hashing.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/IR/Mangler.h" #include "llvm/IR/Module.h" #include "llvm/Support/FormatVariadic.h" +#include <string> using namespace llvm; using namespace llvm::orc; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp index dfb558808c32..12a501f7f98c 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -29,6 +29,8 @@ char SymbolsNotFound::ID = 0; char SymbolsCouldNotBeRemoved::ID = 0; char MissingSymbolDefinitions::ID = 0; char UnexpectedSymbolDefinitions::ID = 0; +char Task::ID = 0; +char MaterializationTask::ID = 0; RegisterDependenciesFunction NoDependenciesToRegister = RegisterDependenciesFunction(); @@ -168,13 +170,30 @@ void AsynchronousSymbolQuery::notifySymbolMetRequiredState( --OutstandingSymbolsCount; } -void AsynchronousSymbolQuery::handleComplete() { +void AsynchronousSymbolQuery::handleComplete(ExecutionSession &ES) { assert(OutstandingSymbolsCount == 0 && "Symbols remain, handleComplete called prematurely"); - auto TmpNotifyComplete = std::move(NotifyComplete); + class RunQueryCompleteTask : public Task { + public: + RunQueryCompleteTask(SymbolMap ResolvedSymbols, + SymbolsResolvedCallback NotifyComplete) + : ResolvedSymbols(std::move(ResolvedSymbols)), + NotifyComplete(std::move(NotifyComplete)) {} + void printDescription(raw_ostream &OS) override { + OS << "Execute query complete callback for " << ResolvedSymbols; + } + void run() override { NotifyComplete(std::move(ResolvedSymbols)); } + + private: + SymbolMap ResolvedSymbols; + SymbolsResolvedCallback NotifyComplete; + }; + + auto T = std::make_unique<RunQueryCompleteTask>(std::move(ResolvedSymbols), + std::move(NotifyComplete)); NotifyComplete = SymbolsResolvedCallback(); - TmpNotifyComplete(std::move(ResolvedSymbols)); + ES.dispatchTask(std::move(T)); } void AsynchronousSymbolQuery::handleFailed(Error Err) { @@ -750,7 +769,8 @@ Error JITDylib::replace(MaterializationResponsibility &FromMR, if (MustRunMU) { assert(MustRunMR && "MustRunMU set implies MustRunMR set"); - ES.dispatchMaterialization(std::move(MustRunMU), std::move(MustRunMR)); + ES.dispatchTask(std::make_unique<MaterializationTask>( + std::move(MustRunMU), std::move(MustRunMR))); } else { assert(!MustRunMR && "MustRunMU unset implies MustRunMR unset"); } @@ -800,76 +820,79 @@ JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { void JITDylib::addDependencies(const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { - assert(Symbols.count(Name) && "Name not in symbol table"); - assert(Symbols[Name].getState() < SymbolState::Emitted && - "Can not add dependencies for a symbol that is not materializing"); + ES.runSessionLocked([&]() { + assert(Symbols.count(Name) && "Name not in symbol table"); + assert(Symbols[Name].getState() < SymbolState::Emitted && + "Can not add dependencies for a symbol that is not materializing"); - LLVM_DEBUG({ - dbgs() << "In " << getName() << " adding dependencies for " - << *Name << ": " << Dependencies << "\n"; + LLVM_DEBUG({ + dbgs() << "In " << getName() << " adding dependencies for " << *Name + << ": " << Dependencies << "\n"; }); - // If Name is already in an error state then just bail out. - if (Symbols[Name].getFlags().hasError()) - return; + // If Name is already in an error state then just bail out. + if (Symbols[Name].getFlags().hasError()) + return; - auto &MI = MaterializingInfos[Name]; - assert(Symbols[Name].getState() != SymbolState::Emitted && - "Can not add dependencies to an emitted symbol"); + auto &MI = MaterializingInfos[Name]; + assert(Symbols[Name].getState() != SymbolState::Emitted && + "Can not add dependencies to an emitted symbol"); - bool DependsOnSymbolInErrorState = false; + bool DependsOnSymbolInErrorState = false; - // Register dependencies, record whether any depenendency is in the error - // state. - for (auto &KV : Dependencies) { - assert(KV.first && "Null JITDylib in dependency?"); - auto &OtherJITDylib = *KV.first; - auto &DepsOnOtherJITDylib = MI.UnemittedDependencies[&OtherJITDylib]; + // Register dependencies, record whether any depenendency is in the error + // state. + for (auto &KV : Dependencies) { + assert(KV.first && "Null JITDylib in dependency?"); + auto &OtherJITDylib = *KV.first; + auto &DepsOnOtherJITDylib = MI.UnemittedDependencies[&OtherJITDylib]; - for (auto &OtherSymbol : KV.second) { + for (auto &OtherSymbol : KV.second) { - // Check the sym entry for the dependency. - auto OtherSymI = OtherJITDylib.Symbols.find(OtherSymbol); + // Check the sym entry for the dependency. + auto OtherSymI = OtherJITDylib.Symbols.find(OtherSymbol); - // Assert that this symbol exists and has not reached the ready state - // already. - assert(OtherSymI != OtherJITDylib.Symbols.end() && - "Dependency on unknown symbol"); + // Assert that this symbol exists and has not reached the ready state + // already. + assert(OtherSymI != OtherJITDylib.Symbols.end() && + "Dependency on unknown symbol"); - auto &OtherSymEntry = OtherSymI->second; + auto &OtherSymEntry = OtherSymI->second; - // If the other symbol is already in the Ready state then there's no - // dependency to add. - if (OtherSymEntry.getState() == SymbolState::Ready) - continue; + // If the other symbol is already in the Ready state then there's no + // dependency to add. + if (OtherSymEntry.getState() == SymbolState::Ready) + continue; - // If the dependency is in an error state then note this and continue, - // we will move this symbol to the error state below. - if (OtherSymEntry.getFlags().hasError()) { - DependsOnSymbolInErrorState = true; - continue; - } + // If the dependency is in an error state then note this and continue, + // we will move this symbol to the error state below. + if (OtherSymEntry.getFlags().hasError()) { + DependsOnSymbolInErrorState = true; + continue; + } - // If the dependency was not in the error state then add it to - // our list of dependencies. - auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; + // If the dependency was not in the error state then add it to + // our list of dependencies. + auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; - if (OtherSymEntry.getState() == SymbolState::Emitted) - transferEmittedNodeDependencies(MI, Name, OtherMI); - else if (&OtherJITDylib != this || OtherSymbol != Name) { - OtherMI.Dependants[this].insert(Name); - DepsOnOtherJITDylib.insert(OtherSymbol); + if (OtherSymEntry.getState() == SymbolState::Emitted) + transferEmittedNodeDependencies(MI, Name, OtherMI); + else if (&OtherJITDylib != this || OtherSymbol != Name) { + OtherMI.Dependants[this].insert(Name); + DepsOnOtherJITDylib.insert(OtherSymbol); + } } - } - if (DepsOnOtherJITDylib.empty()) - MI.UnemittedDependencies.erase(&OtherJITDylib); - } + if (DepsOnOtherJITDylib.empty()) + MI.UnemittedDependencies.erase(&OtherJITDylib); + } - // If this symbol dependended on any symbols in the error state then move - // this symbol to the error state too. - if (DependsOnSymbolInErrorState) - Symbols[Name].setFlags(Symbols[Name].getFlags() | JITSymbolFlags::HasError); + // If this symbol dependended on any symbols in the error state then move + // this symbol to the error state too. + if (DependsOnSymbolInErrorState) + Symbols[Name].setFlags(Symbols[Name].getFlags() | + JITSymbolFlags::HasError); + }); } Error JITDylib::resolve(MaterializationResponsibility &MR, @@ -963,7 +986,7 @@ Error JITDylib::resolve(MaterializationResponsibility &MR, // Otherwise notify all the completed queries. for (auto &Q : CompletedQueries) { assert(Q->isComplete() && "Q not completed"); - Q->handleComplete(); + Q->handleComplete(ES); } return Error::success(); @@ -1087,6 +1110,7 @@ Error JITDylib::emit(MaterializationResponsibility &MR, CompletedQueries.insert(Q); Q->removeQueryDependence(DependantJD, DependantName); } + DependantJD.MaterializingInfos.erase(DependantMII); } } } @@ -1102,6 +1126,7 @@ Error JITDylib::emit(MaterializationResponsibility &MR, CompletedQueries.insert(Q); Q->removeQueryDependence(*this, Name); } + MaterializingInfos.erase(MII); } } @@ -1112,7 +1137,7 @@ Error JITDylib::emit(MaterializationResponsibility &MR, // Otherwise notify all the completed queries. for (auto &Q : CompletedQueries) { assert(Q->isComplete() && "Q is not complete"); - Q->handleComplete(); + Q->handleComplete(ES); } return Error::success(); @@ -1373,6 +1398,11 @@ void JITDylib::dump(raw_ostream &OS) { OS << " Unemitted Dependencies:\n"; for (auto &KV2 : KV.second.UnemittedDependencies) OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; + assert((Symbols[KV.first].getState() != SymbolState::Ready || + !KV.second.pendingQueries().empty() || + !KV.second.Dependants.empty() || + !KV.second.UnemittedDependencies.empty()) && + "Stale materializing info entry"); } }); } @@ -1720,8 +1750,63 @@ Expected<DenseMap<JITDylib *, SymbolMap>> Platform::lookupInitSymbols( return std::move(CompoundResult); } -ExecutionSession::ExecutionSession(std::shared_ptr<SymbolStringPool> SSP) - : SSP(SSP ? std::move(SSP) : std::make_shared<SymbolStringPool>()) {} +void Platform::lookupInitSymbolsAsync( + unique_function<void(Error)> OnComplete, ExecutionSession &ES, + const DenseMap<JITDylib *, SymbolLookupSet> &InitSyms) { + + class TriggerOnComplete { + public: + using OnCompleteFn = unique_function<void(Error)>; + TriggerOnComplete(OnCompleteFn OnComplete) + : OnComplete(std::move(OnComplete)) {} + ~TriggerOnComplete() { OnComplete(std::move(LookupResult)); } + void reportResult(Error Err) { + std::lock_guard<std::mutex> Lock(ResultMutex); + LookupResult = joinErrors(std::move(LookupResult), std::move(Err)); + } + + private: + std::mutex ResultMutex; + Error LookupResult{Error::success()}; + OnCompleteFn OnComplete; + }; + + LLVM_DEBUG({ + dbgs() << "Issuing init-symbol lookup:\n"; + for (auto &KV : InitSyms) + dbgs() << " " << KV.first->getName() << ": " << KV.second << "\n"; + }); + + auto TOC = std::make_shared<TriggerOnComplete>(std::move(OnComplete)); + + for (auto &KV : InitSyms) { + auto *JD = KV.first; + auto Names = std::move(KV.second); + ES.lookup( + LookupKind::Static, + JITDylibSearchOrder({{JD, JITDylibLookupFlags::MatchAllSymbols}}), + std::move(Names), SymbolState::Ready, + [TOC](Expected<SymbolMap> Result) { + TOC->reportResult(Result.takeError()); + }, + NoDependenciesToRegister); + } +} + +void Task::anchor() {} + +void MaterializationTask::printDescription(raw_ostream &OS) { + OS << "Materialization task: " << MU->getName() << " in " + << MR->getTargetJITDylib().getName(); +} + +void MaterializationTask::run() { MU->materialize(std::move(MR)); } + +ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC) + : EPC(std::move(EPC)) { + // Associated EPC and this. + this->EPC->ES = this; +} Error ExecutionSession::endSession() { LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n"); @@ -1736,6 +1821,9 @@ Error ExecutionSession::endSession() { Error Err = Error::success(); for (auto &JD : JITDylibsToClose) Err = joinErrors(std::move(Err), JD->clear()); + + Err = joinErrors(std::move(Err), EPC->disconnect()); + return Err; } @@ -1966,6 +2054,58 @@ ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name, return lookup(SearchOrder, intern(Name), RequiredState); } +Error ExecutionSession::registerJITDispatchHandlers( + JITDylib &JD, JITDispatchHandlerAssociationMap WFs) { + + auto TagAddrs = lookup({{&JD, JITDylibLookupFlags::MatchAllSymbols}}, + SymbolLookupSet::fromMapKeys( + WFs, SymbolLookupFlags::WeaklyReferencedSymbol)); + if (!TagAddrs) + return TagAddrs.takeError(); + + // Associate tag addresses with implementations. + std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex); + for (auto &KV : *TagAddrs) { + auto TagAddr = KV.second.getAddress(); + if (JITDispatchHandlers.count(TagAddr)) + return make_error<StringError>("Tag " + formatv("{0:x16}", TagAddr) + + " (for " + *KV.first + + ") already registered", + inconvertibleErrorCode()); + auto I = WFs.find(KV.first); + assert(I != WFs.end() && I->second && + "JITDispatchHandler implementation missing"); + JITDispatchHandlers[KV.second.getAddress()] = + std::make_shared<JITDispatchHandlerFunction>(std::move(I->second)); + LLVM_DEBUG({ + dbgs() << "Associated function tag \"" << *KV.first << "\" (" + << formatv("{0:x}", KV.second.getAddress()) << ") with handler\n"; + }); + } + return Error::success(); +} + +void ExecutionSession::runJITDispatchHandler( + ExecutorProcessControl::SendResultFunction SendResult, + JITTargetAddress HandlerFnTagAddr, ArrayRef<char> ArgBuffer) { + + std::shared_ptr<JITDispatchHandlerFunction> F; + { + std::lock_guard<std::mutex> Lock(JITDispatchHandlersMutex); + auto I = JITDispatchHandlers.find(HandlerFnTagAddr); + if (I != JITDispatchHandlers.end()) + F = I->second; + } + + if (F) + (*F)(std::move(SendResult), ArgBuffer.data(), ArgBuffer.size()); + else + SendResult(shared::WrapperFunctionResult::createOutOfBandError( + ("No function registered for tag " + + formatv("{0:x16}", HandlerFnTagAddr)) + .str())); +} + void ExecutionSession::dump(raw_ostream &OS) { runSessionLocked([this, &OS]() { for (auto &JD : JDs) @@ -1993,7 +2133,8 @@ void ExecutionSession::dispatchOutstandingMUs() { assert(JMU->first && "No MU?"); LLVM_DEBUG(dbgs() << " Dispatching \"" << JMU->first->getName() << "\"\n"); - dispatchMaterialization(std::move(JMU->first), std::move(JMU->second)); + dispatchTask(std::make_unique<MaterializationTask>(std::move(JMU->first), + std::move(JMU->second))); } LLVM_DEBUG(dbgs() << "Done dispatching MaterializationUnits.\n"); } @@ -2518,7 +2659,7 @@ void ExecutionSession::OL_completeLookup( if (QueryComplete) { LLVM_DEBUG(dbgs() << "Completing query\n"); - Q->handleComplete(); + Q->handleComplete(*this); } dispatchOutstandingMUs(); @@ -2766,9 +2907,10 @@ void ExecutionSession::OL_addDependenciesForAll( } #ifndef NDEBUG -void ExecutionSession::dumpDispatchInfo(JITDylib &JD, MaterializationUnit &MU) { +void ExecutionSession::dumpDispatchInfo(Task &T) { runSessionLocked([&]() { - dbgs() << "Dispatching " << MU << " for " << JD.getName() << "\n"; + dbgs() << "Dispatching: "; + T.printDescription(dbgs()); }); } #endif // NDEBUG diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp new file mode 100644 index 000000000000..36efc744bf30 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp @@ -0,0 +1,507 @@ +//===---- DebugObjectManagerPlugin.h - JITLink debug objects ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkDylib.h" +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Errc.h" +#include "llvm/Support/MSVCErrorWorkarounds.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/raw_ostream.h" + +#include <set> + +#define DEBUG_TYPE "orc" + +using namespace llvm::jitlink; +using namespace llvm::object; + +namespace llvm { +namespace orc { + +class DebugObjectSection { +public: + virtual void setTargetMemoryRange(SectionRange Range) = 0; + virtual void dump(raw_ostream &OS, StringRef Name) {} + virtual ~DebugObjectSection() {} +}; + +template <typename ELFT> +class ELFDebugObjectSection : public DebugObjectSection { +public: + // BinaryFormat ELF is not meant as a mutable format. We can only make changes + // that don't invalidate the file structure. + ELFDebugObjectSection(const typename ELFT::Shdr *Header) + : Header(const_cast<typename ELFT::Shdr *>(Header)) {} + + void setTargetMemoryRange(SectionRange Range) override; + void dump(raw_ostream &OS, StringRef Name) override; + + Error validateInBounds(StringRef Buffer, const char *Name) const; + +private: + typename ELFT::Shdr *Header; + + bool isTextOrDataSection() const; +}; + +template <typename ELFT> +void ELFDebugObjectSection<ELFT>::setTargetMemoryRange(SectionRange Range) { + // Only patch load-addresses for executable and data sections. + if (isTextOrDataSection()) { + Header->sh_addr = static_cast<typename ELFT::uint>(Range.getStart()); + } +} + +template <typename ELFT> +bool ELFDebugObjectSection<ELFT>::isTextOrDataSection() const { + switch (Header->sh_type) { + case ELF::SHT_PROGBITS: + case ELF::SHT_X86_64_UNWIND: + return Header->sh_flags & (ELF::SHF_EXECINSTR | ELF::SHF_ALLOC); + } + return false; +} + +template <typename ELFT> +Error ELFDebugObjectSection<ELFT>::validateInBounds(StringRef Buffer, + const char *Name) const { + const uint8_t *Start = Buffer.bytes_begin(); + const uint8_t *End = Buffer.bytes_end(); + const uint8_t *HeaderPtr = reinterpret_cast<uint8_t *>(Header); + if (HeaderPtr < Start || HeaderPtr + sizeof(typename ELFT::Shdr) > End) + return make_error<StringError>( + formatv("{0} section header at {1:x16} not within bounds of the " + "given debug object buffer [{2:x16} - {3:x16}]", + Name, &Header->sh_addr, Start, End), + inconvertibleErrorCode()); + if (Header->sh_offset + Header->sh_size > Buffer.size()) + return make_error<StringError>( + formatv("{0} section data [{1:x16} - {2:x16}] not within bounds of " + "the given debug object buffer [{3:x16} - {4:x16}]", + Name, Start + Header->sh_offset, + Start + Header->sh_offset + Header->sh_size, Start, End), + inconvertibleErrorCode()); + return Error::success(); +} + +template <typename ELFT> +void ELFDebugObjectSection<ELFT>::dump(raw_ostream &OS, StringRef Name) { + if (auto Addr = static_cast<JITTargetAddress>(Header->sh_addr)) { + OS << formatv(" {0:x16} {1}\n", Addr, Name); + } else { + OS << formatv(" {0}\n", Name); + } +} + +static constexpr sys::Memory::ProtectionFlags ReadOnly = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ); + +enum class Requirement { + // Request final target memory load-addresses for all sections. + ReportFinalSectionLoadAddresses, +}; + +/// The plugin creates a debug object from JITLinkContext when JITLink starts +/// processing the corresponding LinkGraph. It provides access to the pass +/// configuration of the LinkGraph and calls the finalization function, once +/// the resulting link artifact was emitted. +/// +class DebugObject { +public: + DebugObject(JITLinkContext &Ctx, ExecutionSession &ES) : Ctx(Ctx), ES(ES) {} + + void set(Requirement Req) { Reqs.insert(Req); } + bool has(Requirement Req) const { return Reqs.count(Req) > 0; } + + using FinalizeContinuation = std::function<void(Expected<sys::MemoryBlock>)>; + void finalizeAsync(FinalizeContinuation OnFinalize); + + virtual ~DebugObject() { + if (Alloc) + if (Error Err = Alloc->deallocate()) + ES.reportError(std::move(Err)); + } + + virtual void reportSectionTargetMemoryRange(StringRef Name, + SectionRange TargetMem) {} + +protected: + using Allocation = JITLinkMemoryManager::Allocation; + + virtual Expected<std::unique_ptr<Allocation>> + finalizeWorkingMemory(JITLinkContext &Ctx) = 0; + +private: + JITLinkContext &Ctx; + ExecutionSession &ES; + std::set<Requirement> Reqs; + std::unique_ptr<Allocation> Alloc{nullptr}; +}; + +// Finalize working memory and take ownership of the resulting allocation. Start +// copying memory over to the target and pass on the result once we're done. +// Ownership of the allocation remains with us for the rest of our lifetime. +void DebugObject::finalizeAsync(FinalizeContinuation OnFinalize) { + assert(Alloc == nullptr && "Cannot finalize more than once"); + + auto AllocOrErr = finalizeWorkingMemory(Ctx); + if (!AllocOrErr) + OnFinalize(AllocOrErr.takeError()); + Alloc = std::move(*AllocOrErr); + + Alloc->finalizeAsync([this, OnFinalize](Error Err) { + if (Err) + OnFinalize(std::move(Err)); + else + OnFinalize(sys::MemoryBlock( + jitTargetAddressToPointer<void *>(Alloc->getTargetMemory(ReadOnly)), + Alloc->getWorkingMemory(ReadOnly).size())); + }); +} + +/// The current implementation of ELFDebugObject replicates the approach used in +/// RuntimeDyld: It patches executable and data section headers in the given +/// object buffer with load-addresses of their corresponding sections in target +/// memory. +/// +class ELFDebugObject : public DebugObject { +public: + static Expected<std::unique_ptr<DebugObject>> + Create(MemoryBufferRef Buffer, JITLinkContext &Ctx, ExecutionSession &ES); + + void reportSectionTargetMemoryRange(StringRef Name, + SectionRange TargetMem) override; + + StringRef getBuffer() const { return Buffer->getMemBufferRef().getBuffer(); } + +protected: + Expected<std::unique_ptr<Allocation>> + finalizeWorkingMemory(JITLinkContext &Ctx) override; + + template <typename ELFT> + Error recordSection(StringRef Name, + std::unique_ptr<ELFDebugObjectSection<ELFT>> Section); + DebugObjectSection *getSection(StringRef Name); + +private: + template <typename ELFT> + static Expected<std::unique_ptr<ELFDebugObject>> + CreateArchType(MemoryBufferRef Buffer, JITLinkContext &Ctx, + ExecutionSession &ES); + + static std::unique_ptr<WritableMemoryBuffer> + CopyBuffer(MemoryBufferRef Buffer, Error &Err); + + ELFDebugObject(std::unique_ptr<WritableMemoryBuffer> Buffer, + JITLinkContext &Ctx, ExecutionSession &ES) + : DebugObject(Ctx, ES), Buffer(std::move(Buffer)) { + set(Requirement::ReportFinalSectionLoadAddresses); + } + + std::unique_ptr<WritableMemoryBuffer> Buffer; + StringMap<std::unique_ptr<DebugObjectSection>> Sections; +}; + +static const std::set<StringRef> DwarfSectionNames = { +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ + ELF_NAME, +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION +}; + +static bool isDwarfSection(StringRef SectionName) { + return DwarfSectionNames.count(SectionName) == 1; +} + +std::unique_ptr<WritableMemoryBuffer> +ELFDebugObject::CopyBuffer(MemoryBufferRef Buffer, Error &Err) { + ErrorAsOutParameter _(&Err); + size_t Size = Buffer.getBufferSize(); + StringRef Name = Buffer.getBufferIdentifier(); + if (auto Copy = WritableMemoryBuffer::getNewUninitMemBuffer(Size, Name)) { + memcpy(Copy->getBufferStart(), Buffer.getBufferStart(), Size); + return Copy; + } + + Err = errorCodeToError(make_error_code(errc::not_enough_memory)); + return nullptr; +} + +template <typename ELFT> +Expected<std::unique_ptr<ELFDebugObject>> +ELFDebugObject::CreateArchType(MemoryBufferRef Buffer, JITLinkContext &Ctx, + ExecutionSession &ES) { + using SectionHeader = typename ELFT::Shdr; + + Error Err = Error::success(); + std::unique_ptr<ELFDebugObject> DebugObj( + new ELFDebugObject(CopyBuffer(Buffer, Err), Ctx, ES)); + if (Err) + return std::move(Err); + + Expected<ELFFile<ELFT>> ObjRef = ELFFile<ELFT>::create(DebugObj->getBuffer()); + if (!ObjRef) + return ObjRef.takeError(); + + // TODO: Add support for other architectures. + uint16_t TargetMachineArch = ObjRef->getHeader().e_machine; + if (TargetMachineArch != ELF::EM_X86_64) + return nullptr; + + Expected<ArrayRef<SectionHeader>> Sections = ObjRef->sections(); + if (!Sections) + return Sections.takeError(); + + bool HasDwarfSection = false; + for (const SectionHeader &Header : *Sections) { + Expected<StringRef> Name = ObjRef->getSectionName(Header); + if (!Name) + return Name.takeError(); + if (Name->empty()) + continue; + HasDwarfSection |= isDwarfSection(*Name); + + auto Wrapped = std::make_unique<ELFDebugObjectSection<ELFT>>(&Header); + if (Error Err = DebugObj->recordSection(*Name, std::move(Wrapped))) + return std::move(Err); + } + + if (!HasDwarfSection) { + LLVM_DEBUG(dbgs() << "Aborting debug registration for LinkGraph \"" + << DebugObj->Buffer->getBufferIdentifier() + << "\": input object contains no debug info\n"); + return nullptr; + } + + return std::move(DebugObj); +} + +Expected<std::unique_ptr<DebugObject>> +ELFDebugObject::Create(MemoryBufferRef Buffer, JITLinkContext &Ctx, + ExecutionSession &ES) { + unsigned char Class, Endian; + std::tie(Class, Endian) = getElfArchType(Buffer.getBuffer()); + + if (Class == ELF::ELFCLASS32) { + if (Endian == ELF::ELFDATA2LSB) + return CreateArchType<ELF32LE>(Buffer, Ctx, ES); + if (Endian == ELF::ELFDATA2MSB) + return CreateArchType<ELF32BE>(Buffer, Ctx, ES); + return nullptr; + } + if (Class == ELF::ELFCLASS64) { + if (Endian == ELF::ELFDATA2LSB) + return CreateArchType<ELF64LE>(Buffer, Ctx, ES); + if (Endian == ELF::ELFDATA2MSB) + return CreateArchType<ELF64BE>(Buffer, Ctx, ES); + return nullptr; + } + return nullptr; +} + +Expected<std::unique_ptr<DebugObject::Allocation>> +ELFDebugObject::finalizeWorkingMemory(JITLinkContext &Ctx) { + LLVM_DEBUG({ + dbgs() << "Section load-addresses in debug object for \"" + << Buffer->getBufferIdentifier() << "\":\n"; + for (const auto &KV : Sections) + KV.second->dump(dbgs(), KV.first()); + }); + + // TODO: This works, but what actual alignment requirements do we have? + unsigned Alignment = sys::Process::getPageSizeEstimate(); + JITLinkMemoryManager &MemMgr = Ctx.getMemoryManager(); + const JITLinkDylib *JD = Ctx.getJITLinkDylib(); + size_t Size = Buffer->getBufferSize(); + + // Allocate working memory for debug object in read-only segment. + JITLinkMemoryManager::SegmentsRequestMap SingleReadOnlySegment; + SingleReadOnlySegment[ReadOnly] = + JITLinkMemoryManager::SegmentRequest(Alignment, Size, 0); + + auto AllocOrErr = MemMgr.allocate(JD, SingleReadOnlySegment); + if (!AllocOrErr) + return AllocOrErr.takeError(); + + // Initialize working memory with a copy of our object buffer. + // TODO: Use our buffer as working memory directly. + std::unique_ptr<Allocation> Alloc = std::move(*AllocOrErr); + MutableArrayRef<char> WorkingMem = Alloc->getWorkingMemory(ReadOnly); + memcpy(WorkingMem.data(), Buffer->getBufferStart(), Size); + Buffer.reset(); + + return std::move(Alloc); +} + +void ELFDebugObject::reportSectionTargetMemoryRange(StringRef Name, + SectionRange TargetMem) { + if (auto *DebugObjSection = getSection(Name)) + DebugObjSection->setTargetMemoryRange(TargetMem); +} + +template <typename ELFT> +Error ELFDebugObject::recordSection( + StringRef Name, std::unique_ptr<ELFDebugObjectSection<ELFT>> Section) { + if (Error Err = Section->validateInBounds(this->getBuffer(), Name.data())) + return Err; + auto ItInserted = Sections.try_emplace(Name, std::move(Section)); + if (!ItInserted.second) + return make_error<StringError>("Duplicate section", + inconvertibleErrorCode()); + return Error::success(); +} + +DebugObjectSection *ELFDebugObject::getSection(StringRef Name) { + auto It = Sections.find(Name); + return It == Sections.end() ? nullptr : It->second.get(); +} + +/// Creates a debug object based on the input object file from +/// ObjectLinkingLayerJITLinkContext. +/// +static Expected<std::unique_ptr<DebugObject>> +createDebugObjectFromBuffer(ExecutionSession &ES, LinkGraph &G, + JITLinkContext &Ctx, MemoryBufferRef ObjBuffer) { + switch (G.getTargetTriple().getObjectFormat()) { + case Triple::ELF: + return ELFDebugObject::Create(ObjBuffer, Ctx, ES); + + default: + // TODO: Once we add support for other formats, we might want to split this + // into multiple files. + return nullptr; + } +} + +DebugObjectManagerPlugin::DebugObjectManagerPlugin( + ExecutionSession &ES, std::unique_ptr<DebugObjectRegistrar> Target) + : ES(ES), Target(std::move(Target)) {} + +DebugObjectManagerPlugin::~DebugObjectManagerPlugin() = default; + +void DebugObjectManagerPlugin::notifyMaterializing( + MaterializationResponsibility &MR, LinkGraph &G, JITLinkContext &Ctx, + MemoryBufferRef ObjBuffer) { + std::lock_guard<std::mutex> Lock(PendingObjsLock); + assert(PendingObjs.count(&MR) == 0 && + "Cannot have more than one pending debug object per " + "MaterializationResponsibility"); + + if (auto DebugObj = createDebugObjectFromBuffer(ES, G, Ctx, ObjBuffer)) { + // Not all link artifacts allow debugging. + if (*DebugObj != nullptr) + PendingObjs[&MR] = std::move(*DebugObj); + } else { + ES.reportError(DebugObj.takeError()); + } +} + +void DebugObjectManagerPlugin::modifyPassConfig( + MaterializationResponsibility &MR, LinkGraph &G, + PassConfiguration &PassConfig) { + // Not all link artifacts have associated debug objects. + std::lock_guard<std::mutex> Lock(PendingObjsLock); + auto It = PendingObjs.find(&MR); + if (It == PendingObjs.end()) + return; + + DebugObject &DebugObj = *It->second; + if (DebugObj.has(Requirement::ReportFinalSectionLoadAddresses)) { + PassConfig.PostAllocationPasses.push_back( + [&DebugObj](LinkGraph &Graph) -> Error { + for (const Section &GraphSection : Graph.sections()) + DebugObj.reportSectionTargetMemoryRange(GraphSection.getName(), + SectionRange(GraphSection)); + return Error::success(); + }); + } +} + +Error DebugObjectManagerPlugin::notifyEmitted( + MaterializationResponsibility &MR) { + std::lock_guard<std::mutex> Lock(PendingObjsLock); + auto It = PendingObjs.find(&MR); + if (It == PendingObjs.end()) + return Error::success(); + + // During finalization the debug object is registered with the target. + // Materialization must wait for this process to finish. Otherwise we might + // start running code before the debugger processed the corresponding debug + // info. + std::promise<MSVCPError> FinalizePromise; + std::future<MSVCPError> FinalizeErr = FinalizePromise.get_future(); + + It->second->finalizeAsync( + [this, &FinalizePromise, &MR](Expected<sys::MemoryBlock> TargetMem) { + // Any failure here will fail materialization. + if (!TargetMem) { + FinalizePromise.set_value(TargetMem.takeError()); + return; + } + if (Error Err = Target->registerDebugObject(*TargetMem)) { + FinalizePromise.set_value(std::move(Err)); + return; + } + + // Once our tracking info is updated, notifyEmitted() can return and + // finish materialization. + FinalizePromise.set_value(MR.withResourceKeyDo([&](ResourceKey K) { + assert(PendingObjs.count(&MR) && "We still hold PendingObjsLock"); + std::lock_guard<std::mutex> Lock(RegisteredObjsLock); + RegisteredObjs[K].push_back(std::move(PendingObjs[&MR])); + PendingObjs.erase(&MR); + })); + }); + + return FinalizeErr.get(); +} + +Error DebugObjectManagerPlugin::notifyFailed( + MaterializationResponsibility &MR) { + std::lock_guard<std::mutex> Lock(PendingObjsLock); + PendingObjs.erase(&MR); + return Error::success(); +} + +void DebugObjectManagerPlugin::notifyTransferringResources(ResourceKey DstKey, + ResourceKey SrcKey) { + // Debug objects are stored by ResourceKey only after registration. + // Thus, pending objects don't need to be updated here. + std::lock_guard<std::mutex> Lock(RegisteredObjsLock); + auto SrcIt = RegisteredObjs.find(SrcKey); + if (SrcIt != RegisteredObjs.end()) { + // Resources from distinct MaterializationResponsibilitys can get merged + // after emission, so we can have multiple debug objects per resource key. + for (std::unique_ptr<DebugObject> &DebugObj : SrcIt->second) + RegisteredObjs[DstKey].push_back(std::move(DebugObj)); + RegisteredObjs.erase(SrcIt); + } +} + +Error DebugObjectManagerPlugin::notifyRemovingResources(ResourceKey Key) { + // Removing the resource for a pending object fails materialization, so they + // get cleaned up in the notifyFailed() handler. + std::lock_guard<std::mutex> Lock(RegisteredObjsLock); + RegisteredObjs.erase(Key); + + // TODO: Implement unregister notifications. + return Error::success(); +} + +} // namespace orc +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp index 6247158919fa..5b386a458f1f 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp @@ -261,8 +261,7 @@ raw_ostream &operator<<(raw_ostream &OS, "JITDylibList entries must not be null"); OS << " (\"" << SearchOrder.front().first->getName() << "\", " << SearchOrder.begin()->second << ")"; - for (auto &KV : - make_range(std::next(SearchOrder.begin(), 1), SearchOrder.end())) { + for (auto &KV : llvm::drop_begin(SearchOrder)) { assert(KV.first && "JITDylibList entries must not be null"); OS << ", (\"" << KV.first->getName() << "\", " << KV.second << ")"; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp new file mode 100644 index 000000000000..5715eda71eee --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp @@ -0,0 +1,52 @@ +//===----- EPCDebugObjectRegistrar.cpp - EPC-based debug registration -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/EPCDebugObjectRegistrar.h" + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Shared/SimplePackedSerialization.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" +#include "llvm/Support/BinaryStreamWriter.h" + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<EPCDebugObjectRegistrar>> +createJITLoaderGDBRegistrar(ExecutionSession &ES) { + auto &EPC = ES.getExecutorProcessControl(); + auto ProcessHandle = EPC.loadDylib(nullptr); + if (!ProcessHandle) + return ProcessHandle.takeError(); + + SymbolStringPtr RegisterFn = + EPC.getTargetTriple().isOSBinFormatMachO() + ? EPC.intern("_llvm_orc_registerJITLoaderGDBWrapper") + : EPC.intern("llvm_orc_registerJITLoaderGDBWrapper"); + + SymbolLookupSet RegistrationSymbols; + RegistrationSymbols.add(RegisterFn); + + auto Result = EPC.lookupSymbols({{*ProcessHandle, RegistrationSymbols}}); + if (!Result) + return Result.takeError(); + + assert(Result->size() == 1 && "Unexpected number of dylibs in result"); + assert((*Result)[0].size() == 1 && + "Unexpected number of addresses in result"); + + return std::make_unique<EPCDebugObjectRegistrar>(ES, (*Result)[0][0]); +} + +Error EPCDebugObjectRegistrar::registerDebugObject(sys::MemoryBlock TargetMem) { + return ES.callSPSWrapper<void(SPSExecutorAddress, uint64_t)>( + RegisterFn, ExecutorAddress::fromPtr(TargetMem.base()), + static_cast<uint64_t>(TargetMem.allocatedSize())); +} + +} // namespace orc +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp index bbf3ada1d4ba..ba154aaecd1a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp @@ -1,4 +1,4 @@ -//===---------------- TPCDynamicLibrarySearchGenerator.cpp ----------------===// +//===---------------- EPCDynamicLibrarySearchGenerator.cpp ----------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,24 +6,24 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ExecutionEngine/Orc/TPCDynamicLibrarySearchGenerator.h" +#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" namespace llvm { namespace orc { -Expected<std::unique_ptr<TPCDynamicLibrarySearchGenerator>> -TPCDynamicLibrarySearchGenerator::Load(TargetProcessControl &TPC, +Expected<std::unique_ptr<EPCDynamicLibrarySearchGenerator>> +EPCDynamicLibrarySearchGenerator::Load(ExecutionSession &ES, const char *LibraryPath, SymbolPredicate Allow) { - auto Handle = TPC.loadDylib(LibraryPath); + auto Handle = ES.getExecutorProcessControl().loadDylib(LibraryPath); if (!Handle) return Handle.takeError(); - return std::make_unique<TPCDynamicLibrarySearchGenerator>(TPC, *Handle, + return std::make_unique<EPCDynamicLibrarySearchGenerator>(ES, *Handle, std::move(Allow)); } -Error TPCDynamicLibrarySearchGenerator::tryToGenerate( +Error EPCDynamicLibrarySearchGenerator::tryToGenerate( LookupState &LS, LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { @@ -41,8 +41,8 @@ Error TPCDynamicLibrarySearchGenerator::tryToGenerate( SymbolMap NewSymbols; - TargetProcessControl::LookupRequest Request(H, LookupSymbols); - auto Result = TPC.lookupSymbols(Request); + ExecutorProcessControl::LookupRequest Request(H, LookupSymbols); + auto Result = EPC.lookupSymbols(Request); if (!Result) return Result.takeError(); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp new file mode 100644 index 000000000000..8cdda9ab5a15 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp @@ -0,0 +1,73 @@ +//===------ EPCEHFrameRegistrar.cpp - EPC-based eh-frame registration -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" + +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/Support/BinaryStreamWriter.h" + +using namespace llvm::orc::shared; + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<EPCEHFrameRegistrar>> +EPCEHFrameRegistrar::Create(ExecutionSession &ES) { + // FIXME: Proper mangling here -- we really need to decouple linker mangling + // from DataLayout. + + // Find the addresses of the registration/deregistration functions in the + // executor process. + auto &EPC = ES.getExecutorProcessControl(); + auto ProcessHandle = EPC.loadDylib(nullptr); + if (!ProcessHandle) + return ProcessHandle.takeError(); + + std::string RegisterWrapperName, DeregisterWrapperName; + if (EPC.getTargetTriple().isOSBinFormatMachO()) { + RegisterWrapperName += '_'; + DeregisterWrapperName += '_'; + } + RegisterWrapperName += "llvm_orc_registerEHFrameSectionWrapper"; + DeregisterWrapperName += "llvm_orc_deregisterEHFrameSectionWrapper"; + + SymbolLookupSet RegistrationSymbols; + RegistrationSymbols.add(EPC.intern(RegisterWrapperName)); + RegistrationSymbols.add(EPC.intern(DeregisterWrapperName)); + + auto Result = EPC.lookupSymbols({{*ProcessHandle, RegistrationSymbols}}); + if (!Result) + return Result.takeError(); + + assert(Result->size() == 1 && "Unexpected number of dylibs in result"); + assert((*Result)[0].size() == 2 && + "Unexpected number of addresses in result"); + + auto RegisterEHFrameWrapperFnAddr = (*Result)[0][0]; + auto DeregisterEHFrameWrapperFnAddr = (*Result)[0][1]; + + return std::make_unique<EPCEHFrameRegistrar>(ES, RegisterEHFrameWrapperFnAddr, + DeregisterEHFrameWrapperFnAddr); +} + +Error EPCEHFrameRegistrar::registerEHFrames(JITTargetAddress EHFrameSectionAddr, + size_t EHFrameSectionSize) { + return ES.callSPSWrapper<void(SPSExecutorAddress, uint64_t)>( + RegisterEHFrameWrapperFnAddr, EHFrameSectionAddr, + static_cast<uint64_t>(EHFrameSectionSize)); +} + +Error EPCEHFrameRegistrar::deregisterEHFrames( + JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { + return ES.callSPSWrapper<void(SPSExecutorAddress, uint64_t)>( + DeregisterEHFrameWrapperFnAddr, EHFrameSectionAddr, + static_cast<uint64_t>(EHFrameSectionSize)); +} + +} // end namespace orc +} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp index 7989ec41952d..b9c70b0aeb3c 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCIndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp @@ -1,4 +1,4 @@ -//===------ TargetProcessControl.cpp -- Target process control APIs -------===// +//===------- EPCIndirectionUtils.cpp -- EPC based indirection APIs --------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,9 +6,9 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ExecutionEngine/Orc/TPCIndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/EPCIndirectionUtils.h" -#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" #include "llvm/Support/MathExtras.h" #include <future> @@ -19,14 +19,14 @@ using namespace llvm::orc; namespace llvm { namespace orc { -class TPCIndirectionUtilsAccess { +class EPCIndirectionUtilsAccess { public: - using IndirectStubInfo = TPCIndirectionUtils::IndirectStubInfo; - using IndirectStubInfoVector = TPCIndirectionUtils::IndirectStubInfoVector; + using IndirectStubInfo = EPCIndirectionUtils::IndirectStubInfo; + using IndirectStubInfoVector = EPCIndirectionUtils::IndirectStubInfoVector; static Expected<IndirectStubInfoVector> - getIndirectStubs(TPCIndirectionUtils &TPCIU, unsigned NumStubs) { - return TPCIU.getIndirectStubs(NumStubs); + getIndirectStubs(EPCIndirectionUtils &EPCIU, unsigned NumStubs) { + return EPCIU.getIndirectStubs(NumStubs); }; }; @@ -35,9 +35,9 @@ public: namespace { -class TPCTrampolinePool : public TrampolinePool { +class EPCTrampolinePool : public TrampolinePool { public: - TPCTrampolinePool(TPCIndirectionUtils &TPCIU); + EPCTrampolinePool(EPCIndirectionUtils &EPCIU); Error deallocatePool(); protected: @@ -45,16 +45,16 @@ protected: using Allocation = jitlink::JITLinkMemoryManager::Allocation; - TPCIndirectionUtils &TPCIU; + EPCIndirectionUtils &EPCIU; unsigned TrampolineSize = 0; unsigned TrampolinesPerPage = 0; std::vector<std::unique_ptr<Allocation>> TrampolineBlocks; }; -class TPCIndirectStubsManager : public IndirectStubsManager, - private TPCIndirectionUtilsAccess { +class EPCIndirectStubsManager : public IndirectStubsManager, + private EPCIndirectionUtilsAccess { public: - TPCIndirectStubsManager(TPCIndirectionUtils &TPCIU) : TPCIU(TPCIU) {} + EPCIndirectStubsManager(EPCIndirectionUtils &EPCIU) : EPCIU(EPCIU) {} Error deallocateStubs(); @@ -73,43 +73,43 @@ private: using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; std::mutex ISMMutex; - TPCIndirectionUtils &TPCIU; + EPCIndirectionUtils &EPCIU; StringMap<StubInfo> StubInfos; }; -TPCTrampolinePool::TPCTrampolinePool(TPCIndirectionUtils &TPCIU) - : TPCIU(TPCIU) { - auto &TPC = TPCIU.getTargetProcessControl(); - auto &ABI = TPCIU.getABISupport(); +EPCTrampolinePool::EPCTrampolinePool(EPCIndirectionUtils &EPCIU) + : EPCIU(EPCIU) { + auto &EPC = EPCIU.getExecutorProcessControl(); + auto &ABI = EPCIU.getABISupport(); TrampolineSize = ABI.getTrampolineSize(); TrampolinesPerPage = - (TPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; + (EPC.getPageSize() - ABI.getPointerSize()) / TrampolineSize; } -Error TPCTrampolinePool::deallocatePool() { +Error EPCTrampolinePool::deallocatePool() { Error Err = Error::success(); for (auto &Alloc : TrampolineBlocks) Err = joinErrors(std::move(Err), Alloc->deallocate()); return Err; } -Error TPCTrampolinePool::grow() { +Error EPCTrampolinePool::grow() { assert(AvailableTrampolines.empty() && "Grow called with trampolines still available"); - auto ResolverAddress = TPCIU.getResolverBlockAddress(); + auto ResolverAddress = EPCIU.getResolverBlockAddress(); assert(ResolverAddress && "Resolver address can not be null"); - auto &TPC = TPCIU.getTargetProcessControl(); + auto &EPC = EPCIU.getExecutorProcessControl(); constexpr auto TrampolinePagePermissions = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | sys::Memory::MF_EXEC); - auto PageSize = TPC.getPageSize(); + auto PageSize = EPC.getPageSize(); jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; Request[TrampolinePagePermissions] = {PageSize, static_cast<size_t>(PageSize), 0}; - auto Alloc = TPC.getMemMgr().allocate(nullptr, Request); + auto Alloc = EPC.getMemMgr().allocate(nullptr, Request); if (!Alloc) return Alloc.takeError(); @@ -119,7 +119,7 @@ Error TPCTrampolinePool::grow() { auto WorkingMemory = (*Alloc)->getWorkingMemory(TrampolinePagePermissions); auto TargetAddress = (*Alloc)->getTargetMemory(TrampolinePagePermissions); - TPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress, + EPCIU.getABISupport().writeTrampolines(WorkingMemory.data(), TargetAddress, ResolverAddress, NumTrampolines); auto TargetAddr = (*Alloc)->getTargetMemory(TrampolinePagePermissions); @@ -134,7 +134,7 @@ Error TPCTrampolinePool::grow() { return Error::success(); } -Error TPCIndirectStubsManager::createStub(StringRef StubName, +Error EPCIndirectStubsManager::createStub(StringRef StubName, JITTargetAddress StubAddr, JITSymbolFlags StubFlags) { StubInitsMap SIM; @@ -142,8 +142,8 @@ Error TPCIndirectStubsManager::createStub(StringRef StubName, return createStubs(SIM); } -Error TPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { - auto AvailableStubInfos = getIndirectStubs(TPCIU, StubInits.size()); +Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { + auto AvailableStubInfos = getIndirectStubs(EPCIU, StubInits.size()); if (!AvailableStubInfos) return AvailableStubInfos.takeError(); @@ -156,8 +156,8 @@ Error TPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { } } - auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess(); - switch (TPCIU.getABISupport().getPointerSize()) { + auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); + switch (EPCIU.getABISupport().getPointerSize()) { case 4: { unsigned ASIdx = 0; std::vector<tpctypes::UInt32Write> PtrUpdates; @@ -180,7 +180,7 @@ Error TPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { } } -JITEvaluatedSymbol TPCIndirectStubsManager::findStub(StringRef Name, +JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, bool ExportedStubsOnly) { std::lock_guard<std::mutex> Lock(ISMMutex); auto I = StubInfos.find(Name); @@ -189,7 +189,7 @@ JITEvaluatedSymbol TPCIndirectStubsManager::findStub(StringRef Name, return {I->second.first.StubAddress, I->second.second}; } -JITEvaluatedSymbol TPCIndirectStubsManager::findPointer(StringRef Name) { +JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { std::lock_guard<std::mutex> Lock(ISMMutex); auto I = StubInfos.find(Name); if (I == StubInfos.end()) @@ -197,7 +197,7 @@ JITEvaluatedSymbol TPCIndirectStubsManager::findPointer(StringRef Name) { return {I->second.first.PointerAddress, I->second.second}; } -Error TPCIndirectStubsManager::updatePointer(StringRef Name, +Error EPCIndirectStubsManager::updatePointer(StringRef Name, JITTargetAddress NewAddr) { JITTargetAddress PtrAddr = 0; @@ -210,8 +210,8 @@ Error TPCIndirectStubsManager::updatePointer(StringRef Name, PtrAddr = I->second.first.PointerAddress; } - auto &MemAccess = TPCIU.getTargetProcessControl().getMemoryAccess(); - switch (TPCIU.getABISupport().getPointerSize()) { + auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); + switch (EPCIU.getABISupport().getPointerSize()) { case 4: { tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr); return MemAccess.writeUInt32s(PUpdate); @@ -231,42 +231,42 @@ Error TPCIndirectStubsManager::updatePointer(StringRef Name, namespace llvm { namespace orc { -TPCIndirectionUtils::ABISupport::~ABISupport() {} +EPCIndirectionUtils::ABISupport::~ABISupport() {} -Expected<std::unique_ptr<TPCIndirectionUtils>> -TPCIndirectionUtils::Create(TargetProcessControl &TPC) { - const auto &TT = TPC.getTargetTriple(); +Expected<std::unique_ptr<EPCIndirectionUtils>> +EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { + const auto &TT = EPC.getTargetTriple(); switch (TT.getArch()) { default: return make_error<StringError>( - std::string("No TPCIndirectionUtils available for ") + TT.str(), + std::string("No EPCIndirectionUtils available for ") + TT.str(), inconvertibleErrorCode()); case Triple::aarch64: case Triple::aarch64_32: - return CreateWithABI<OrcAArch64>(TPC); + return CreateWithABI<OrcAArch64>(EPC); case Triple::x86: - return CreateWithABI<OrcI386>(TPC); + return CreateWithABI<OrcI386>(EPC); case Triple::mips: - return CreateWithABI<OrcMips32Be>(TPC); + return CreateWithABI<OrcMips32Be>(EPC); case Triple::mipsel: - return CreateWithABI<OrcMips32Le>(TPC); + return CreateWithABI<OrcMips32Le>(EPC); case Triple::mips64: case Triple::mips64el: - return CreateWithABI<OrcMips64>(TPC); + return CreateWithABI<OrcMips64>(EPC); case Triple::x86_64: if (TT.getOS() == Triple::OSType::Win32) - return CreateWithABI<OrcX86_64_Win32>(TPC); + return CreateWithABI<OrcX86_64_Win32>(EPC); else - return CreateWithABI<OrcX86_64_SysV>(TPC); + return CreateWithABI<OrcX86_64_SysV>(EPC); } } -Error TPCIndirectionUtils::cleanup() { +Error EPCIndirectionUtils::cleanup() { Error Err = Error::success(); for (auto &A : IndirectStubAllocs) @@ -274,7 +274,7 @@ Error TPCIndirectionUtils::cleanup() { if (TP) Err = joinErrors(std::move(Err), - static_cast<TPCTrampolinePool &>(*TP).deallocatePool()); + static_cast<EPCTrampolinePool &>(*TP).deallocatePool()); if (ResolverBlock) Err = joinErrors(std::move(Err), ResolverBlock->deallocate()); @@ -283,7 +283,7 @@ Error TPCIndirectionUtils::cleanup() { } Expected<JITTargetAddress> -TPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, +EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, JITTargetAddress ReentryCtxAddr) { assert(ABI && "ABI can not be null"); constexpr auto ResolverBlockPermissions = @@ -292,9 +292,9 @@ TPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, auto ResolverSize = ABI->getResolverCodeSize(); jitlink::JITLinkMemoryManager::SegmentsRequestMap Request; - Request[ResolverBlockPermissions] = {TPC.getPageSize(), + Request[ResolverBlockPermissions] = {EPC.getPageSize(), static_cast<size_t>(ResolverSize), 0}; - auto Alloc = TPC.getMemMgr().allocate(nullptr, Request); + auto Alloc = EPC.getMemMgr().allocate(nullptr, Request); if (!Alloc) return Alloc.takeError(); @@ -311,17 +311,17 @@ TPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, } std::unique_ptr<IndirectStubsManager> -TPCIndirectionUtils::createIndirectStubsManager() { - return std::make_unique<TPCIndirectStubsManager>(*this); +EPCIndirectionUtils::createIndirectStubsManager() { + return std::make_unique<EPCIndirectStubsManager>(*this); } -TrampolinePool &TPCIndirectionUtils::getTrampolinePool() { +TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { if (!TP) - TP = std::make_unique<TPCTrampolinePool>(*this); + TP = std::make_unique<EPCTrampolinePool>(*this); return *TP; } -LazyCallThroughManager &TPCIndirectionUtils::createLazyCallThroughManager( +LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { assert(!LCTM && "createLazyCallThroughManager can not have been called before"); @@ -330,24 +330,24 @@ LazyCallThroughManager &TPCIndirectionUtils::createLazyCallThroughManager( return *LCTM; } -TPCIndirectionUtils::TPCIndirectionUtils(TargetProcessControl &TPC, +EPCIndirectionUtils::EPCIndirectionUtils(ExecutorProcessControl &EPC, std::unique_ptr<ABISupport> ABI) - : TPC(TPC), ABI(std::move(ABI)) { + : EPC(EPC), ABI(std::move(ABI)) { assert(this->ABI && "ABI can not be null"); - assert(TPC.getPageSize() > getABISupport().getStubSize() && + assert(EPC.getPageSize() > getABISupport().getStubSize() && "Stubs larger than one page are not supported"); } -Expected<TPCIndirectionUtils::IndirectStubInfoVector> -TPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { +Expected<EPCIndirectionUtils::IndirectStubInfoVector> +EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { - std::lock_guard<std::mutex> Lock(TPCUIMutex); + std::lock_guard<std::mutex> Lock(EPCUIMutex); // If there aren't enough stubs available then allocate some more. if (NumStubs > AvailableIndirectStubs.size()) { auto NumStubsToAllocate = NumStubs; - auto PageSize = TPC.getPageSize(); + auto PageSize = EPC.getPageSize(); auto StubBytes = alignTo(NumStubsToAllocate * ABI->getStubSize(), PageSize); NumStubsToAllocate = StubBytes / ABI->getStubSize(); auto PointerBytes = @@ -364,7 +364,7 @@ TPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { Request[StubPagePermissions] = {PageSize, static_cast<size_t>(StubBytes), 0}; Request[PointerPagePermissions] = {PageSize, 0, PointerBytes}; - auto Alloc = TPC.getMemMgr().allocate(nullptr, Request); + auto Alloc = EPC.getMemMgr().allocate(nullptr, Request); if (!Alloc) return Alloc.takeError(); @@ -411,9 +411,9 @@ static JITTargetAddress reentry(JITTargetAddress LCTMAddr, return LandingAddrF.get(); } -Error setUpInProcessLCTMReentryViaTPCIU(TPCIndirectionUtils &TPCIU) { - auto &LCTM = TPCIU.getLazyCallThroughManager(); - return TPCIU +Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { + auto &LCTM = EPCIU.getLazyCallThroughManager(); + return EPCIU .writeResolverBlock(pointerToJITTargetAddress(&reentry), pointerToJITTargetAddress(&LCTM)) .takeError(); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 6a1a41a13a1b..7a76a6ccc122 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" - #include "llvm/ExecutionEngine/Orc/Layer.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" @@ -17,6 +16,7 @@ #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Target/TargetMachine.h" +#include <string> namespace llvm { namespace orc { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index 7bf874e88c26..7d86d125d1db 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcessControl.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -1,4 +1,4 @@ -//===------ TargetProcessControl.cpp -- Target process control APIs -------===// +//===---- ExecutorProcessControl.cpp -- Executor process control APIs -----===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,26 +6,27 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" +#include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Host.h" #include "llvm/Support/Process.h" -#include <mutex> +#define DEBUG_TYPE "orc" namespace llvm { namespace orc { -TargetProcessControl::MemoryAccess::~MemoryAccess() {} +ExecutorProcessControl::MemoryAccess::~MemoryAccess() {} -TargetProcessControl::~TargetProcessControl() {} +ExecutorProcessControl::~ExecutorProcessControl() {} -SelfTargetProcessControl::SelfTargetProcessControl( +SelfExecutorProcessControl::SelfExecutorProcessControl( std::shared_ptr<SymbolStringPool> SSP, Triple TargetTriple, unsigned PageSize, std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) - : TargetProcessControl(std::move(SSP)) { + : ExecutorProcessControl(std::move(SSP)) { OwnedMemMgr = std::move(MemMgr); if (!OwnedMemMgr) @@ -35,26 +36,32 @@ SelfTargetProcessControl::SelfTargetProcessControl( this->PageSize = PageSize; this->MemMgr = OwnedMemMgr.get(); this->MemAccess = this; + this->JDI = {ExecutorAddress::fromPtr(jitDispatchViaWrapperFunctionManager), + ExecutorAddress::fromPtr(this)}; if (this->TargetTriple.isOSBinFormatMachO()) GlobalManglingPrefix = '_'; } -Expected<std::unique_ptr<SelfTargetProcessControl>> -SelfTargetProcessControl::Create( +Expected<std::unique_ptr<SelfExecutorProcessControl>> +SelfExecutorProcessControl::Create( std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) { + + if (!SSP) + SSP = std::make_shared<SymbolStringPool>(); + auto PageSize = sys::Process::getPageSize(); if (!PageSize) return PageSize.takeError(); Triple TT(sys::getProcessTriple()); - return std::make_unique<SelfTargetProcessControl>( + return std::make_unique<SelfExecutorProcessControl>( std::move(SSP), std::move(TT), *PageSize, std::move(MemMgr)); } Expected<tpctypes::DylibHandle> -SelfTargetProcessControl::loadDylib(const char *DylibPath) { +SelfExecutorProcessControl::loadDylib(const char *DylibPath) { std::string ErrMsg; auto Dylib = std::make_unique<sys::DynamicLibrary>( sys::DynamicLibrary::getPermanentLibrary(DylibPath, &ErrMsg)); @@ -65,7 +72,7 @@ SelfTargetProcessControl::loadDylib(const char *DylibPath) { } Expected<std::vector<tpctypes::LookupResult>> -SelfTargetProcessControl::lookupSymbols(ArrayRef<LookupRequest> Request) { +SelfExecutorProcessControl::lookupSymbols(ArrayRef<LookupRequest> Request) { std::vector<tpctypes::LookupResult> R; for (auto &Elem : Request) { @@ -96,58 +103,81 @@ SelfTargetProcessControl::lookupSymbols(ArrayRef<LookupRequest> Request) { } Expected<int32_t> -SelfTargetProcessControl::runAsMain(JITTargetAddress MainFnAddr, - ArrayRef<std::string> Args) { +SelfExecutorProcessControl::runAsMain(JITTargetAddress MainFnAddr, + ArrayRef<std::string> Args) { using MainTy = int (*)(int, char *[]); return orc::runAsMain(jitTargetAddressToFunction<MainTy>(MainFnAddr), Args); } -Expected<tpctypes::WrapperFunctionResult> -SelfTargetProcessControl::runWrapper(JITTargetAddress WrapperFnAddr, - ArrayRef<uint8_t> ArgBuffer) { +void SelfExecutorProcessControl::callWrapperAsync( + SendResultFunction SendResult, JITTargetAddress WrapperFnAddr, + ArrayRef<char> ArgBuffer) { using WrapperFnTy = - tpctypes::CWrapperFunctionResult (*)(const uint8_t *Data, uint64_t Size); + shared::detail::CWrapperFunctionResult (*)(const char *Data, size_t Size); auto *WrapperFn = jitTargetAddressToFunction<WrapperFnTy>(WrapperFnAddr); - return WrapperFn(ArgBuffer.data(), ArgBuffer.size()); + SendResult(WrapperFn(ArgBuffer.data(), ArgBuffer.size())); } -Error SelfTargetProcessControl::disconnect() { return Error::success(); } +Error SelfExecutorProcessControl::disconnect() { return Error::success(); } -void SelfTargetProcessControl::writeUInt8s(ArrayRef<tpctypes::UInt8Write> Ws, - WriteResultFn OnWriteComplete) { +void SelfExecutorProcessControl::writeUInt8s(ArrayRef<tpctypes::UInt8Write> Ws, + WriteResultFn OnWriteComplete) { for (auto &W : Ws) *jitTargetAddressToPointer<uint8_t *>(W.Address) = W.Value; OnWriteComplete(Error::success()); } -void SelfTargetProcessControl::writeUInt16s(ArrayRef<tpctypes::UInt16Write> Ws, - WriteResultFn OnWriteComplete) { +void SelfExecutorProcessControl::writeUInt16s( + ArrayRef<tpctypes::UInt16Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) *jitTargetAddressToPointer<uint16_t *>(W.Address) = W.Value; OnWriteComplete(Error::success()); } -void SelfTargetProcessControl::writeUInt32s(ArrayRef<tpctypes::UInt32Write> Ws, - WriteResultFn OnWriteComplete) { +void SelfExecutorProcessControl::writeUInt32s( + ArrayRef<tpctypes::UInt32Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) *jitTargetAddressToPointer<uint32_t *>(W.Address) = W.Value; OnWriteComplete(Error::success()); } -void SelfTargetProcessControl::writeUInt64s(ArrayRef<tpctypes::UInt64Write> Ws, - WriteResultFn OnWriteComplete) { +void SelfExecutorProcessControl::writeUInt64s( + ArrayRef<tpctypes::UInt64Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) *jitTargetAddressToPointer<uint64_t *>(W.Address) = W.Value; OnWriteComplete(Error::success()); } -void SelfTargetProcessControl::writeBuffers(ArrayRef<tpctypes::BufferWrite> Ws, - WriteResultFn OnWriteComplete) { +void SelfExecutorProcessControl::writeBuffers( + ArrayRef<tpctypes::BufferWrite> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) memcpy(jitTargetAddressToPointer<char *>(W.Address), W.Buffer.data(), W.Buffer.size()); OnWriteComplete(Error::success()); } +shared::detail::CWrapperFunctionResult +SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager( + void *Ctx, const void *FnTag, const char *Data, size_t Size) { + + LLVM_DEBUG({ + dbgs() << "jit-dispatch call with tag " << FnTag << " and " << Size + << " byte payload.\n"; + }); + + std::promise<shared::WrapperFunctionResult> ResultP; + auto ResultF = ResultP.get_future(); + static_cast<SelfExecutorProcessControl *>(Ctx) + ->getExecutionSession() + .runJITDispatchHandler( + [ResultP = std::move(ResultP)]( + shared::WrapperFunctionResult Result) mutable { + ResultP.set_value(std::move(Result)); + }, + pointerToJITTargetAddress(FnTag), {Data, Size}); + + return ResultF.get().release(); +} + } // end namespace orc } // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 1cfcf8ae943d..e8dd1bb90c9a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -316,8 +316,9 @@ void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, "modules."); SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned. - CloneFunctionInto(NewF, &OrigF, VMap, /*ModuleLevelChanges=*/true, Returns, - "", nullptr, nullptr, Materializer); + CloneFunctionInto(NewF, &OrigF, VMap, + CloneFunctionChangeType::DifferentModule, Returns, "", + nullptr, nullptr, Materializer); OrigF.deleteBody(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index 8cf66c9e759a..4257137a2212 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -65,9 +65,13 @@ JITTargetMachineBuilder &JITTargetMachineBuilder::addFeatures( } #ifndef NDEBUG -raw_ostream &operator<<(raw_ostream &OS, const JITTargetMachineBuilder &JTMB) { - OS << "{ Triple = \"" << JTMB.TT.str() << "\", CPU = \"" << JTMB.CPU - << "\", Options = <not-printable>, Relocation Model = "; +void JITTargetMachineBuilderPrinter::print(raw_ostream &OS) const { + OS << Indent << "{\n" + << Indent << " Triple = \"" << JTMB.TT.str() << "\"\n" + << Indent << " CPU = \"" << JTMB.CPU << "\"\n" + << Indent << " Features = \"" << JTMB.Features.getString() << "\"\n" + << Indent << " Options = <not-printable>\n" + << Indent << " Relocation Model = "; if (JTMB.RM) { switch (*JTMB.RM) { @@ -91,9 +95,10 @@ raw_ostream &operator<<(raw_ostream &OS, const JITTargetMachineBuilder &JTMB) { break; } } else - OS << "unspecified"; + OS << "unspecified (will use target default)"; - OS << ", Code Model = "; + OS << "\n" + << Indent << " Code Model = "; if (JTMB.CM) { switch (*JTMB.CM) { @@ -114,9 +119,10 @@ raw_ostream &operator<<(raw_ostream &OS, const JITTargetMachineBuilder &JTMB) { break; } } else - OS << "unspecified"; + OS << "unspecified (will use target default)"; - OS << ", Optimization Level = "; + OS << "\n" + << Indent << " Optimization Level = "; switch (JTMB.OptLevel) { case CodeGenOpt::None: OS << "None"; @@ -132,8 +138,7 @@ raw_ostream &operator<<(raw_ostream &OS, const JITTargetMachineBuilder &JTMB) { break; } - OS << " }"; - return OS; + OS << "\n" << Indent << "}\n"; } #endif // NDEBUG diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index c368c1e37134..2ac32293e4db 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -9,12 +9,12 @@ #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" -#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" @@ -124,7 +124,6 @@ private: /// some runtime API, including __cxa_atexit, dlopen, and dlclose. class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { public: - // GenericLLVMIRPlatform &P) : P(P) { GenericLLVMIRPlatformSupport(LLJIT &J) : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")) { @@ -503,395 +502,26 @@ GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM, return std::move(TSM); } -class MachOPlatformSupport : public LLJIT::PlatformSupport { +/// Inactive Platform Support +/// +/// Explicitly disables platform support. JITDylibs are not scanned for special +/// init/deinit symbols. No runtime API interposes are injected. +class InactivePlatformSupport : public LLJIT::PlatformSupport { public: - using DLOpenType = void *(*)(const char *Name, int Mode); - using DLCloseType = int (*)(void *Handle); - using DLSymType = void *(*)(void *Handle, const char *Name); - using DLErrorType = const char *(*)(); - - struct DlFcnValues { - Optional<void *> RTLDDefault; - DLOpenType dlopen = nullptr; - DLCloseType dlclose = nullptr; - DLSymType dlsym = nullptr; - DLErrorType dlerror = nullptr; - }; - - static Expected<std::unique_ptr<MachOPlatformSupport>> - Create(LLJIT &J, JITDylib &PlatformJITDylib) { - - // Make process symbols visible. - { - std::string ErrMsg; - auto Lib = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg); - if (!Lib.isValid()) - return make_error<StringError>(std::move(ErrMsg), - inconvertibleErrorCode()); - } - - DlFcnValues DlFcn; - - // Add support for RTLDDefault on known platforms. -#ifdef __APPLE__ - DlFcn.RTLDDefault = reinterpret_cast<void *>(-2); -#endif // __APPLE__ - - if (auto Err = hookUpFunction(DlFcn.dlopen, "dlopen")) - return std::move(Err); - if (auto Err = hookUpFunction(DlFcn.dlclose, "dlclose")) - return std::move(Err); - if (auto Err = hookUpFunction(DlFcn.dlsym, "dlsym")) - return std::move(Err); - if (auto Err = hookUpFunction(DlFcn.dlerror, "dlerror")) - return std::move(Err); - - std::unique_ptr<MachOPlatformSupport> MP( - new MachOPlatformSupport(J, PlatformJITDylib, DlFcn)); - return std::move(MP); - } + InactivePlatformSupport() = default; Error initialize(JITDylib &JD) override { - LLVM_DEBUG({ - dbgs() << "MachOPlatformSupport initializing \"" << JD.getName() - << "\"\n"; - }); - - auto InitSeq = MP.getInitializerSequence(JD); - if (!InitSeq) - return InitSeq.takeError(); - - // If ObjC is not enabled but there are JIT'd ObjC inits then return - // an error. - if (!objCRegistrationEnabled()) - for (auto &KV : *InitSeq) { - if (!KV.second.getObjCSelRefsSections().empty() || - !KV.second.getObjCClassListSections().empty()) - return make_error<StringError>("JITDylib " + KV.first->getName() + - " contains objc metadata but objc" - " is not enabled", - inconvertibleErrorCode()); - } - - // Run the initializers. - for (auto &KV : *InitSeq) { - if (objCRegistrationEnabled()) { - KV.second.registerObjCSelectors(); - if (auto Err = KV.second.registerObjCClasses()) { - // FIXME: Roll back registrations on error? - return Err; - } - } - KV.second.runModInits(); - } - + LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for " + << JD.getName() << "\n"); return Error::success(); } Error deinitialize(JITDylib &JD) override { - auto &ES = J.getExecutionSession(); - if (auto DeinitSeq = MP.getDeinitializerSequence(JD)) { - for (auto &KV : *DeinitSeq) { - auto DSOHandleName = ES.intern("___dso_handle"); - - // FIXME: Run DeInits here. - auto Result = ES.lookup( - {{KV.first, JITDylibLookupFlags::MatchAllSymbols}}, - SymbolLookupSet(DSOHandleName, - SymbolLookupFlags::WeaklyReferencedSymbol)); - if (!Result) - return Result.takeError(); - if (Result->empty()) - continue; - assert(Result->count(DSOHandleName) && - "Result does not contain __dso_handle"); - auto *DSOHandle = jitTargetAddressToPointer<void *>( - Result->begin()->second.getAddress()); - AtExitMgr.runAtExits(DSOHandle); - } - } else - return DeinitSeq.takeError(); + LLVM_DEBUG( + dbgs() << "InactivePlatformSupport: no deinitializers running for " + << JD.getName() << "\n"); return Error::success(); } - -private: - template <typename FunctionPtrTy> - static Error hookUpFunction(FunctionPtrTy &Fn, const char *Name) { - if (auto *FnAddr = sys::DynamicLibrary::SearchForAddressOfSymbol(Name)) { - Fn = reinterpret_cast<FunctionPtrTy>(Fn); - return Error::success(); - } - - return make_error<StringError>((Twine("Can not enable MachO JIT Platform: " - "missing function: ") + - Name) - .str(), - inconvertibleErrorCode()); - } - - MachOPlatformSupport(LLJIT &J, JITDylib &PlatformJITDylib, DlFcnValues DlFcn) - : J(J), MP(setupPlatform(J)), DlFcn(std::move(DlFcn)) { - - SymbolMap HelperSymbols; - - // platform and atexit helpers. - HelperSymbols[J.mangleAndIntern("__lljit.platform_support_instance")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper), - JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.run_atexits_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper), - JITSymbolFlags()); - - // dlfcn helpers. - HelperSymbols[J.mangleAndIntern("__lljit.dlopen_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(dlopenHelper), - JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.dlclose_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(dlcloseHelper), - JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.dlsym_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(dlsymHelper), - JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.dlerror_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(dlerrorHelper), - JITSymbolFlags()); - - cantFail( - PlatformJITDylib.define(absoluteSymbols(std::move(HelperSymbols)))); - cantFail(MP.setupJITDylib(J.getMainJITDylib())); - cantFail(J.addIRModule(PlatformJITDylib, createPlatformRuntimeModule())); - } - - static MachOPlatform &setupPlatform(LLJIT &J) { - auto Tmp = std::make_unique<MachOPlatform>( - J.getExecutionSession(), - static_cast<ObjectLinkingLayer &>(J.getObjLinkingLayer()), - createStandardSymbolsObject(J)); - auto &MP = *Tmp; - J.getExecutionSession().setPlatform(std::move(Tmp)); - return MP; - } - - static std::unique_ptr<MemoryBuffer> createStandardSymbolsObject(LLJIT &J) { - LLVMContext Ctx; - Module M("__standard_symbols", Ctx); - M.setDataLayout(J.getDataLayout()); - - auto *Int64Ty = Type::getInt64Ty(Ctx); - - auto *DSOHandle = - new GlobalVariable(M, Int64Ty, true, GlobalValue::ExternalLinkage, - ConstantInt::get(Int64Ty, 0), "__dso_handle"); - DSOHandle->setVisibility(GlobalValue::DefaultVisibility); - - return cantFail(J.getIRCompileLayer().getCompiler()(M)); - } - - ThreadSafeModule createPlatformRuntimeModule() { - auto Ctx = std::make_unique<LLVMContext>(); - auto M = std::make_unique<Module>("__standard_lib", *Ctx); - M->setDataLayout(J.getDataLayout()); - - auto *MachOPlatformSupportTy = - StructType::create(*Ctx, "lljit.MachOPlatformSupport"); - - auto *PlatformInstanceDecl = new GlobalVariable( - *M, MachOPlatformSupportTy, true, GlobalValue::ExternalLinkage, nullptr, - "__lljit.platform_support_instance"); - - auto *Int8Ty = Type::getInt8Ty(*Ctx); - auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); - auto *VoidTy = Type::getVoidTy(*Ctx); - auto *BytePtrTy = PointerType::getUnqual(Int8Ty); - auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); - auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); - - addHelperAndWrapper( - *M, "__cxa_atexit", - FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, - false), - GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper", - {PlatformInstanceDecl}); - - addHelperAndWrapper(*M, "dlopen", - FunctionType::get(BytePtrTy, {BytePtrTy, IntTy}, false), - GlobalValue::DefaultVisibility, "__lljit.dlopen_helper", - {PlatformInstanceDecl}); - - addHelperAndWrapper(*M, "dlclose", - FunctionType::get(IntTy, {BytePtrTy}, false), - GlobalValue::DefaultVisibility, - "__lljit.dlclose_helper", {PlatformInstanceDecl}); - - addHelperAndWrapper( - *M, "dlsym", - FunctionType::get(BytePtrTy, {BytePtrTy, BytePtrTy}, false), - GlobalValue::DefaultVisibility, "__lljit.dlsym_helper", - {PlatformInstanceDecl}); - - addHelperAndWrapper(*M, "dlerror", FunctionType::get(BytePtrTy, {}, false), - GlobalValue::DefaultVisibility, - "__lljit.dlerror_helper", {PlatformInstanceDecl}); - - return ThreadSafeModule(std::move(M), std::move(Ctx)); - } - - static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, - void *DSOHandle) { - static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.registerAtExit( - F, Ctx, DSOHandle); - } - - static void runAtExitsHelper(void *Self, void *DSOHandle) { - static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.runAtExits(DSOHandle); - } - - void *jit_dlopen(const char *Path, int Mode) { - JITDylib *JDToOpen = nullptr; - // FIXME: Do the right thing with Mode flags. - { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - - // Clear any existing error messages. - dlErrorMsgs.erase(std::this_thread::get_id()); - - if (auto *JD = J.getExecutionSession().getJITDylibByName(Path)) { - auto I = JDRefCounts.find(JD); - if (I != JDRefCounts.end()) { - ++I->second; - return JD; - } - - JDRefCounts[JD] = 1; - JDToOpen = JD; - } - } - - if (JDToOpen) { - if (auto Err = initialize(*JDToOpen)) { - recordError(std::move(Err)); - return 0; - } - } - - // Fall through to dlopen if no JITDylib found for Path. - return DlFcn.dlopen(Path, Mode); - } - - static void *dlopenHelper(void *Self, const char *Path, int Mode) { - return static_cast<MachOPlatformSupport *>(Self)->jit_dlopen(Path, Mode); - } - - int jit_dlclose(void *Handle) { - JITDylib *JDToClose = nullptr; - - { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - - // Clear any existing error messages. - dlErrorMsgs.erase(std::this_thread::get_id()); - - auto I = JDRefCounts.find(Handle); - if (I != JDRefCounts.end()) { - --I->second; - if (I->second == 0) { - JDRefCounts.erase(I); - JDToClose = static_cast<JITDylib *>(Handle); - } else - return 0; - } - } - - if (JDToClose) { - if (auto Err = deinitialize(*JDToClose)) { - recordError(std::move(Err)); - return -1; - } - return 0; - } - - // Fall through to dlclose if no JITDylib found for Path. - return DlFcn.dlclose(Handle); - } - - static int dlcloseHelper(void *Self, void *Handle) { - return static_cast<MachOPlatformSupport *>(Self)->jit_dlclose(Handle); - } - - void *jit_dlsym(void *Handle, const char *Name) { - JITDylibSearchOrder JITSymSearchOrder; - - // FIXME: RTLD_NEXT, RTLD_SELF not supported. - { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - - // Clear any existing error messages. - dlErrorMsgs.erase(std::this_thread::get_id()); - - if (JDRefCounts.count(Handle)) { - JITSymSearchOrder.push_back( - {static_cast<JITDylib *>(Handle), - JITDylibLookupFlags::MatchExportedSymbolsOnly}); - } else if (Handle == DlFcn.RTLDDefault) { - for (auto &KV : JDRefCounts) - JITSymSearchOrder.push_back( - {static_cast<JITDylib *>(KV.first), - JITDylibLookupFlags::MatchExportedSymbolsOnly}); - } - } - - if (!JITSymSearchOrder.empty()) { - auto MangledName = J.mangleAndIntern(Name); - SymbolLookupSet Syms(MangledName, - SymbolLookupFlags::WeaklyReferencedSymbol); - if (auto Result = J.getExecutionSession().lookup(JITSymSearchOrder, Syms, - LookupKind::DLSym)) { - auto I = Result->find(MangledName); - if (I != Result->end()) - return jitTargetAddressToPointer<void *>(I->second.getAddress()); - } else { - recordError(Result.takeError()); - return 0; - } - } - - // Fall through to dlsym. - return DlFcn.dlsym(Handle, Name); - } - - static void *dlsymHelper(void *Self, void *Handle, const char *Name) { - return static_cast<MachOPlatformSupport *>(Self)->jit_dlsym(Handle, Name); - } - - const char *jit_dlerror() { - { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - auto I = dlErrorMsgs.find(std::this_thread::get_id()); - if (I != dlErrorMsgs.end()) - return I->second->c_str(); - } - return DlFcn.dlerror(); - } - - static const char *dlerrorHelper(void *Self) { - return static_cast<MachOPlatformSupport *>(Self)->jit_dlerror(); - } - - void recordError(Error Err) { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - dlErrorMsgs[std::this_thread::get_id()] = - std::make_unique<std::string>(toString(std::move(Err))); - } - - std::mutex PlatformSupportMutex; - LLJIT &J; - MachOPlatform &MP; - DlFcnValues DlFcn; - ItaniumCXAAtExitSupport AtExitMgr; - DenseMap<void *, unsigned> JDRefCounts; - std::map<std::thread::id, std::unique_ptr<std::string>> dlErrorMsgs; }; } // end anonymous namespace @@ -922,7 +552,8 @@ Error LLJITBuilderState::prepareForConstruction() { } LLVM_DEBUG({ - dbgs() << " JITTargetMachineBuilder is " << JTMB << "\n" + dbgs() << " JITTargetMachineBuilder is " + << JITTargetMachineBuilderPrinter(*JTMB, " ") << " Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No") << "\n" << " DataLayout: "; @@ -944,6 +575,22 @@ Error LLJITBuilderState::prepareForConstruction() { dbgs() << "\n"; }); + // If neither ES nor EPC has been set then create an EPC instance. + if (!ES && !EPC) { + LLVM_DEBUG({ + dbgs() << "ExecutorProcessControl not specified, " + "Creating SelfExecutorProcessControl instance\n"; + }); + if (auto EPCOrErr = SelfExecutorProcessControl::Create()) + EPC = std::move(*EPCOrErr); + else + return EPCOrErr.takeError(); + } else + LLVM_DEBUG({ + dbgs() << "Using explicitly specified ExecutorProcessControl instance " + << EPC.get() << "\n"; + }); + // If the client didn't configure any linker options then auto-configure the // JIT linker. if (!CreateObjectLinkingLayer) { @@ -954,16 +601,9 @@ Error LLJITBuilderState::prepareForConstruction() { JTMB->setRelocationModel(Reloc::PIC_); JTMB->setCodeModel(CodeModel::Small); CreateObjectLinkingLayer = - [TPC = this->TPC]( - ExecutionSession &ES, - const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> { - std::unique_ptr<ObjectLinkingLayer> ObjLinkingLayer; - if (TPC) - ObjLinkingLayer = - std::make_unique<ObjectLinkingLayer>(ES, TPC->getMemMgr()); - else - ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>( - ES, std::make_unique<jitlink::InProcessMemoryManager>()); + [](ExecutionSession &ES, + const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> { + auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES); ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>( ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>())); return std::move(ObjLinkingLayer); @@ -1022,18 +662,18 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs // a new SectionMemoryManager for each object. auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); }; - auto ObjLinkingLayer = + auto Layer = std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) { - ObjLinkingLayer->setOverrideObjectFlagsWithResponsibilityFlags(true); - ObjLinkingLayer->setAutoClaimResponsibilityForObjectSymbols(true); + Layer->setOverrideObjectFlagsWithResponsibilityFlags(true); + Layer->setAutoClaimResponsibilityForObjectSymbols(true); } // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence // errors from some GCC / libstdc++ bots. Remove this conversion (i.e. // just return ObjLinkingLayer) once those bots are upgraded. - return std::unique_ptr<ObjectLayer>(std::move(ObjLinkingLayer)); + return std::unique_ptr<ObjectLayer>(std::move(Layer)); } Expected<std::unique_ptr<IRCompileLayer::IRCompiler>> @@ -1057,11 +697,25 @@ LLJIT::createCompileFunction(LLJITBuilderState &S, } LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) - : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), Main(), - DL(""), TT(S.JTMB->getTargetTriple()) { + : DL(""), TT(S.JTMB->getTargetTriple()) { ErrorAsOutParameter _(&Err); + assert(!(S.EPC && S.ES) && "EPC and ES should not both be set"); + + if (S.EPC) { + ES = std::make_unique<ExecutionSession>(std::move(S.EPC)); + } else if (S.ES) + ES = std::move(S.ES); + else { + if (auto EPC = SelfExecutorProcessControl::Create()) { + ES = std::make_unique<ExecutionSession>(std::move(*EPC)); + } else { + Err = EPC.takeError(); + return; + } + } + if (auto MainOrErr = this->ES->createJITDylib("main")) Main = &*MainOrErr; else { @@ -1104,20 +758,16 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) InitHelperTransformLayer->setCloneToNewContextOnEmit(true); CompileThreads = std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads)); - ES->setDispatchMaterialization( - [this](std::unique_ptr<MaterializationUnit> MU, - std::unique_ptr<MaterializationResponsibility> MR) { - // FIXME: We should be able to use move-capture here, but ThreadPool's - // AsyncTaskTys are std::functions rather than unique_functions - // (because MSVC's std::packaged_tasks don't support move-only types). - // Fix this when all the above gets sorted out. - CompileThreads->async( - [UnownedMU = MU.release(), UnownedMR = MR.release()]() mutable { - std::unique_ptr<MaterializationUnit> MU(UnownedMU); - std::unique_ptr<MaterializationResponsibility> MR(UnownedMR); - MU->materialize(std::move(MR)); - }); - }); + ES->setDispatchTask([this](std::unique_ptr<Task> T) { + // FIXME: We should be able to use move-capture here, but ThreadPool's + // AsyncTaskTys are std::functions rather than unique_functions + // (because MSVC's std::packaged_tasks don't support move-only types). + // Fix this when all the above gets sorted out. + CompileThreads->async([UnownedT = T.release()]() mutable { + std::unique_ptr<Task> T(UnownedT); + T->run(); + }); + }); } if (S.SetUpPlatform) @@ -1155,12 +805,10 @@ void setUpGenericLLVMIRPlatform(LLJIT &J) { J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J)); } -Error setUpMachOPlatform(LLJIT &J) { - LLVM_DEBUG({ dbgs() << "Setting up MachOPlatform support for LLJIT\n"; }); - auto MP = MachOPlatformSupport::Create(J, J.getMainJITDylib()); - if (!MP) - return MP.takeError(); - J.setPlatformSupport(std::move(*MP)); +Error setUpInactivePlatform(LLJIT &J) { + LLVM_DEBUG( + { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; }); + J.setPlatformSupport(std::make_unique<InactivePlatformSupport>()); return Error::success(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp index 5e27e343d23b..20dfba23bf10 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -155,6 +155,8 @@ void BasicIRLayerMaterializationUnit::materialize( });); } +char ObjectLayer::ID; + ObjectLayer::ObjectLayer(ExecutionSession &ES) : ES(ES) {} ObjectLayer::~ObjectLayer() {} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 17b9465a0541..66ef835dc34d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -9,154 +9,195 @@ #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" #define DEBUG_TYPE "orc" -namespace { +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; -struct objc_class; -struct objc_image_info; -struct objc_object; -struct objc_selector; +namespace { -using Class = objc_class *; -using id = objc_object *; -using SEL = objc_selector *; +class MachOHeaderMaterializationUnit : public MaterializationUnit { +public: + MachOHeaderMaterializationUnit(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) + : MaterializationUnit(createHeaderSymbols(MOP, HeaderStartSymbol), + HeaderStartSymbol), + MOP(MOP) {} + + StringRef getName() const override { return "MachOHeaderMU"; } + + void materialize(std::unique_ptr<MaterializationResponsibility> R) override { + unsigned PointerSize; + support::endianness Endianness; + const auto &TT = + MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + + switch (TT.getArch()) { + case Triple::aarch64: + case Triple::x86_64: + PointerSize = 8; + Endianness = support::endianness::little; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } -using ObjCMsgSendTy = id (*)(id, SEL, ...); -using ObjCReadClassPairTy = Class (*)(Class, const objc_image_info *); -using SelRegisterNameTy = SEL (*)(const char *); + auto G = std::make_unique<jitlink::LinkGraph>( + "<MachOHeaderMU>", TT, PointerSize, Endianness, + jitlink::getGenericEdgeKindName); + auto &HeaderSection = G->createSection("__header", sys::Memory::MF_READ); + auto &HeaderBlock = createHeaderBlock(*G, HeaderSection); + + // Init symbol is header-start symbol. + G->addDefinedSymbol(HeaderBlock, 0, *R->getInitializerSymbol(), + HeaderBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Default, false, true); + for (auto &HS : AdditionalHeaderSymbols) + G->addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, + HeaderBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Default, false, true); + + MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + } -enum class ObjCRegistrationAPI { Uninitialized, Unavailable, Initialized }; + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} -ObjCRegistrationAPI ObjCRegistrationAPIState = - ObjCRegistrationAPI::Uninitialized; -ObjCMsgSendTy objc_msgSend = nullptr; -ObjCReadClassPairTy objc_readClassPair = nullptr; -SelRegisterNameTy sel_registerName = nullptr; +private: + struct HeaderSymbol { + const char *Name; + uint64_t Offset; + }; -} // end anonymous namespace + static constexpr HeaderSymbol AdditionalHeaderSymbols[] = { + {"___mh_executable_header", 0}}; + + static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + MachO::mach_header_64 Hdr; + Hdr.magic = MachO::MH_MAGIC_64; + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + Hdr.cputype = MachO::CPU_TYPE_ARM64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; + break; + case Triple::x86_64: + Hdr.cputype = MachO::CPU_TYPE_X86_64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + Hdr.filetype = MachO::MH_DYLIB; // Custom file type? + Hdr.ncmds = 0; + Hdr.sizeofcmds = 0; + Hdr.flags = 0; + Hdr.reserved = 0; -namespace llvm { -namespace orc { + if (G.getEndianness() != support::endian::system_endianness()) + MachO::swapStruct(Hdr); -template <typename FnTy> -static Error setUpObjCRegAPIFunc(FnTy &Target, sys::DynamicLibrary &LibObjC, - const char *Name) { - if (void *Addr = LibObjC.getAddressOfSymbol(Name)) - Target = reinterpret_cast<FnTy>(Addr); - else - return make_error<StringError>( - (Twine("Could not find address for ") + Name).str(), - inconvertibleErrorCode()); - return Error::success(); -} + auto HeaderContent = G.allocateString( + StringRef(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); -Error enableObjCRegistration(const char *PathToLibObjC) { - // If we've already tried to initialize then just bail out. - if (ObjCRegistrationAPIState != ObjCRegistrationAPI::Uninitialized) - return Error::success(); + return G.createContentBlock(HeaderSection, HeaderContent, 0, 8, 0); + } - ObjCRegistrationAPIState = ObjCRegistrationAPI::Unavailable; + static SymbolFlagsMap + createHeaderSymbols(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) { + SymbolFlagsMap HeaderSymbolFlags; - std::string ErrMsg; - auto LibObjC = - sys::DynamicLibrary::getPermanentLibrary(PathToLibObjC, &ErrMsg); + HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; + for (auto &HS : AdditionalHeaderSymbols) + HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = + JITSymbolFlags::Exported; - if (!LibObjC.isValid()) - return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); + return HeaderSymbolFlags; + } - if (auto Err = setUpObjCRegAPIFunc(objc_msgSend, LibObjC, "objc_msgSend")) - return Err; - if (auto Err = setUpObjCRegAPIFunc(objc_readClassPair, LibObjC, - "objc_readClassPair")) - return Err; - if (auto Err = - setUpObjCRegAPIFunc(sel_registerName, LibObjC, "sel_registerName")) - return Err; + MachOPlatform &MOP; +}; - ObjCRegistrationAPIState = ObjCRegistrationAPI::Initialized; - return Error::success(); -} +constexpr MachOHeaderMaterializationUnit::HeaderSymbol + MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[]; -bool objCRegistrationEnabled() { - return ObjCRegistrationAPIState == ObjCRegistrationAPI::Initialized; -} +StringRef EHFrameSectionName = "__TEXT,__eh_frame"; +StringRef ModInitFuncSectionName = "__DATA,__mod_init_func"; +StringRef ObjCClassListSectionName = "__DATA,__objc_classlist"; +StringRef ObjCImageInfoSectionName = "__DATA,__objc_image_info"; +StringRef ObjCSelRefsSectionName = "__DATA,__objc_selrefs"; +StringRef Swift5ProtoSectionName = "__TEXT,__swift5_proto"; +StringRef Swift5ProtosSectionName = "__TEXT,__swift5_protos"; +StringRef ThreadBSSSectionName = "__DATA,__thread_bss"; +StringRef ThreadDataSectionName = "__DATA,__thread_data"; +StringRef ThreadVarsSectionName = "__DATA,__thread_vars"; -void MachOJITDylibInitializers::runModInits() const { - for (const auto &ModInit : ModInitSections) { - for (uint64_t I = 0; I != ModInit.NumPtrs; ++I) { - auto *InitializerAddr = jitTargetAddressToPointer<uintptr_t *>( - ModInit.Address + (I * sizeof(uintptr_t))); - auto *Initializer = - jitTargetAddressToFunction<void (*)()>(*InitializerAddr); - Initializer(); - } - } -} +StringRef InitSectionNames[] = { + ModInitFuncSectionName, ObjCSelRefsSectionName, ObjCClassListSectionName, + Swift5ProtosSectionName, Swift5ProtoSectionName}; -void MachOJITDylibInitializers::registerObjCSelectors() const { - assert(objCRegistrationEnabled() && "ObjC registration not enabled."); +} // end anonymous namespace - for (const auto &ObjCSelRefs : ObjCSelRefsSections) { - for (uint64_t I = 0; I != ObjCSelRefs.NumPtrs; ++I) { - auto SelEntryAddr = ObjCSelRefs.Address + (I * sizeof(uintptr_t)); - const auto *SelName = - *jitTargetAddressToPointer<const char **>(SelEntryAddr); - auto Sel = sel_registerName(SelName); - *jitTargetAddressToPointer<SEL *>(SelEntryAddr) = Sel; - } - } -} +namespace llvm { +namespace orc { -Error MachOJITDylibInitializers::registerObjCClasses() const { - assert(objCRegistrationEnabled() && "ObjC registration not enabled."); +Expected<std::unique_ptr<MachOPlatform>> +MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + Optional<SymbolAliasMap> RuntimeAliases) { - struct ObjCClassCompiled { - void *Metaclass; - void *Parent; - void *Cache1; - void *Cache2; - void *Data; - }; + auto &EPC = ES.getExecutorProcessControl(); - auto *ImageInfo = - jitTargetAddressToPointer<const objc_image_info *>(ObjCImageInfoAddr); - auto ClassSelector = sel_registerName("class"); - - for (const auto &ObjCClassList : ObjCClassListSections) { - for (uint64_t I = 0; I != ObjCClassList.NumPtrs; ++I) { - auto ClassPtrAddr = ObjCClassList.Address + (I * sizeof(uintptr_t)); - auto Cls = *jitTargetAddressToPointer<Class *>(ClassPtrAddr); - auto *ClassCompiled = - *jitTargetAddressToPointer<ObjCClassCompiled **>(ClassPtrAddr); - objc_msgSend(reinterpret_cast<id>(ClassCompiled->Parent), ClassSelector); - auto Registered = objc_readClassPair(Cls, ImageInfo); - - // FIXME: Improve diagnostic by reporting the failed class's name. - if (Registered != Cls) - return make_error<StringError>("Unable to register Objective-C class", - inconvertibleErrorCode()); - } - } - return Error::success(); -} + // If the target is not supported then bail out immediately. + if (!supportedTarget(EPC.getTargetTriple())) + return make_error<StringError>("Unsupported MachOPlatform triple: " + + EPC.getTargetTriple().str(), + inconvertibleErrorCode()); -MachOPlatform::MachOPlatform( - ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - std::unique_ptr<MemoryBuffer> StandardSymbolsObject) - : ES(ES), ObjLinkingLayer(ObjLinkingLayer), - StandardSymbolsObject(std::move(StandardSymbolsObject)) { - ObjLinkingLayer.addPlugin(std::make_unique<InitScraperPlugin>(*this)); + // Create default aliases if the caller didn't supply any. + if (!RuntimeAliases) + RuntimeAliases = standardPlatformAliases(ES); + + // Define the aliases. + if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases)))) + return std::move(Err); + + // Add JIT-dispatch function support symbols. + if (auto Err = PlatformJD.define(absoluteSymbols( + {{ES.intern("___orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunctionAddress.getValue(), + JITSymbolFlags::Exported}}, + {ES.intern("___orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContextAddress.getValue(), + JITSymbolFlags::Exported}}}))) + return std::move(Err); + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load( + ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple()); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + // Create the instance. + Error Err = Error::success(); + auto P = std::unique_ptr<MachOPlatform>( + new MachOPlatform(ES, ObjLinkingLayer, PlatformJD, + std::move(*OrcRuntimeArchiveGenerator), Err)); + if (Err) + return std::move(Err); + return std::move(P); } Error MachOPlatform::setupJITDylib(JITDylib &JD) { - auto ObjBuffer = MemoryBuffer::getMemBuffer( - StandardSymbolsObject->getMemBufferRef(), false); - return ObjLinkingLayer.add(JD, std::move(ObjBuffer)); + return JD.define(std::make_unique<MachOHeaderMaterializationUnit>( + *this, MachOHeaderStartSymbol)); } Error MachOPlatform::notifyAdding(ResourceTracker &RT, @@ -179,58 +220,127 @@ Error MachOPlatform::notifyRemoving(ResourceTracker &RT) { llvm_unreachable("Not supported yet"); } -Expected<MachOPlatform::InitializerSequence> -MachOPlatform::getInitializerSequence(JITDylib &JD) { +static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, + ArrayRef<std::pair<const char *, const char *>> AL) { + for (auto &KV : AL) { + auto AliasName = ES.intern(KV.first); + assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map"); + Aliases[std::move(AliasName)] = {ES.intern(KV.second), + JITSymbolFlags::Exported}; + } +} - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Building initializer sequence for " - << JD.getName() << "\n"; - }); +SymbolAliasMap MachOPlatform::standardPlatformAliases(ExecutionSession &ES) { + SymbolAliasMap Aliases; + addAliases(ES, Aliases, requiredCXXAliases()); + addAliases(ES, Aliases, standardRuntimeUtilityAliases()); + return Aliases; +} - std::vector<JITDylibSP> DFSLinkOrder; +ArrayRef<std::pair<const char *, const char *>> +MachOPlatform::requiredCXXAliases() { + static const std::pair<const char *, const char *> RequiredCXXAliases[] = { + {"___cxa_atexit", "___orc_rt_macho_cxa_atexit"}}; - while (true) { + return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases); +} - DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; +ArrayRef<std::pair<const char *, const char *>> +MachOPlatform::standardRuntimeUtilityAliases() { + static const std::pair<const char *, const char *> + StandardRuntimeUtilityAliases[] = { + {"___orc_rt_run_program", "___orc_rt_macho_run_program"}, + {"___orc_rt_log_error", "___orc_rt_log_error_to_stderr"}}; - ES.runSessionLocked([&]() { - DFSLinkOrder = JD.getDFSLinkOrder(); + return ArrayRef<std::pair<const char *, const char *>>( + StandardRuntimeUtilityAliases); +} - for (auto &InitJD : DFSLinkOrder) { - auto RISItr = RegisteredInitSymbols.find(InitJD.get()); - if (RISItr != RegisteredInitSymbols.end()) { - NewInitSymbols[InitJD.get()] = std::move(RISItr->second); - RegisteredInitSymbols.erase(RISItr); - } - } - }); +bool MachOPlatform::isInitializerSection(StringRef SegName, + StringRef SectName) { + for (auto &Name : InitSectionNames) { + if (Name.startswith(SegName) && Name.substr(7) == SectName) + return true; + } + return false; +} - if (NewInitSymbols.empty()) - break; +bool MachOPlatform::supportedTarget(const Triple &TT) { + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } +} - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Issuing lookups for new init symbols: " - "(lookup may require multiple rounds)\n"; - for (auto &KV : NewInitSymbols) - dbgs() << " \"" << KV.first->getName() << "\": " << KV.second << "\n"; - }); +MachOPlatform::MachOPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, + std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), + MachOHeaderStartSymbol(ES.intern("___dso_handle")) { + ErrorAsOutParameter _(&Err); + + ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this)); - // Outside the lock, issue the lookup. - if (auto R = lookupInitSymbols(JD.getExecutionSession(), NewInitSymbols)) - ; // Nothing to do in the success case. - else - return R.takeError(); + PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); + + // PlatformJD hasn't been 'set-up' by the platform yet (since we're creating + // the platform now), so set it up. + if (auto E2 = setupJITDylib(PlatformJD)) { + Err = std::move(E2); + return; } - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Init symbol lookup complete, building init " - "sequence\n"; - }); + RegisteredInitSymbols[&PlatformJD].add( + MachOHeaderStartSymbol, SymbolLookupFlags::WeaklyReferencedSymbol); - // Lock again to collect the initializers. - InitializerSequence FullInitSeq; + // Associate wrapper function tags with JIT-side function implementations. + if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { + Err = std::move(E2); + return; + } + + // Lookup addresses of runtime functions callable by the platform, + // call the platform bootstrap function to initialize the platform-state + // object in the executor. + if (auto E2 = bootstrapMachORuntime(PlatformJD)) { + Err = std::move(E2); + return; + } +} + +Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { + ExecutionSession::JITDispatchHandlerAssociationMap WFs; + + using GetInitializersSPSSig = + SPSExpected<SPSMachOJITDylibInitializerSequence>(SPSString); + WFs[ES.intern("___orc_rt_macho_get_initializers_tag")] = + ES.wrapAsyncWithSPS<GetInitializersSPSSig>( + this, &MachOPlatform::rt_getInitializers); + + using GetDeinitializersSPSSig = + SPSExpected<SPSMachOJITDylibDeinitializerSequence>(SPSExecutorAddress); + WFs[ES.intern("___orc_rt_macho_get_deinitializers_tag")] = + ES.wrapAsyncWithSPS<GetDeinitializersSPSSig>( + this, &MachOPlatform::rt_getDeinitializers); + + using LookupSymbolSPSSig = + SPSExpected<SPSExecutorAddress>(SPSExecutorAddress, SPSString); + WFs[ES.intern("___orc_rt_macho_symbol_lookup_tag")] = + ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this, + &MachOPlatform::rt_lookupSymbol); + + return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs)); +} + +void MachOPlatform::getInitializersBuildSequencePhase( + SendInitializerSequenceFn SendResult, JITDylib &JD, + std::vector<JITDylibSP> DFSLinkOrder) { + MachOJITDylibInitializerSequence FullInitSeq; { - std::lock_guard<std::mutex> Lock(InitSeqsMutex); + std::lock_guard<std::mutex> Lock(PlatformMutex); for (auto &InitJD : reverse(DFSLinkOrder)) { LLVM_DEBUG({ dbgs() << "MachOPlatform: Appending inits for \"" << InitJD->getName() @@ -238,176 +348,451 @@ MachOPlatform::getInitializerSequence(JITDylib &JD) { }); auto ISItr = InitSeqs.find(InitJD.get()); if (ISItr != InitSeqs.end()) { - FullInitSeq.emplace_back(InitJD.get(), std::move(ISItr->second)); + FullInitSeq.emplace_back(std::move(ISItr->second)); InitSeqs.erase(ISItr); } } } - return FullInitSeq; + SendResult(std::move(FullInitSeq)); +} + +void MachOPlatform::getInitializersLookupPhase( + SendInitializerSequenceFn SendResult, JITDylib &JD) { + + auto DFSLinkOrder = JD.getDFSLinkOrder(); + DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; + ES.runSessionLocked([&]() { + for (auto &InitJD : DFSLinkOrder) { + auto RISItr = RegisteredInitSymbols.find(InitJD.get()); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[InitJD.get()] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } + } + }); + + // If there are no further init symbols to look up then move on to the next + // phase. + if (NewInitSymbols.empty()) { + getInitializersBuildSequencePhase(std::move(SendResult), JD, + std::move(DFSLinkOrder)); + return; + } + + // Otherwise issue a lookup and re-run this phase when it completes. + lookupInitSymbolsAsync( + [this, SendResult = std::move(SendResult), &JD](Error Err) mutable { + if (Err) + SendResult(std::move(Err)); + else + getInitializersLookupPhase(std::move(SendResult), JD); + }, + ES, std::move(NewInitSymbols)); } -Expected<MachOPlatform::DeinitializerSequence> -MachOPlatform::getDeinitializerSequence(JITDylib &JD) { - std::vector<JITDylibSP> DFSLinkOrder = JD.getDFSLinkOrder(); +void MachOPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, + StringRef JDName) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_getInitializers(\"" << JDName << "\")\n"; + }); + + JITDylib *JD = ES.getJITDylibByName(JDName); + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No such JITDylib \"" << JDName << "\". Sending error.\n"; + }); + SendResult(make_error<StringError>("No JITDylib named " + JDName, + inconvertibleErrorCode())); + return; + } + + getInitializersLookupPhase(std::move(SendResult), *JD); +} + +void MachOPlatform::rt_getDeinitializers(SendDeinitializerSequenceFn SendResult, + ExecutorAddress Handle) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_getDeinitializers(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; - DeinitializerSequence FullDeinitSeq; { - std::lock_guard<std::mutex> Lock(InitSeqsMutex); - for (auto &DeinitJD : DFSLinkOrder) { - FullDeinitSeq.emplace_back(DeinitJD.get(), MachOJITDylibDeinitializers()); + std::lock_guard<std::mutex> Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle.getValue()); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error<StringError>("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + SendResult(MachOJITDylibDeinitializerSequence()); +} + +void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, + ExecutorAddress Handle, + StringRef SymbolName) { + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_lookupSymbol(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; + + { + std::lock_guard<std::mutex> Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle.getValue()); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error<StringError>("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + // Use functor class to work around XL build compiler issue on AIX. + class RtLookupNotifyComplete { + public: + RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult) + : SendResult(std::move(SendResult)) {} + void operator()(Expected<SymbolMap> Result) { + if (Result) { + assert(Result->size() == 1 && "Unexpected result map count"); + SendResult(ExecutorAddress(Result->begin()->second.getAddress())); + } else { + SendResult(Result.takeError()); + } } + + private: + SendSymbolAddressFn SendResult; + }; + + // FIXME: Proper mangling. + auto MangledName = ("_" + SymbolName).str(); + ES.lookup( + LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, + SymbolLookupSet(ES.intern(MangledName)), SymbolState::Ready, + RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister); +} + +Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) { + + std::pair<const char *, ExecutorAddress *> Symbols[] = { + {"___orc_rt_macho_platform_bootstrap", &orc_rt_macho_platform_bootstrap}, + {"___orc_rt_macho_platform_shutdown", &orc_rt_macho_platform_shutdown}, + {"___orc_rt_macho_register_object_sections", + &orc_rt_macho_register_object_sections}, + {"___orc_rt_macho_create_pthread_key", &orc_rt_macho_create_pthread_key}}; + + SymbolLookupSet RuntimeSymbols; + std::vector<std::pair<SymbolStringPtr, ExecutorAddress *>> AddrsToRecord; + for (const auto &KV : Symbols) { + auto Name = ES.intern(KV.first); + RuntimeSymbols.add(Name); + AddrsToRecord.push_back({std::move(Name), KV.second}); } - return FullDeinitSeq; + auto RuntimeSymbolAddrs = ES.lookup( + {{&PlatformJD, JITDylibLookupFlags::MatchAllSymbols}}, RuntimeSymbols); + if (!RuntimeSymbolAddrs) + return RuntimeSymbolAddrs.takeError(); + + for (const auto &KV : AddrsToRecord) { + auto &Name = KV.first; + assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?"); + KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress()); + } + + if (auto Err = + ES.callSPSWrapper<void()>(orc_rt_macho_platform_bootstrap.getValue())) + return Err; + + // FIXME: Ordering is fuzzy here. We're probably best off saying + // "behavior is undefined if code that uses the runtime is added before + // the platform constructor returns", then move all this to the constructor. + RuntimeBootstrapped = true; + std::vector<MachOPerObjectSectionsToRegister> DeferredPOSRs; + { + std::lock_guard<std::mutex> Lock(PlatformMutex); + DeferredPOSRs = std::move(BootstrapPOSRs); + } + + for (auto &D : DeferredPOSRs) + if (auto Err = registerPerObjectSections(D)) + return Err; + + return Error::success(); } -void MachOPlatform::registerInitInfo( - JITDylib &JD, JITTargetAddress ObjCImageInfoAddr, - MachOJITDylibInitializers::SectionExtent ModInits, - MachOJITDylibInitializers::SectionExtent ObjCSelRefs, - MachOJITDylibInitializers::SectionExtent ObjCClassList) { - std::lock_guard<std::mutex> Lock(InitSeqsMutex); +Error MachOPlatform::registerInitInfo( + JITDylib &JD, ExecutorAddress ObjCImageInfoAddr, + ArrayRef<jitlink::Section *> InitSections) { - auto &InitSeq = InitSeqs[&JD]; + std::unique_lock<std::mutex> Lock(PlatformMutex); - InitSeq.setObjCImageInfoAddr(ObjCImageInfoAddr); + MachOJITDylibInitializers *InitSeq = nullptr; + { + auto I = InitSeqs.find(&JD); + if (I == InitSeqs.end()) { + // If there's no init sequence entry yet then we need to look up the + // header symbol to force creation of one. + Lock.unlock(); + + auto SearchOrder = + JD.withLinkOrderDo([](const JITDylibSearchOrder &SO) { return SO; }); + if (auto Err = ES.lookup(SearchOrder, MachOHeaderStartSymbol).takeError()) + return Err; + + Lock.lock(); + I = InitSeqs.find(&JD); + assert(I != InitSeqs.end() && + "Entry missing after header symbol lookup?"); + } + InitSeq = &I->second; + } - if (ModInits.Address) - InitSeq.addModInitsSection(std::move(ModInits)); + InitSeq->ObjCImageInfoAddress = ObjCImageInfoAddr; - if (ObjCSelRefs.Address) - InitSeq.addObjCSelRefsSection(std::move(ObjCSelRefs)); + for (auto *Sec : InitSections) { + // FIXME: Avoid copy here. + jitlink::SectionRange R(*Sec); + InitSeq->InitSections[Sec->getName()].push_back( + {ExecutorAddress(R.getStart()), ExecutorAddress(R.getEnd())}); + } - if (ObjCClassList.Address) - InitSeq.addObjCClassListSection(std::move(ObjCClassList)); + return Error::success(); } -static Expected<MachOJITDylibInitializers::SectionExtent> -getSectionExtent(jitlink::LinkGraph &G, StringRef SectionName) { - auto *Sec = G.findSectionByName(SectionName); - if (!Sec) - return MachOJITDylibInitializers::SectionExtent(); - jitlink::SectionRange R(*Sec); - if (R.getSize() % G.getPointerSize() != 0) - return make_error<StringError>(SectionName + " section size is not a " - "multiple of the pointer size", +Error MachOPlatform::registerPerObjectSections( + const MachOPerObjectSectionsToRegister &POSR) { + + if (!orc_rt_macho_register_object_sections) + return make_error<StringError>("Attempting to register per-object " + "sections, but runtime support has not " + "been loaded yet", inconvertibleErrorCode()); - return MachOJITDylibInitializers::SectionExtent( - R.getStart(), R.getSize() / G.getPointerSize()); + + Error ErrResult = Error::success(); + if (auto Err = ES.callSPSWrapper<shared::SPSError( + SPSMachOPerObjectSectionsToRegister)>( + orc_rt_macho_register_object_sections.getValue(), ErrResult, POSR)) + return Err; + return ErrResult; } -void MachOPlatform::InitScraperPlugin::modifyPassConfig( - MaterializationResponsibility &MR, const Triple &TT, +Expected<uint64_t> MachOPlatform::createPThreadKey() { + if (!orc_rt_macho_create_pthread_key) + return make_error<StringError>( + "Attempting to create pthread key in target, but runtime support has " + "not been loaded yet", + inconvertibleErrorCode()); + + Expected<uint64_t> Result(0); + if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>( + orc_rt_macho_create_pthread_key.getValue(), Result)) + return std::move(Err); + return Result; +} + +void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( + MaterializationResponsibility &MR, jitlink::LinkGraph &LG, jitlink::PassConfiguration &Config) { - if (!MR.getInitializerSymbol()) + // If the initializer symbol is the MachOHeader start symbol then just add + // the macho header support passes. + if (MR.getInitializerSymbol() == MP.MachOHeaderStartSymbol) { + addMachOHeaderSupportPasses(MR, Config); + // The header materialization unit doesn't require any other support, so we + // can bail out early. return; + } - Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) -> Error { - JITLinkSymbolVector InitSectionSymbols; - preserveInitSectionIfPresent(InitSectionSymbols, G, "__mod_init_func"); - preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_selrefs"); - preserveInitSectionIfPresent(InitSectionSymbols, G, "__objc_classlist"); + // If the object contains initializers then add passes to record them. + if (MR.getInitializerSymbol()) + addInitializerSupportPasses(MR, Config); - if (!InitSectionSymbols.empty()) { - std::lock_guard<std::mutex> Lock(InitScraperMutex); - InitSymbolDeps[&MR] = std::move(InitSectionSymbols); - } + // Add passes for eh-frame and TLV support. + addEHAndTLVSupportPasses(MR, Config); +} - if (auto Err = processObjCImageInfo(G, MR)) +ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap +MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies( + MaterializationResponsibility &MR) { + std::lock_guard<std::mutex> Lock(PluginMutex); + auto I = InitSymbolDeps.find(&MR); + if (I != InitSymbolDeps.end()) { + SyntheticSymbolDependenciesMap Result; + Result[MR.getInitializerSymbol()] = std::move(I->second); + InitSymbolDeps.erase(&MR); + return Result; + } + return SyntheticSymbolDependenciesMap(); +} + +void MachOPlatform::MachOPlatformPlugin::addInitializerSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + /// Preserve init sections. + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { + if (auto Err = preserveInitSections(G, MR)) return Err; + return processObjCImageInfo(G, MR); + }); + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerInitSections(G, JD); + }); +} + +void MachOPlatform::MachOPlatformPlugin::addMachOHeaderSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + Config.PostAllocationPasses.push_back([this, &JD = MR.getTargetJITDylib()]( + jitlink::LinkGraph &G) -> Error { + auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { + return Sym->getName() == *MP.MachOHeaderStartSymbol; + }); + assert(I != G.defined_symbols().end() && + "Missing MachO header start symbol"); + { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + JITTargetAddress HeaderAddr = (*I)->getAddress(); + MP.HeaderAddrToJITDylib[HeaderAddr] = &JD; + assert(!MP.InitSeqs.count(&JD) && "InitSeq entry for JD already exists"); + MP.InitSeqs.insert( + std::make_pair(&JD, MachOJITDylibInitializers( + JD.getName(), ExecutorAddress(HeaderAddr)))); + } return Error::success(); }); +} + +void MachOPlatform::MachOPlatformPlugin::addEHAndTLVSupportPasses( + MaterializationResponsibility &MR, jitlink::PassConfiguration &Config) { + + // Insert TLV lowering at the start of the PostPrunePasses, since we want + // it to run before GOT/PLT lowering. + Config.PostPrunePasses.insert( + Config.PostPrunePasses.begin(), + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return fixTLVSectionsAndEdges(G, JD); + }); - Config.PostFixupPasses.push_back([this, &JD = MR.getTargetJITDylib()]( - jitlink::LinkGraph &G) -> Error { - MachOJITDylibInitializers::SectionExtent ModInits, ObjCSelRefs, - ObjCClassList; + // Add a pass to register the final addresses of the eh-frame and TLV sections + // with the runtime. + Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error { + MachOPerObjectSectionsToRegister POSR; - JITTargetAddress ObjCImageInfoAddr = 0; - if (auto *ObjCImageInfoSec = G.findSectionByName("__objc_image_info")) { - if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) - ObjCImageInfoAddr = Addr; + if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { + jitlink::SectionRange R(*EHFrameSection); + if (!R.empty()) + POSR.EHFrameSection = {ExecutorAddress(R.getStart()), + ExecutorAddress(R.getEnd())}; } - // Record __mod_init_func. - if (auto ModInitsOrErr = getSectionExtent(G, "__mod_init_func")) - ModInits = std::move(*ModInitsOrErr); - else - return ModInitsOrErr.takeError(); - - // Record __objc_selrefs. - if (auto ObjCSelRefsOrErr = getSectionExtent(G, "__objc_selrefs")) - ObjCSelRefs = std::move(*ObjCSelRefsOrErr); - else - return ObjCSelRefsOrErr.takeError(); - - // Record __objc_classlist. - if (auto ObjCClassListOrErr = getSectionExtent(G, "__objc_classlist")) - ObjCClassList = std::move(*ObjCClassListOrErr); - else - return ObjCClassListOrErr.takeError(); - - // Dump the scraped inits. - LLVM_DEBUG({ - dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; - dbgs() << " __objc_selrefs: "; - if (ObjCSelRefs.NumPtrs) - dbgs() << ObjCSelRefs.NumPtrs << " pointer(s) at " - << formatv("{0:x16}", ObjCSelRefs.Address) << "\n"; + // Get a pointer to the thread data section if there is one. It will be used + // below. + jitlink::Section *ThreadDataSection = + G.findSectionByName(ThreadDataSectionName); + + // Handle thread BSS section if there is one. + if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) { + // If there's already a thread data section in this graph then merge the + // thread BSS section content into it, otherwise just treat the thread + // BSS section as the thread data section. + if (ThreadDataSection) + G.mergeSections(*ThreadDataSection, *ThreadBSSSection); else - dbgs() << "none\n"; + ThreadDataSection = ThreadBSSSection; + } - dbgs() << " __objc_classlist: "; - if (ObjCClassList.NumPtrs) - dbgs() << ObjCClassList.NumPtrs << " pointer(s) at " - << formatv("{0:x16}", ObjCClassList.Address) << "\n"; - else - dbgs() << "none\n"; + // Having merged thread BSS (if present) and thread data (if present), + // record the resulting section range. + if (ThreadDataSection) { + jitlink::SectionRange R(*ThreadDataSection); + if (!R.empty()) + POSR.ThreadDataSection = {ExecutorAddress(R.getStart()), + ExecutorAddress(R.getEnd())}; + } - dbgs() << " __mod_init_func: "; - if (ModInits.NumPtrs) - dbgs() << ModInits.NumPtrs << " pointer(s) at " - << formatv("{0:x16}", ModInits.Address) << "\n"; - else - dbgs() << "none\n"; - }); + if (POSR.EHFrameSection.StartAddress || + POSR.ThreadDataSection.StartAddress) { - MP.registerInitInfo(JD, ObjCImageInfoAddr, std::move(ModInits), - std::move(ObjCSelRefs), std::move(ObjCClassList)); + // If we're still bootstrapping the runtime then just record this + // frame for now. + if (!MP.RuntimeBootstrapped) { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + MP.BootstrapPOSRs.push_back(POSR); + return Error::success(); + } + + // Otherwise register it immediately. + if (auto Err = MP.registerPerObjectSections(POSR)) + return Err; + } return Error::success(); }); } -ObjectLinkingLayer::Plugin::LocalDependenciesMap -MachOPlatform::InitScraperPlugin::getSyntheticSymbolLocalDependencies( - MaterializationResponsibility &MR) { - std::lock_guard<std::mutex> Lock(InitScraperMutex); - auto I = InitSymbolDeps.find(&MR); - if (I != InitSymbolDeps.end()) { - LocalDependenciesMap Result; - Result[MR.getInitializerSymbol()] = std::move(I->second); - InitSymbolDeps.erase(&MR); - return Result; +Error MachOPlatform::MachOPlatformPlugin::preserveInitSections( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + + JITLinkSymbolSet InitSectionSymbols; + for (auto &InitSectionName : InitSectionNames) { + // Skip non-init sections. + auto *InitSection = G.findSectionByName(InitSectionName); + if (!InitSection) + continue; + + // Make a pass over live symbols in the section: those blocks are already + // preserved. + DenseSet<jitlink::Block *> AlreadyLiveBlocks; + for (auto &Sym : InitSection->symbols()) { + auto &B = Sym->getBlock(); + if (Sym->isLive() && Sym->getOffset() == 0 && + Sym->getSize() == B.getSize() && !AlreadyLiveBlocks.count(&B)) { + InitSectionSymbols.insert(Sym); + AlreadyLiveBlocks.insert(&B); + } + } + + // Add anonymous symbols to preserve any not-already-preserved blocks. + for (auto *B : InitSection->blocks()) + if (!AlreadyLiveBlocks.count(B)) + InitSectionSymbols.insert( + &G.addAnonymousSymbol(*B, 0, B->getSize(), false, true)); } - return LocalDependenciesMap(); -} -void MachOPlatform::InitScraperPlugin::preserveInitSectionIfPresent( - JITLinkSymbolVector &Symbols, jitlink::LinkGraph &G, - StringRef SectionName) { - if (auto *Sec = G.findSectionByName(SectionName)) { - auto SecBlocks = Sec->blocks(); - if (!llvm::empty(SecBlocks)) - Symbols.push_back( - &G.addAnonymousSymbol(**SecBlocks.begin(), 0, 0, false, true)); + if (!InitSectionSymbols.empty()) { + std::lock_guard<std::mutex> Lock(PluginMutex); + InitSymbolDeps[&MR] = std::move(InitSectionSymbols); } + + return Error::success(); } -Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( +Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( jitlink::LinkGraph &G, MaterializationResponsibility &MR) { // If there's an ObjC imagine info then either @@ -416,7 +801,7 @@ Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( // OR // (2) We already have a recorded __objc_imageinfo for this JITDylib, // in which case we just verify it. - auto *ObjCImageInfo = G.findSectionByName("__objc_imageinfo"); + auto *ObjCImageInfo = G.findSectionByName(ObjCImageInfoSectionName); if (!ObjCImageInfo) return Error::success(); @@ -424,15 +809,15 @@ Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( // Check that the section is not empty if present. if (llvm::empty(ObjCImageInfoBlocks)) - return make_error<StringError>("Empty __objc_imageinfo section in " + - G.getName(), + return make_error<StringError>("Empty " + ObjCImageInfoSectionName + + " section in " + G.getName(), inconvertibleErrorCode()); // Check that there's only one block in the section. if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) - return make_error<StringError>("Multiple blocks in __objc_imageinfo " - "section in " + - G.getName(), + return make_error<StringError>("Multiple blocks in " + + ObjCImageInfoSectionName + + " section in " + G.getName(), inconvertibleErrorCode()); // Check that the __objc_imageinfo section is unreferenced. @@ -443,8 +828,8 @@ Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( for (auto &E : B->edges()) if (E.getTarget().isDefined() && &E.getTarget().getBlock().getSection() == ObjCImageInfo) - return make_error<StringError>("__objc_imageinfo is referenced " - "within file " + + return make_error<StringError>(ObjCImageInfoSectionName + + " is referenced within file " + G.getName(), inconvertibleErrorCode()); } @@ -456,7 +841,7 @@ Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( support::endian::read32(ObjCImageInfoData + 4, G.getEndianness()); // Lock the mutex while we verify / update the ObjCImageInfos map. - std::lock_guard<std::mutex> Lock(InitScraperMutex); + std::lock_guard<std::mutex> Lock(PluginMutex); auto ObjCImageInfoItr = ObjCImageInfos.find(&MR.getTargetJITDylib()); if (ObjCImageInfoItr != ObjCImageInfos.end()) { @@ -485,5 +870,92 @@ Error MachOPlatform::InitScraperPlugin::processObjCImageInfo( return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::registerInitSections( + jitlink::LinkGraph &G, JITDylib &JD) { + + ExecutorAddress ObjCImageInfoAddr; + SmallVector<jitlink::Section *> InitSections; + + if (auto *ObjCImageInfoSec = G.findSectionByName(ObjCImageInfoSectionName)) { + if (auto Addr = jitlink::SectionRange(*ObjCImageInfoSec).getStart()) + ObjCImageInfoAddr.setValue(Addr); + } + + for (auto InitSectionName : InitSectionNames) + if (auto *Sec = G.findSectionByName(InitSectionName)) + InitSections.push_back(Sec); + + // Dump the scraped inits. + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; + if (ObjCImageInfoAddr) + dbgs() << " " << ObjCImageInfoSectionName << ": " + << formatv("{0:x}", ObjCImageInfoAddr.getValue()) << "\n"; + for (auto *Sec : InitSections) { + jitlink::SectionRange R(*Sec); + dbgs() << " " << Sec->getName() << ": " + << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n"; + } + }); + + return MP.registerInitInfo(JD, ObjCImageInfoAddr, InitSections); +} + +Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( + jitlink::LinkGraph &G, JITDylib &JD) { + + // Rename external references to __tlv_bootstrap to ___orc_rt_tlv_get_addr. + for (auto *Sym : G.external_symbols()) + if (Sym->getName() == "__tlv_bootstrap") { + Sym->setName("___orc_rt_macho_tlv_get_addr"); + break; + } + + // Store key in __thread_vars struct fields. + if (auto *ThreadDataSec = G.findSectionByName(ThreadVarsSectionName)) { + Optional<uint64_t> Key; + { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + auto I = MP.JITDylibToPThreadKey.find(&JD); + if (I != MP.JITDylibToPThreadKey.end()) + Key = I->second; + } + + if (!Key) { + if (auto KeyOrErr = MP.createPThreadKey()) + Key = *KeyOrErr; + else + return KeyOrErr.takeError(); + } + + uint64_t PlatformKeyBits = + support::endian::byte_swap(*Key, G.getEndianness()); + + for (auto *B : ThreadDataSec->blocks()) { + if (B->getSize() != 3 * G.getPointerSize()) + return make_error<StringError>("__thread_vars block at " + + formatv("{0:x}", B->getAddress()) + + " has unexpected size", + inconvertibleErrorCode()); + + auto NewBlockContent = G.allocateBuffer(B->getSize()); + llvm::copy(B->getContent(), NewBlockContent.data()); + memcpy(NewBlockContent.data() + G.getPointerSize(), &PlatformKeyBits, + G.getPointerSize()); + B->setContent(NewBlockContent); + } + } + + // Transform any TLV edges into GOT edges. + for (auto *B : G.blocks()) + for (auto &E : B->edges()) + if (E.getKind() == + jitlink::x86_64::RequestTLVPAndTransformToPCRel32TLVPLoadRelaxable) + E.setKind( + jitlink::x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable); + + return Error::success(); +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Mangling.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Mangling.cpp index 606304741cf7..14b22880ab7e 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Mangling.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Mangling.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/Mangling.h" +#include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Mangler.h" #include "llvm/Object/MachO.h" @@ -130,24 +131,34 @@ getObjectSymbolInfo(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { SymbolStringPtr InitSymbol; + size_t Counter = 0; + auto AddInitSymbol = [&]() { + while (true) { + std::string InitSymString; + raw_string_ostream(InitSymString) + << "$." << ObjBuffer.getBufferIdentifier() << ".__inits." + << Counter++; + InitSymbol = ES.intern(InitSymString); + if (SymbolFlags.count(InitSymbol)) + continue; + SymbolFlags[InitSymbol] = JITSymbolFlags::MaterializationSideEffectsOnly; + return; + } + }; + if (IsMachO) { auto &MachOObj = cast<object::MachOObjectFile>(*Obj->get()); for (auto &Sec : MachOObj.sections()) { auto SecType = MachOObj.getSectionType(Sec); if ((SecType & MachO::SECTION_TYPE) == MachO::S_MOD_INIT_FUNC_POINTERS) { - size_t Counter = 0; - while (true) { - std::string InitSymString; - raw_string_ostream(InitSymString) - << "$." << ObjBuffer.getBufferIdentifier() << ".__inits." - << Counter++; - InitSymbol = ES.intern(InitSymString); - if (SymbolFlags.count(InitSymbol)) - continue; - SymbolFlags[InitSymbol] = - JITSymbolFlags::MaterializationSideEffectsOnly; - break; - } + AddInitSymbol(); + break; + } + auto SegName = + MachOObj.getSectionFinalSegmentName(Sec.getRawDataRefImpl()); + auto SecName = cantFail(MachOObj.getSectionName(Sec.getRawDataRefImpl())); + if (MachOPlatform::isInitializerSection(SegName, SecName)) { + AddInitSymbol(); break; } } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 26f77acd91fc..fd260089c04b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -7,10 +7,11 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" - #include "llvm/ADT/Optional.h" #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" - +#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" +#include "llvm/Support/MemoryBuffer.h" +#include <string> #include <vector> #define DEBUG_TYPE "orc" @@ -19,6 +20,101 @@ using namespace llvm; using namespace llvm::jitlink; using namespace llvm::orc; +namespace { + +class LinkGraphMaterializationUnit : public MaterializationUnit { +private: + struct LinkGraphInterface { + SymbolFlagsMap SymbolFlags; + SymbolStringPtr InitSymbol; + }; + +public: + static std::unique_ptr<LinkGraphMaterializationUnit> + Create(ObjectLinkingLayer &ObjLinkingLayer, std::unique_ptr<LinkGraph> G) { + auto LGI = scanLinkGraph(ObjLinkingLayer.getExecutionSession(), *G); + return std::unique_ptr<LinkGraphMaterializationUnit>( + new LinkGraphMaterializationUnit(ObjLinkingLayer, std::move(G), + std::move(LGI))); + } + + StringRef getName() const override { return G->getName(); } + void materialize(std::unique_ptr<MaterializationResponsibility> MR) override { + ObjLinkingLayer.emit(std::move(MR), std::move(G)); + } + +private: + static LinkGraphInterface scanLinkGraph(ExecutionSession &ES, LinkGraph &G) { + + LinkGraphInterface LGI; + + for (auto *Sym : G.defined_symbols()) { + // Skip local symbols. + if (Sym->getScope() == Scope::Local) + continue; + assert(Sym->hasName() && "Anonymous non-local symbol?"); + + JITSymbolFlags Flags; + if (Sym->getScope() == Scope::Default) + Flags |= JITSymbolFlags::Exported; + + if (Sym->isCallable()) + Flags |= JITSymbolFlags::Callable; + + LGI.SymbolFlags[ES.intern(Sym->getName())] = Flags; + } + + if (G.getTargetTriple().isOSBinFormatMachO()) + if (hasMachOInitSection(G)) + LGI.InitSymbol = makeInitSymbol(ES, G); + + return LGI; + } + + static bool hasMachOInitSection(LinkGraph &G) { + for (auto &Sec : G.sections()) + if (Sec.getName() == "__DATA,__obj_selrefs" || + Sec.getName() == "__DATA,__objc_classlist" || + Sec.getName() == "__TEXT,__swift5_protos" || + Sec.getName() == "__TEXT,__swift5_proto" || + Sec.getName() == "__DATA,__mod_init_func") + return true; + return false; + } + + static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) { + std::string InitSymString; + raw_string_ostream(InitSymString) + << "$." << G.getName() << ".__inits" << Counter++; + return ES.intern(InitSymString); + } + + LinkGraphMaterializationUnit(ObjectLinkingLayer &ObjLinkingLayer, + std::unique_ptr<LinkGraph> G, + LinkGraphInterface LGI) + : MaterializationUnit(std::move(LGI.SymbolFlags), + std::move(LGI.InitSymbol)), + ObjLinkingLayer(ObjLinkingLayer), G(std::move(G)) {} + + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { + for (auto *Sym : G->defined_symbols()) + if (Sym->getName() == *Name) { + assert(Sym->getLinkage() == Linkage::Weak && + "Discarding non-weak definition"); + G->makeExternal(*Sym); + break; + } + } + + ObjectLinkingLayer &ObjLinkingLayer; + std::unique_ptr<LinkGraph> G; + static std::atomic<uint64_t> Counter; +}; + +std::atomic<uint64_t> LinkGraphMaterializationUnit::Counter{0}; + +} // end anonymous namespace + namespace llvm { namespace orc { @@ -40,6 +136,13 @@ public: JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } + void notifyMaterializing(LinkGraph &G) { + for (auto &P : Layer.Plugins) + P->notifyMaterializing(*MR, G, *this, + ObjBuffer ? ObjBuffer->getMemBufferRef() + : MemoryBufferRef()); + } + void notifyFailed(Error Err) override { for (auto &P : Layer.Plugins) Err = joinErrors(std::move(Err), P->notifyFailed(*MR)); @@ -211,14 +314,14 @@ public: return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); }; } - Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { + Error modifyPassConfig(LinkGraph &LG, PassConfiguration &Config) override { // Add passes to mark duplicate defs as should-discard, and to walk the // link graph to build the symbol dependence graph. Config.PrePrunePasses.push_back([this](LinkGraph &G) { return claimOrExternalizeWeakAndCommonSymbols(G); }); - Layer.modifyPassConfig(*MR, TT, Config); + Layer.modifyPassConfig(*MR, LG, Config); Config.PostPrunePasses.push_back( [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); }); @@ -227,12 +330,82 @@ public: } private: - struct LocalSymbolNamedDependencies { + // Symbol name dependencies: + // Internal: Defined in this graph. + // External: Defined externally. + struct BlockSymbolDependencies { SymbolNameSet Internal, External; }; - using LocalSymbolNamedDependenciesMap = - DenseMap<const Symbol *, LocalSymbolNamedDependencies>; + // Lazily populated map of blocks to BlockSymbolDependencies values. + class BlockDependenciesMap { + public: + BlockDependenciesMap(ExecutionSession &ES, + DenseMap<const Block *, DenseSet<Block *>> BlockDeps) + : ES(ES), BlockDeps(std::move(BlockDeps)) {} + + const BlockSymbolDependencies &operator[](const Block &B) { + // Check the cache first. + auto I = BlockTransitiveDepsCache.find(&B); + if (I != BlockTransitiveDepsCache.end()) + return I->second; + + // No value. Populate the cache. + BlockSymbolDependencies BTDCacheVal; + auto BDI = BlockDeps.find(&B); + assert(BDI != BlockDeps.end() && "No block dependencies"); + + for (auto *BDep : BDI->second) { + auto &BID = getBlockImmediateDeps(*BDep); + for (auto &ExternalDep : BID.External) + BTDCacheVal.External.insert(ExternalDep); + for (auto &InternalDep : BID.Internal) + BTDCacheVal.Internal.insert(InternalDep); + } + + return BlockTransitiveDepsCache + .insert(std::make_pair(&B, std::move(BTDCacheVal))) + .first->second; + } + + SymbolStringPtr &getInternedName(Symbol &Sym) { + auto I = NameCache.find(&Sym); + if (I != NameCache.end()) + return I->second; + + return NameCache.insert(std::make_pair(&Sym, ES.intern(Sym.getName()))) + .first->second; + } + + private: + BlockSymbolDependencies &getBlockImmediateDeps(Block &B) { + // Check the cache first. + auto I = BlockImmediateDepsCache.find(&B); + if (I != BlockImmediateDepsCache.end()) + return I->second; + + BlockSymbolDependencies BIDCacheVal; + for (auto &E : B.edges()) { + auto &Tgt = E.getTarget(); + if (Tgt.getScope() != Scope::Local) { + if (Tgt.isExternal()) + BIDCacheVal.External.insert(getInternedName(Tgt)); + else + BIDCacheVal.Internal.insert(getInternedName(Tgt)); + } + } + + return BlockImmediateDepsCache + .insert(std::make_pair(&B, std::move(BIDCacheVal))) + .first->second; + } + + ExecutionSession &ES; + DenseMap<const Block *, DenseSet<Block *>> BlockDeps; + DenseMap<const Symbol *, SymbolStringPtr> NameCache; + DenseMap<const Block *, BlockSymbolDependencies> BlockImmediateDepsCache; + DenseMap<const Block *, BlockSymbolDependencies> BlockTransitiveDepsCache; + }; Error claimOrExternalizeWeakAndCommonSymbols(LinkGraph &G) { auto &ES = Layer.getExecutionSession(); @@ -280,7 +453,7 @@ private: Error computeNamedSymbolDependencies(LinkGraph &G) { auto &ES = MR->getTargetJITDylib().getExecutionSession(); - auto LocalDeps = computeLocalDeps(G); + auto BlockDeps = computeBlockNonLocalDeps(G); // Compute dependencies for symbols defined in the JITLink graph. for (auto *Sym : G.defined_symbols()) { @@ -291,58 +464,41 @@ private: assert(Sym->hasName() && "Defined non-local jitlink::Symbol should have a name"); - SymbolNameSet ExternalSymDeps, InternalSymDeps; - - // Find internal and external named symbol dependencies. - for (auto &E : Sym->getBlock().edges()) { - auto &TargetSym = E.getTarget(); - - if (TargetSym.getScope() != Scope::Local) { - if (TargetSym.isExternal()) - ExternalSymDeps.insert(ES.intern(TargetSym.getName())); - else if (&TargetSym != Sym) - InternalSymDeps.insert(ES.intern(TargetSym.getName())); - } else { - assert(TargetSym.isDefined() && - "local symbols must be defined"); - auto I = LocalDeps.find(&TargetSym); - if (I != LocalDeps.end()) { - for (auto &S : I->second.External) - ExternalSymDeps.insert(S); - for (auto &S : I->second.Internal) - InternalSymDeps.insert(S); - } - } - } - - if (ExternalSymDeps.empty() && InternalSymDeps.empty()) + auto &SymDeps = BlockDeps[Sym->getBlock()]; + if (SymDeps.External.empty() && SymDeps.Internal.empty()) continue; auto SymName = ES.intern(Sym->getName()); - if (!ExternalSymDeps.empty()) - ExternalNamedSymbolDeps[SymName] = std::move(ExternalSymDeps); - if (!InternalSymDeps.empty()) - InternalNamedSymbolDeps[SymName] = std::move(InternalSymDeps); + if (!SymDeps.External.empty()) + ExternalNamedSymbolDeps[SymName] = SymDeps.External; + if (!SymDeps.Internal.empty()) + InternalNamedSymbolDeps[SymName] = SymDeps.Internal; } for (auto &P : Layer.Plugins) { - auto SyntheticLocalDeps = P->getSyntheticSymbolLocalDependencies(*MR); - if (SyntheticLocalDeps.empty()) + auto SynthDeps = P->getSyntheticSymbolDependencies(*MR); + if (SynthDeps.empty()) continue; - for (auto &KV : SyntheticLocalDeps) { + DenseSet<Block *> BlockVisited; + for (auto &KV : SynthDeps) { auto &Name = KV.first; - auto &LocalDepsForName = KV.second; - for (auto *Local : LocalDepsForName) { - assert(Local->getScope() == Scope::Local && - "Dependence on non-local symbol"); - auto LocalNamedDepsItr = LocalDeps.find(Local); - if (LocalNamedDepsItr == LocalDeps.end()) - continue; - for (auto &S : LocalNamedDepsItr->second.Internal) - InternalNamedSymbolDeps[Name].insert(S); - for (auto &S : LocalNamedDepsItr->second.External) - ExternalNamedSymbolDeps[Name].insert(S); + auto &DepsForName = KV.second; + for (auto *Sym : DepsForName) { + if (Sym->getScope() == Scope::Local) { + auto &BDeps = BlockDeps[Sym->getBlock()]; + for (auto &S : BDeps.Internal) + InternalNamedSymbolDeps[Name].insert(S); + for (auto &S : BDeps.External) + ExternalNamedSymbolDeps[Name].insert(S); + } else { + if (Sym->isExternal()) + ExternalNamedSymbolDeps[Name].insert( + BlockDeps.getInternedName(*Sym)); + else + InternalNamedSymbolDeps[Name].insert( + BlockDeps.getInternedName(*Sym)); + } } } } @@ -350,81 +506,69 @@ private: return Error::success(); } - LocalSymbolNamedDependenciesMap computeLocalDeps(LinkGraph &G) { - DenseMap<jitlink::Symbol *, DenseSet<jitlink::Symbol *>> DepMap; - - // For all local symbols: - // (1) Add their named dependencies. - // (2) Add them to the worklist for further iteration if they have any - // depend on any other local symbols. - struct WorklistEntry { - WorklistEntry(Symbol *Sym, DenseSet<Symbol *> LocalDeps) - : Sym(Sym), LocalDeps(std::move(LocalDeps)) {} - - Symbol *Sym = nullptr; - DenseSet<Symbol *> LocalDeps; + BlockDependenciesMap computeBlockNonLocalDeps(LinkGraph &G) { + // First calculate the reachable-via-non-local-symbol blocks for each block. + struct BlockInfo { + DenseSet<Block *> Dependencies; + DenseSet<Block *> Dependants; + bool DependenciesChanged = true; }; - std::vector<WorklistEntry> Worklist; - for (auto *Sym : G.defined_symbols()) - if (Sym->getScope() == Scope::Local) { - auto &SymNamedDeps = DepMap[Sym]; - DenseSet<Symbol *> LocalDeps; - - for (auto &E : Sym->getBlock().edges()) { - auto &TargetSym = E.getTarget(); - if (TargetSym.getScope() != Scope::Local) - SymNamedDeps.insert(&TargetSym); - else { - assert(TargetSym.isDefined() && - "local symbols must be defined"); - LocalDeps.insert(&TargetSym); + DenseMap<Block *, BlockInfo> BlockInfos; + SmallVector<Block *> WorkList; + + // Pre-allocate map entries. This prevents any iterator/reference + // invalidation in the next loop. + for (auto *B : G.blocks()) + (void)BlockInfos[B]; + + // Build initial worklist, record block dependencies/dependants and + // non-local symbol dependencies. + for (auto *B : G.blocks()) { + auto &BI = BlockInfos[B]; + for (auto &E : B->edges()) { + if (E.getTarget().getScope() == Scope::Local) { + auto &TgtB = E.getTarget().getBlock(); + if (&TgtB != B) { + BI.Dependencies.insert(&TgtB); + BlockInfos[&TgtB].Dependants.insert(B); } } - - if (!LocalDeps.empty()) - Worklist.push_back(WorklistEntry(Sym, std::move(LocalDeps))); } - // Loop over all local symbols with local dependencies, propagating - // their respective non-local dependencies. Iterate until we hit a stable - // state. - bool Changed; - do { - Changed = false; - for (auto &WLEntry : Worklist) { - auto *Sym = WLEntry.Sym; - auto &NamedDeps = DepMap[Sym]; - auto &LocalDeps = WLEntry.LocalDeps; - - for (auto *TargetSym : LocalDeps) { - auto I = DepMap.find(TargetSym); - if (I != DepMap.end()) - for (const auto &S : I->second) - Changed |= NamedDeps.insert(S).second; - } - } - } while (Changed); + // If this node has both dependants and dependencies then add it to the + // worklist to propagate the dependencies to the dependants. + if (!BI.Dependants.empty() && !BI.Dependencies.empty()) + WorkList.push_back(B); + } - // Intern the results to produce a mapping of jitlink::Symbol* to internal - // and external symbol names. - auto &ES = Layer.getExecutionSession(); - LocalSymbolNamedDependenciesMap Result; - for (auto &KV : DepMap) { - auto *Local = KV.first; - assert(Local->getScope() == Scope::Local && - "DepMap keys should all be local symbols"); - auto &LocalNamedDeps = Result[Local]; - for (auto *Named : KV.second) { - assert(Named->getScope() != Scope::Local && - "DepMap values should all be non-local symbol sets"); - if (Named->isExternal()) - LocalNamedDeps.External.insert(ES.intern(Named->getName())); - else - LocalNamedDeps.Internal.insert(ES.intern(Named->getName())); + // Propagate block-level dependencies through the block-dependence graph. + while (!WorkList.empty()) { + auto *B = WorkList.back(); + WorkList.pop_back(); + + auto &BI = BlockInfos[B]; + assert(BI.DependenciesChanged && + "Block in worklist has unchanged dependencies"); + BI.DependenciesChanged = false; + for (auto *Dependant : BI.Dependants) { + auto &DependantBI = BlockInfos[Dependant]; + for (auto *Dependency : BI.Dependencies) { + if (Dependant != Dependency && + DependantBI.Dependencies.insert(Dependency).second) + if (!DependantBI.DependenciesChanged) { + DependantBI.DependenciesChanged = true; + WorkList.push_back(Dependant); + } + } } } - return Result; + DenseMap<const Block *, DenseSet<Block *>> BlockDeps; + for (auto &KV : BlockInfos) + BlockDeps[KV.first] = std::move(KV.second.Dependencies); + + return BlockDependenciesMap(Layer.getExecutionSession(), + std::move(BlockDeps)); } void registerDependencies(const SymbolDependenceMap &QueryDeps) { @@ -459,15 +603,24 @@ private: ObjectLinkingLayer::Plugin::~Plugin() {} +char ObjectLinkingLayer::ID; + +using BaseT = RTTIExtends<ObjectLinkingLayer, ObjectLayer>; + +ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES) + : BaseT(ES), MemMgr(ES.getExecutorProcessControl().getMemMgr()) { + ES.registerResourceManager(*this); +} + ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, JITLinkMemoryManager &MemMgr) - : ObjectLayer(ES), MemMgr(MemMgr) { + : BaseT(ES), MemMgr(MemMgr) { ES.registerResourceManager(*this); } ObjectLinkingLayer::ObjectLinkingLayer( ExecutionSession &ES, std::unique_ptr<JITLinkMemoryManager> MemMgr) - : ObjectLayer(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) { + : BaseT(ES), MemMgr(*MemMgr), MemMgrOwnership(std::move(MemMgr)) { ES.registerResourceManager(*this); } @@ -476,29 +629,41 @@ ObjectLinkingLayer::~ObjectLinkingLayer() { getExecutionSession().deregisterResourceManager(*this); } +Error ObjectLinkingLayer::add(ResourceTrackerSP RT, + std::unique_ptr<LinkGraph> G) { + auto &JD = RT->getJITDylib(); + return JD.define(LinkGraphMaterializationUnit::Create(*this, std::move(G)), + std::move(RT)); +} + void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, std::unique_ptr<MemoryBuffer> O) { assert(O && "Object must not be null"); - auto ObjBuffer = O->getMemBufferRef(); + MemoryBufferRef ObjBuffer = O->getMemBufferRef(); + auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( *this, std::move(R), std::move(O)); - if (auto G = createLinkGraphFromObject(std::move(ObjBuffer))) + if (auto G = createLinkGraphFromObject(ObjBuffer)) { + Ctx->notifyMaterializing(**G); link(std::move(*G), std::move(Ctx)); - else + } else { Ctx->notifyFailed(G.takeError()); + } } void ObjectLinkingLayer::emit(std::unique_ptr<MaterializationResponsibility> R, std::unique_ptr<LinkGraph> G) { - link(std::move(G), std::make_unique<ObjectLinkingLayerJITLinkContext>( - *this, std::move(R), nullptr)); + auto Ctx = std::make_unique<ObjectLinkingLayerJITLinkContext>( + *this, std::move(R), nullptr); + Ctx->notifyMaterializing(*G); + link(std::move(G), std::move(Ctx)); } void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, - const Triple &TT, + LinkGraph &G, PassConfiguration &PassConfig) { for (auto &P : Plugins) - P->modifyPassConfig(MR, TT, PassConfig); + P->modifyPassConfig(MR, G, PassConfig); } void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { @@ -567,11 +732,11 @@ EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( : ES(ES), Registrar(std::move(Registrar)) {} void EHFrameRegistrationPlugin::modifyPassConfig( - MaterializationResponsibility &MR, const Triple &TT, + MaterializationResponsibility &MR, LinkGraph &G, PassConfiguration &PassConfig) { PassConfig.PostFixupPasses.push_back(createEHFrameRecorderPass( - TT, [this, &MR](JITTargetAddress Addr, size_t Size) { + G.getTargetTriple(), [this, &MR](JITTargetAddress Addr, size_t Size) { if (Addr) { std::lock_guard<std::mutex> Lock(EHFramePluginMutex); assert(!InProcessLinks.count(&MR) && @@ -638,13 +803,23 @@ Error EHFrameRegistrationPlugin::notifyRemovingResources(ResourceKey K) { void EHFrameRegistrationPlugin::notifyTransferringResources( ResourceKey DstKey, ResourceKey SrcKey) { auto SI = EHFrameRanges.find(SrcKey); - if (SI != EHFrameRanges.end()) { + if (SI == EHFrameRanges.end()) + return; + + auto DI = EHFrameRanges.find(DstKey); + if (DI != EHFrameRanges.end()) { auto &SrcRanges = SI->second; - auto &DstRanges = EHFrameRanges[DstKey]; + auto &DstRanges = DI->second; DstRanges.reserve(DstRanges.size() + SrcRanges.size()); for (auto &SrcRange : SrcRanges) DstRanges.push_back(std::move(SrcRange)); EHFrameRanges.erase(SI); + } else { + // We need to move SrcKey's ranges over without invalidating the SI + // iterator. + auto Tmp = std::move(SI->second); + EHFrameRanges.erase(SI); + EHFrameRanges[DstKey] = std::move(Tmp); } } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp index a57662e10a79..207a31ec1940 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -12,10 +12,14 @@ namespace llvm { namespace orc { +char ObjectTransformLayer::ID; + +using BaseT = RTTIExtends<ObjectTransformLayer, ObjectLayer>; + ObjectTransformLayer::ObjectTransformLayer(ExecutionSession &ES, - ObjectLayer &BaseLayer, - TransformFunction Transform) - : ObjectLayer(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + ObjectLayer &BaseLayer, + TransformFunction Transform) + : BaseT(ES), BaseLayer(BaseLayer), Transform(std::move(Transform)) {} void ObjectTransformLayer::emit( std::unique_ptr<MaterializationResponsibility> R, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp index 834d4cc8f514..d6f73a8b0864 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -13,6 +13,7 @@ #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/LLJIT.h" +#include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" @@ -29,13 +30,22 @@ public: using PoolEntry = SymbolStringPtr::PoolEntry; using PoolEntryPtr = SymbolStringPtr::PoolEntryPtr; - static PoolEntryPtr releaseSymbolStringPtr(SymbolStringPtr S) { + // Move from SymbolStringPtr to PoolEntryPtr (no change in ref count). + static PoolEntryPtr moveFromSymbolStringPtr(SymbolStringPtr S) { PoolEntryPtr Result = nullptr; std::swap(Result, S.S); return Result; } - static SymbolStringPtr retainSymbolStringPtr(PoolEntryPtr P) { + // Move from a PoolEntryPtr to a SymbolStringPtr (no change in ref count). + static SymbolStringPtr moveToSymbolStringPtr(PoolEntryPtr P) { + SymbolStringPtr S; + S.S = P; + return S; + } + + // Copy a pool entry to a SymbolStringPtr (increments ref count). + static SymbolStringPtr copyToSymbolStringPtr(PoolEntryPtr P) { return SymbolStringPtr(P); } @@ -71,6 +81,8 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcV2CAPIHelper::PoolEntry, LLVMOrcSymbolStringPoolEntryRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(MaterializationUnit, LLVMOrcMaterializationUnitRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(MaterializationResponsibility, + LLVMOrcMaterializationResponsibilityRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITDylib, LLVMOrcJITDylibRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ResourceTracker, LLVMOrcResourceTrackerRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DefinitionGenerator, @@ -82,9 +94,16 @@ DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ThreadSafeModule, LLVMOrcThreadSafeModuleRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(JITTargetMachineBuilder, LLVMOrcJITTargetMachineBuilderRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ObjectLayer, LLVMOrcObjectLayerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IRTransformLayer, LLVMOrcIRTransformLayerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ObjectTransformLayer, + LLVMOrcObjectTransformLayerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(DumpObjects, LLVMOrcDumpObjectsRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(IndirectStubsManager, + LLVMOrcIndirectStubsManagerRef) +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LazyCallThroughManager, + LLVMOrcLazyCallThroughManagerRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJITBuilder, LLVMOrcLLJITBuilderRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJIT, LLVMOrcLLJITRef) - DEFINE_SIMPLE_CONVERSION_FUNCTIONS(TargetMachine, LLVMTargetMachineRef) namespace llvm { @@ -163,6 +182,108 @@ private: } // end namespace orc } // end namespace llvm +namespace { + +class OrcCAPIMaterializationUnit : public llvm::orc::MaterializationUnit { +public: + OrcCAPIMaterializationUnit( + std::string Name, SymbolFlagsMap InitialSymbolFlags, + SymbolStringPtr InitSymbol, void *Ctx, + LLVMOrcMaterializationUnitMaterializeFunction Materialize, + LLVMOrcMaterializationUnitDiscardFunction Discard, + LLVMOrcMaterializationUnitDestroyFunction Destroy) + : llvm::orc::MaterializationUnit(std::move(InitialSymbolFlags), + std::move(InitSymbol)), + Name(std::move(Name)), Ctx(Ctx), Materialize(Materialize), + Discard(Discard), Destroy(Destroy) {} + + ~OrcCAPIMaterializationUnit() { + if (Ctx) + Destroy(Ctx); + } + + StringRef getName() const override { return Name; } + + void materialize(std::unique_ptr<MaterializationResponsibility> R) override { + void *Tmp = Ctx; + Ctx = nullptr; + Materialize(Tmp, wrap(R.release())); + } + +private: + void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { + Discard(Ctx, wrap(&JD), wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); + } + + std::string Name; + void *Ctx = nullptr; + LLVMOrcMaterializationUnitMaterializeFunction Materialize = nullptr; + LLVMOrcMaterializationUnitDiscardFunction Discard = nullptr; + LLVMOrcMaterializationUnitDestroyFunction Destroy = nullptr; +}; + +static JITSymbolFlags toJITSymbolFlags(LLVMJITSymbolFlags F) { + + JITSymbolFlags JSF; + + if (F.GenericFlags & LLVMJITSymbolGenericFlagsExported) + JSF |= JITSymbolFlags::Exported; + if (F.GenericFlags & LLVMJITSymbolGenericFlagsWeak) + JSF |= JITSymbolFlags::Weak; + if (F.GenericFlags & LLVMJITSymbolGenericFlagsCallable) + JSF |= JITSymbolFlags::Callable; + if (F.GenericFlags & LLVMJITSymbolGenericFlagsMaterializationSideEffectsOnly) + JSF |= JITSymbolFlags::MaterializationSideEffectsOnly; + + JSF.getTargetFlags() = F.TargetFlags; + + return JSF; +} + +static LLVMJITSymbolFlags fromJITSymbolFlags(JITSymbolFlags JSF) { + LLVMJITSymbolFlags F = {0, 0}; + if (JSF & JITSymbolFlags::Exported) + F.GenericFlags |= LLVMJITSymbolGenericFlagsExported; + if (JSF & JITSymbolFlags::Weak) + F.GenericFlags |= LLVMJITSymbolGenericFlagsWeak; + if (JSF & JITSymbolFlags::Callable) + F.GenericFlags |= LLVMJITSymbolGenericFlagsCallable; + if (JSF & JITSymbolFlags::MaterializationSideEffectsOnly) + F.GenericFlags |= LLVMJITSymbolGenericFlagsMaterializationSideEffectsOnly; + + F.TargetFlags = JSF.getTargetFlags(); + + return F; +} + +static SymbolMap toSymbolMap(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs) { + SymbolMap SM; + for (size_t I = 0; I != NumPairs; ++I) { + JITSymbolFlags Flags = toJITSymbolFlags(Syms[I].Sym.Flags); + SM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = + JITEvaluatedSymbol(Syms[I].Sym.Address, Flags); + } + return SM; +} + +static SymbolDependenceMap +toSymbolDependenceMap(LLVMOrcCDependenceMapPairs Pairs, size_t NumPairs) { + SymbolDependenceMap SDM; + for (size_t I = 0; I != NumPairs; ++I) { + JITDylib *JD = unwrap(Pairs[I].JD); + SymbolNameSet Names; + + for (size_t J = 0; J != Pairs[I].Names.Length; ++J) { + auto Sym = Pairs[I].Names.Symbols[J]; + Names.insert(OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Sym))); + } + SDM[JD] = Names; + } + return SDM; +} + +} // end anonymous namespace + void LLVMOrcExecutionSessionSetErrorReporter( LLVMOrcExecutionSessionRef ES, LLVMOrcErrorReporterFunction ReportError, void *Ctx) { @@ -172,7 +293,8 @@ void LLVMOrcExecutionSessionSetErrorReporter( LLVMOrcSymbolStringPoolRef LLVMOrcExecutionSessionGetSymbolStringPool(LLVMOrcExecutionSessionRef ES) { - return wrap(unwrap(ES)->getSymbolStringPool().get()); + return wrap( + unwrap(ES)->getExecutorProcessControl().getSymbolStringPool().get()); } void LLVMOrcSymbolStringPoolClearDeadEntries(LLVMOrcSymbolStringPoolRef SSP) { @@ -182,7 +304,7 @@ void LLVMOrcSymbolStringPoolClearDeadEntries(LLVMOrcSymbolStringPoolRef SSP) { LLVMOrcSymbolStringPoolEntryRef LLVMOrcExecutionSessionIntern(LLVMOrcExecutionSessionRef ES, const char *Name) { return wrap( - OrcV2CAPIHelper::releaseSymbolStringPtr(unwrap(ES)->intern(Name))); + OrcV2CAPIHelper::moveFromSymbolStringPtr(unwrap(ES)->intern(Name))); } void LLVMOrcRetainSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { @@ -236,24 +358,191 @@ void LLVMOrcDisposeMaterializationUnit(LLVMOrcMaterializationUnitRef MU) { std::unique_ptr<MaterializationUnit> TmpMU(unwrap(MU)); } +LLVMOrcMaterializationUnitRef LLVMOrcCreateCustomMaterializationUnit( + const char *Name, void *Ctx, LLVMOrcCSymbolFlagsMapPairs Syms, + size_t NumSyms, LLVMOrcSymbolStringPoolEntryRef InitSym, + LLVMOrcMaterializationUnitMaterializeFunction Materialize, + LLVMOrcMaterializationUnitDiscardFunction Discard, + LLVMOrcMaterializationUnitDestroyFunction Destroy) { + SymbolFlagsMap SFM; + for (size_t I = 0; I != NumSyms; ++I) + SFM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = + toJITSymbolFlags(Syms[I].Flags); + + auto IS = OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(InitSym)); + + return wrap(new OrcCAPIMaterializationUnit( + Name, std::move(SFM), std::move(IS), Ctx, Materialize, Discard, Destroy)); +} + LLVMOrcMaterializationUnitRef LLVMOrcAbsoluteSymbols(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs) { - SymbolMap SM; + SymbolMap SM = toSymbolMap(Syms, NumPairs); + return wrap(absoluteSymbols(std::move(SM)).release()); +} + +LLVMOrcMaterializationUnitRef LLVMOrcLazyReexports( + LLVMOrcLazyCallThroughManagerRef LCTM, LLVMOrcIndirectStubsManagerRef ISM, + LLVMOrcJITDylibRef SourceJD, LLVMOrcCSymbolAliasMapPairs CallableAliases, + size_t NumPairs) { + + SymbolAliasMap SAM; for (size_t I = 0; I != NumPairs; ++I) { - JITSymbolFlags Flags; + auto pair = CallableAliases[I]; + JITSymbolFlags Flags = toJITSymbolFlags(pair.Entry.Flags); + SymbolStringPtr Name = + OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(pair.Entry.Name)); + SAM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(pair.Name))] = + SymbolAliasMapEntry(Name, Flags); + } - if (Syms[I].Sym.Flags.GenericFlags & LLVMJITSymbolGenericFlagsExported) - Flags |= JITSymbolFlags::Exported; - if (Syms[I].Sym.Flags.GenericFlags & LLVMJITSymbolGenericFlagsWeak) - Flags |= JITSymbolFlags::Weak; + return wrap(lazyReexports(*unwrap(LCTM), *unwrap(ISM), *unwrap(SourceJD), + std::move(SAM)) + .release()); +} - Flags.getTargetFlags() = Syms[I].Sym.Flags.TargetFlags; +void LLVMOrcDisposeMaterializationResponsibility( + LLVMOrcMaterializationResponsibilityRef MR) { + std::unique_ptr<MaterializationResponsibility> TmpMR(unwrap(MR)); +} - SM[OrcV2CAPIHelper::retainSymbolStringPtr(unwrap(Syms[I].Name))] = - JITEvaluatedSymbol(Syms[I].Sym.Address, Flags); +LLVMOrcJITDylibRef LLVMOrcMaterializationResponsibilityGetTargetDylib( + LLVMOrcMaterializationResponsibilityRef MR) { + return wrap(&unwrap(MR)->getTargetJITDylib()); +} + +LLVMOrcExecutionSessionRef +LLVMOrcMaterializationResponsibilityGetExecutionSession( + LLVMOrcMaterializationResponsibilityRef MR) { + return wrap(&unwrap(MR)->getExecutionSession()); +} + +LLVMOrcCSymbolFlagsMapPairs LLVMOrcMaterializationResponsibilityGetSymbols( + LLVMOrcMaterializationResponsibilityRef MR, size_t *NumPairs) { + + auto Symbols = unwrap(MR)->getSymbols(); + LLVMOrcCSymbolFlagsMapPairs Result = static_cast<LLVMOrcCSymbolFlagsMapPairs>( + safe_malloc(Symbols.size() * sizeof(LLVMOrcCSymbolFlagsMapPair))); + size_t I = 0; + for (auto const &pair : Symbols) { + auto Name = wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(pair.first)); + auto Flags = pair.second; + Result[I] = {Name, fromJITSymbolFlags(Flags)}; + I++; } + *NumPairs = Symbols.size(); + return Result; +} - return wrap(absoluteSymbols(std::move(SM)).release()); +void LLVMOrcDisposeCSymbolFlagsMap(LLVMOrcCSymbolFlagsMapPairs Pairs) { + free(Pairs); +} + +LLVMOrcSymbolStringPoolEntryRef +LLVMOrcMaterializationResponsibilityGetInitializerSymbol( + LLVMOrcMaterializationResponsibilityRef MR) { + auto Sym = unwrap(MR)->getInitializerSymbol(); + return wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Sym)); +} + +LLVMOrcSymbolStringPoolEntryRef * +LLVMOrcMaterializationResponsibilityGetRequestedSymbols( + LLVMOrcMaterializationResponsibilityRef MR, size_t *NumSymbols) { + + auto Symbols = unwrap(MR)->getRequestedSymbols(); + LLVMOrcSymbolStringPoolEntryRef *Result = + static_cast<LLVMOrcSymbolStringPoolEntryRef *>(safe_malloc( + Symbols.size() * sizeof(LLVMOrcSymbolStringPoolEntryRef))); + size_t I = 0; + for (auto &Name : Symbols) { + Result[I] = wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name)); + I++; + } + *NumSymbols = Symbols.size(); + return Result; +} + +void LLVMOrcDisposeSymbols(LLVMOrcSymbolStringPoolEntryRef *Symbols) { + free(Symbols); +} + +LLVMErrorRef LLVMOrcMaterializationResponsibilityNotifyResolved( + LLVMOrcMaterializationResponsibilityRef MR, LLVMOrcCSymbolMapPairs Symbols, + size_t NumPairs) { + SymbolMap SM = toSymbolMap(Symbols, NumPairs); + return wrap(unwrap(MR)->notifyResolved(std::move(SM))); +} + +LLVMErrorRef LLVMOrcMaterializationResponsibilityNotifyEmitted( + LLVMOrcMaterializationResponsibilityRef MR) { + return wrap(unwrap(MR)->notifyEmitted()); +} + +LLVMErrorRef LLVMOrcMaterializationResponsibilityDefineMaterializing( + LLVMOrcMaterializationResponsibilityRef MR, + LLVMOrcCSymbolFlagsMapPairs Syms, size_t NumSyms) { + SymbolFlagsMap SFM; + for (size_t I = 0; I != NumSyms; ++I) + SFM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = + toJITSymbolFlags(Syms[I].Flags); + + return wrap(unwrap(MR)->defineMaterializing(std::move(SFM))); +} + +LLVMErrorRef LLVMOrcMaterializationResponsibilityReplace( + LLVMOrcMaterializationResponsibilityRef MR, + LLVMOrcMaterializationUnitRef MU) { + std::unique_ptr<MaterializationUnit> TmpMU(unwrap(MU)); + return wrap(unwrap(MR)->replace(std::move(TmpMU))); +} + +LLVMErrorRef LLVMOrcMaterializationResponsibilityDelegate( + LLVMOrcMaterializationResponsibilityRef MR, + LLVMOrcSymbolStringPoolEntryRef *Symbols, size_t NumSymbols, + LLVMOrcMaterializationResponsibilityRef *Result) { + SymbolNameSet Syms; + for (size_t I = 0; I != NumSymbols; I++) { + Syms.insert(OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Symbols[I]))); + } + auto OtherMR = unwrap(MR)->delegate(Syms); + + if (!OtherMR) { + return wrap(OtherMR.takeError()); + } + *Result = wrap(OtherMR->release()); + return LLVMErrorSuccess; +} + +void LLVMOrcMaterializationResponsibilityAddDependencies( + LLVMOrcMaterializationResponsibilityRef MR, + LLVMOrcSymbolStringPoolEntryRef Name, + LLVMOrcCDependenceMapPairs Dependencies, size_t NumPairs) { + + SymbolDependenceMap SDM = toSymbolDependenceMap(Dependencies, NumPairs); + auto Sym = OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Name)); + unwrap(MR)->addDependencies(Sym, SDM); +} + +void LLVMOrcMaterializationResponsibilityAddDependenciesForAll( + LLVMOrcMaterializationResponsibilityRef MR, + LLVMOrcCDependenceMapPairs Dependencies, size_t NumPairs) { + + SymbolDependenceMap SDM = toSymbolDependenceMap(Dependencies, NumPairs); + unwrap(MR)->addDependenciesForAll(SDM); +} + +void LLVMOrcMaterializationResponsibilityFailMaterialization( + LLVMOrcMaterializationResponsibilityRef MR) { + unwrap(MR)->failMaterialization(); +} + +void LLVMOrcIRTransformLayerEmit(LLVMOrcIRTransformLayerRef IRLayer, + LLVMOrcMaterializationResponsibilityRef MR, + LLVMOrcThreadSafeModuleRef TSM) { + std::unique_ptr<ThreadSafeModule> TmpTSM(unwrap(TSM)); + unwrap(IRLayer)->emit( + std::unique_ptr<MaterializationResponsibility>(unwrap(MR)), + std::move(*TmpTSM)); } LLVMOrcJITDylibRef @@ -343,6 +632,14 @@ void LLVMOrcDisposeThreadSafeContext(LLVMOrcThreadSafeContextRef TSCtx) { delete unwrap(TSCtx); } +LLVMErrorRef +LLVMOrcThreadSafeModuleWithModuleDo(LLVMOrcThreadSafeModuleRef TSM, + LLVMOrcGenericIRModuleOperationFunction F, + void *Ctx) { + return wrap(unwrap(TSM)->withModuleDo( + [&](Module &M) { return unwrap(F(Ctx, wrap(&M))); })); +} + LLVMOrcThreadSafeModuleRef LLVMOrcCreateNewThreadSafeModule(LLVMModuleRef M, LLVMOrcThreadSafeContextRef TSCtx) { @@ -393,10 +690,101 @@ void LLVMOrcDisposeJITTargetMachineBuilder( delete unwrap(JTMB); } +char *LLVMOrcJITTargetMachineBuilderGetTargetTriple( + LLVMOrcJITTargetMachineBuilderRef JTMB) { + auto Tmp = unwrap(JTMB)->getTargetTriple().str(); + char *TargetTriple = (char *)malloc(Tmp.size() + 1); + strcpy(TargetTriple, Tmp.c_str()); + return TargetTriple; +} + +void LLVMOrcJITTargetMachineBuilderSetTargetTriple( + LLVMOrcJITTargetMachineBuilderRef JTMB, const char *TargetTriple) { + unwrap(JTMB)->getTargetTriple() = Triple(TargetTriple); +} + +LLVMErrorRef LLVMOrcObjectLayerAddObjectFile(LLVMOrcObjectLayerRef ObjLayer, + LLVMOrcJITDylibRef JD, + LLVMMemoryBufferRef ObjBuffer) { + return wrap(unwrap(ObjLayer)->add( + *unwrap(JD), std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); +} + +LLVMErrorRef LLVMOrcLLJITAddObjectFileWithRT(LLVMOrcObjectLayerRef ObjLayer, + LLVMOrcResourceTrackerRef RT, + LLVMMemoryBufferRef ObjBuffer) { + return wrap( + unwrap(ObjLayer)->add(ResourceTrackerSP(unwrap(RT)), + std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); +} + +void LLVMOrcObjectLayerEmit(LLVMOrcObjectLayerRef ObjLayer, + LLVMOrcMaterializationResponsibilityRef R, + LLVMMemoryBufferRef ObjBuffer) { + unwrap(ObjLayer)->emit( + std::unique_ptr<MaterializationResponsibility>(unwrap(R)), + std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer))); +} + void LLVMOrcDisposeObjectLayer(LLVMOrcObjectLayerRef ObjLayer) { delete unwrap(ObjLayer); } +void LLVMOrcIRTransformLayerSetTransform( + LLVMOrcIRTransformLayerRef IRTransformLayer, + LLVMOrcIRTransformLayerTransformFunction TransformFunction, void *Ctx) { + unwrap(IRTransformLayer) + ->setTransform( + [=](ThreadSafeModule TSM, + MaterializationResponsibility &R) -> Expected<ThreadSafeModule> { + LLVMOrcThreadSafeModuleRef TSMRef = + wrap(new ThreadSafeModule(std::move(TSM))); + if (LLVMErrorRef Err = TransformFunction(Ctx, &TSMRef, wrap(&R))) { + assert(!TSMRef && "TSMRef was not reset to null on error"); + return unwrap(Err); + } + return std::move(*unwrap(TSMRef)); + }); +} + +void LLVMOrcObjectTransformLayerSetTransform( + LLVMOrcObjectTransformLayerRef ObjTransformLayer, + LLVMOrcObjectTransformLayerTransformFunction TransformFunction, void *Ctx) { + unwrap(ObjTransformLayer) + ->setTransform([TransformFunction, Ctx](std::unique_ptr<MemoryBuffer> Obj) + -> Expected<std::unique_ptr<MemoryBuffer>> { + LLVMMemoryBufferRef ObjBuffer = wrap(Obj.release()); + if (LLVMErrorRef Err = TransformFunction(Ctx, &ObjBuffer)) { + assert(!ObjBuffer && "ObjBuffer was not reset to null on error"); + return unwrap(Err); + } + return std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)); + }); +} + +LLVMOrcDumpObjectsRef LLVMOrcCreateDumpObjects(const char *DumpDir, + const char *IdentifierOverride) { + assert(DumpDir && "DumpDir should not be null"); + assert(IdentifierOverride && "IdentifierOverride should not be null"); + return wrap(new DumpObjects(DumpDir, IdentifierOverride)); +} + +void LLVMOrcDisposeDumpObjects(LLVMOrcDumpObjectsRef DumpObjects) { + delete unwrap(DumpObjects); +} + +LLVMErrorRef LLVMOrcDumpObjects_CallOperator(LLVMOrcDumpObjectsRef DumpObjects, + LLVMMemoryBufferRef *ObjBuffer) { + std::unique_ptr<MemoryBuffer> OB(unwrap(*ObjBuffer)); + if (auto Result = (*unwrap(DumpObjects))(std::move(OB))) { + *ObjBuffer = wrap(Result->release()); + return LLVMErrorSuccess; + } else { + *ObjBuffer = nullptr; + return wrap(Result.takeError()); + } +} + LLVMOrcLLJITBuilderRef LLVMOrcCreateLLJITBuilder(void) { return wrap(new LLJITBuilder()); } @@ -407,7 +795,8 @@ void LLVMOrcDisposeLLJITBuilder(LLVMOrcLLJITBuilderRef Builder) { void LLVMOrcLLJITBuilderSetJITTargetMachineBuilder( LLVMOrcLLJITBuilderRef Builder, LLVMOrcJITTargetMachineBuilderRef JTMB) { - unwrap(Builder)->setJITTargetMachineBuilder(*unwrap(JTMB)); + unwrap(Builder)->setJITTargetMachineBuilder(std::move(*unwrap(JTMB))); + LLVMOrcDisposeJITTargetMachineBuilder(JTMB); } void LLVMOrcLLJITBuilderSetObjectLinkingLayerCreator( @@ -463,7 +852,7 @@ char LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J) { LLVMOrcSymbolStringPoolEntryRef LLVMOrcLLJITMangleAndIntern(LLVMOrcLLJITRef J, const char *UnmangledName) { - return wrap(OrcV2CAPIHelper::releaseSymbolStringPtr( + return wrap(OrcV2CAPIHelper::moveFromSymbolStringPtr( unwrap(J)->mangleAndIntern(UnmangledName))); } @@ -511,6 +900,15 @@ LLVMErrorRef LLVMOrcLLJITLookup(LLVMOrcLLJITRef J, return LLVMErrorSuccess; } +LLVMOrcObjectLayerRef LLVMOrcLLJITGetObjLinkingLayer(LLVMOrcLLJITRef J) { + return wrap(&unwrap(J)->getObjLinkingLayer()); +} + +LLVMOrcObjectTransformLayerRef +LLVMOrcLLJITGetObjTransformLayer(LLVMOrcLLJITRef J) { + return wrap(&unwrap(J)->getObjTransformLayer()); +} + LLVMOrcObjectLayerRef LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager( LLVMOrcExecutionSessionRef ES) { @@ -527,3 +925,39 @@ void LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener( reinterpret_cast<RTDyldObjectLinkingLayer *>(unwrap(RTDyldObjLinkingLayer)) ->registerJITEventListener(*unwrap(Listener)); } + +LLVMOrcIRTransformLayerRef LLVMOrcLLJITGetIRTransformLayer(LLVMOrcLLJITRef J) { + return wrap(&unwrap(J)->getIRTransformLayer()); +} + +const char *LLVMOrcLLJITGetDataLayoutStr(LLVMOrcLLJITRef J) { + return unwrap(J)->getDataLayout().getStringRepresentation().c_str(); +} + +LLVMOrcIndirectStubsManagerRef +LLVMOrcCreateLocalIndirectStubsManager(const char *TargetTriple) { + auto builder = createLocalIndirectStubsManagerBuilder(Triple(TargetTriple)); + return wrap(builder().release()); +} + +void LLVMOrcDisposeIndirectStubsManager(LLVMOrcIndirectStubsManagerRef ISM) { + std::unique_ptr<IndirectStubsManager> TmpISM(unwrap(ISM)); +} + +LLVMErrorRef LLVMOrcCreateLocalLazyCallThroughManager( + const char *TargetTriple, LLVMOrcExecutionSessionRef ES, + LLVMOrcJITTargetAddress ErrorHandlerAddr, + LLVMOrcLazyCallThroughManagerRef *Result) { + auto LCTM = createLocalLazyCallThroughManager(Triple(TargetTriple), + *unwrap(ES), ErrorHandlerAddr); + + if (!LCTM) + return wrap(LCTM.takeError()); + *Result = wrap(LCTM->release()); + return LLVMErrorSuccess; +} + +void LLVMOrcDisposeLazyCallThroughManager( + LLVMOrcLazyCallThroughManagerRef LCM) { + std::unique_ptr<LazyCallThroughManager> TmpLCM(unwrap(LCM)); +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 0ad666ebbebd..27044f66a55d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -75,9 +75,13 @@ private: namespace llvm { namespace orc { +char RTDyldObjectLinkingLayer::ID; + +using BaseT = RTTIExtends<RTDyldObjectLinkingLayer, ObjectLayer>; + RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager) - : ObjectLayer(ES), GetMemoryManager(GetMemoryManager) { + : BaseT(ES), GetMemoryManager(GetMemoryManager) { ES.registerResourceManager(*this); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.cpp deleted file mode 100644 index 52d11f0741d4..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.cpp +++ /dev/null @@ -1,44 +0,0 @@ -//===---------- TargetProcessControlTypes.cpp - Shared TPC types ----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// TargetProcessControl types. -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" - -namespace llvm { -namespace orc { -namespace tpctypes { - -WrapperFunctionResult WrapperFunctionResult::from(StringRef S) { - CWrapperFunctionResult R; - zeroInit(R); - R.Size = S.size(); - if (R.Size > sizeof(uint64_t)) { - R.Data.ValuePtr = new uint8_t[R.Size]; - memcpy(R.Data.ValuePtr, S.data(), R.Size); - R.Destroy = destroyWithDeleteArray; - } else - memcpy(R.Data.Value, S.data(), R.Size); - return R; -} - -void WrapperFunctionResult::destroyWithFree(CWrapperFunctionResultData Data, - uint64_t Size) { - free(Data.ValuePtr); -} - -void WrapperFunctionResult::destroyWithDeleteArray( - CWrapperFunctionResultData Data, uint64_t Size) { - delete[] Data.ValuePtr; -} - -} // end namespace tpctypes -} // end namespace orc -} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCEHFrameRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCEHFrameRegistrar.cpp deleted file mode 100644 index 4f901ce6d445..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TPCEHFrameRegistrar.cpp +++ /dev/null @@ -1,80 +0,0 @@ -//===------ TPCEHFrameRegistrar.cpp - TPC-based eh-frame registration -----===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/Orc/TPCEHFrameRegistrar.h" -#include "llvm/Support/BinaryStreamWriter.h" - -namespace llvm { -namespace orc { - -Expected<std::unique_ptr<TPCEHFrameRegistrar>> -TPCEHFrameRegistrar::Create(TargetProcessControl &TPC) { - // FIXME: Proper mangling here -- we really need to decouple linker mangling - // from DataLayout. - - // Find the addresses of the registration/deregistration functions in the - // target process. - auto ProcessHandle = TPC.loadDylib(nullptr); - if (!ProcessHandle) - return ProcessHandle.takeError(); - - std::string RegisterWrapperName, DeregisterWrapperName; - if (TPC.getTargetTriple().isOSBinFormatMachO()) { - RegisterWrapperName += '_'; - DeregisterWrapperName += '_'; - } - RegisterWrapperName += "llvm_orc_registerEHFrameSectionWrapper"; - DeregisterWrapperName += "llvm_orc_deregisterEHFrameSectionWrapper"; - - SymbolLookupSet RegistrationSymbols; - RegistrationSymbols.add(TPC.intern(RegisterWrapperName)); - RegistrationSymbols.add(TPC.intern(DeregisterWrapperName)); - - auto Result = TPC.lookupSymbols({{*ProcessHandle, RegistrationSymbols}}); - if (!Result) - return Result.takeError(); - - assert(Result->size() == 1 && "Unexpected number of dylibs in result"); - assert((*Result)[0].size() == 2 && - "Unexpected number of addresses in result"); - - auto RegisterEHFrameWrapperFnAddr = (*Result)[0][0]; - auto DeregisterEHFrameWrapperFnAddr = (*Result)[0][1]; - - return std::make_unique<TPCEHFrameRegistrar>( - TPC, RegisterEHFrameWrapperFnAddr, DeregisterEHFrameWrapperFnAddr); -} - -Error TPCEHFrameRegistrar::registerEHFrames(JITTargetAddress EHFrameSectionAddr, - size_t EHFrameSectionSize) { - constexpr size_t ArgBufferSize = sizeof(uint64_t) + sizeof(uint64_t); - uint8_t ArgBuffer[ArgBufferSize]; - BinaryStreamWriter ArgWriter( - MutableArrayRef<uint8_t>(ArgBuffer, ArgBufferSize), - support::endianness::big); - cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionAddr))); - cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionSize))); - - return TPC.runWrapper(RegisterEHFrameWrapperFnAddr, ArgBuffer).takeError(); -} - -Error TPCEHFrameRegistrar::deregisterEHFrames( - JITTargetAddress EHFrameSectionAddr, size_t EHFrameSectionSize) { - constexpr size_t ArgBufferSize = sizeof(uint64_t) + sizeof(uint64_t); - uint8_t ArgBuffer[ArgBufferSize]; - BinaryStreamWriter ArgWriter( - MutableArrayRef<uint8_t>(ArgBuffer, ArgBufferSize), - support::endianness::big); - cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionAddr))); - cantFail(ArgWriter.writeInteger(static_cast<uint64_t>(EHFrameSectionSize))); - - return TPC.runWrapper(DeregisterEHFrameWrapperFnAddr, ArgBuffer).takeError(); -} - -} // end namespace orc -} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp new file mode 100644 index 000000000000..43c2a44835fd --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp @@ -0,0 +1,101 @@ +//===- JITLoaderGDB.h - Register objects via GDB JIT interface -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" + +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/ManagedStatic.h" + +#include <cstdint> +#include <mutex> +#include <utility> + +#define DEBUG_TYPE "orc" + +// First version as landed in August 2009 +static constexpr uint32_t JitDescriptorVersion = 1; + +// Keep in sync with gdb/gdb/jit.h +extern "C" { + +typedef enum { + JIT_NOACTION = 0, + JIT_REGISTER_FN, + JIT_UNREGISTER_FN +} jit_actions_t; + +struct jit_code_entry { + struct jit_code_entry *next_entry; + struct jit_code_entry *prev_entry; + const char *symfile_addr; + uint64_t symfile_size; +}; + +struct jit_descriptor { + uint32_t version; + // This should be jit_actions_t, but we want to be specific about the + // bit-width. + uint32_t action_flag; + struct jit_code_entry *relevant_entry; + struct jit_code_entry *first_entry; +}; + +// We put information about the JITed function in this global, which the +// debugger reads. Make sure to specify the version statically, because the +// debugger checks the version before we can set it during runtime. +struct jit_descriptor __jit_debug_descriptor = {JitDescriptorVersion, 0, + nullptr, nullptr}; + +// Debuggers that implement the GDB JIT interface put a special breakpoint in +// this function. +LLVM_ATTRIBUTE_NOINLINE void __jit_debug_register_code() { + // The noinline and the asm prevent calls to this function from being + // optimized out. +#if !defined(_MSC_VER) + asm volatile("" ::: "memory"); +#endif +} +} + +using namespace llvm; + +// Serialize rendezvous with the debugger as well as access to shared data. +ManagedStatic<std::mutex> JITDebugLock; + +// Register debug object, return error message or null for success. +static void registerJITLoaderGDBImpl(JITTargetAddress Addr, uint64_t Size) { + jit_code_entry *E = new jit_code_entry; + E->symfile_addr = jitTargetAddressToPointer<const char *>(Addr); + E->symfile_size = Size; + E->prev_entry = nullptr; + + std::lock_guard<std::mutex> Lock(*JITDebugLock); + + // Insert this entry at the head of the list. + jit_code_entry *NextEntry = __jit_debug_descriptor.first_entry; + E->next_entry = NextEntry; + if (NextEntry) { + NextEntry->prev_entry = E; + } + + __jit_debug_descriptor.first_entry = E; + __jit_debug_descriptor.relevant_entry = E; + + // Run into the rendezvous breakpoint. + __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; + __jit_debug_register_code(); +} + +extern "C" orc::shared::detail::CWrapperFunctionResult +llvm_orc_registerJITLoaderGDBWrapper(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction<void(SPSExecutorAddress, uint64_t)>::handle( + Data, Size, registerJITLoaderGDBImpl) + .release(); +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp index aff7296cb6e3..4a408d61ee38 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.cpp @@ -23,7 +23,7 @@ using namespace llvm; using namespace llvm::orc; -using namespace llvm::orc::tpctypes; +using namespace llvm::orc::shared; namespace llvm { namespace orc { @@ -155,54 +155,26 @@ Error deregisterEHFrameSection(const void *EHFrameSectionAddr, } // end namespace orc } // end namespace llvm -extern "C" CWrapperFunctionResult -llvm_orc_registerEHFrameSectionWrapper(uint8_t *Data, uint64_t Size) { - if (Size != sizeof(uint64_t) + sizeof(uint64_t)) - return WrapperFunctionResult::from( - "Invalid arguments to llvm_orc_registerEHFrameSectionWrapper") - .release(); - - uint64_t EHFrameSectionAddr; - uint64_t EHFrameSectionSize; - - { - BinaryStreamReader ArgReader(ArrayRef<uint8_t>(Data, Size), - support::endianness::big); - cantFail(ArgReader.readInteger(EHFrameSectionAddr)); - cantFail(ArgReader.readInteger(EHFrameSectionSize)); - } +static Error registerEHFrameWrapper(JITTargetAddress Addr, uint64_t Size) { + return llvm::orc::registerEHFrameSection( + jitTargetAddressToPointer<const void *>(Addr), Size); +} - if (auto Err = registerEHFrameSection( - jitTargetAddressToPointer<void *>(EHFrameSectionAddr), - EHFrameSectionSize)) { - auto ErrMsg = toString(std::move(Err)); - return WrapperFunctionResult::from(ErrMsg).release(); - } - return WrapperFunctionResult().release(); +static Error deregisterEHFrameWrapper(JITTargetAddress Addr, uint64_t Size) { + return llvm::orc::deregisterEHFrameSection( + jitTargetAddressToPointer<const void *>(Addr), Size); } -extern "C" CWrapperFunctionResult -llvm_orc_deregisterEHFrameSectionWrapper(uint8_t *Data, uint64_t Size) { - if (Size != sizeof(uint64_t) + sizeof(uint64_t)) - return WrapperFunctionResult::from( - "Invalid arguments to llvm_orc_registerEHFrameSectionWrapper") - .release(); - - uint64_t EHFrameSectionAddr; - uint64_t EHFrameSectionSize; - - { - BinaryStreamReader ArgReader(ArrayRef<uint8_t>(Data, Size), - support::endianness::big); - cantFail(ArgReader.readInteger(EHFrameSectionAddr)); - cantFail(ArgReader.readInteger(EHFrameSectionSize)); - } +extern "C" orc::shared::detail::CWrapperFunctionResult +llvm_orc_registerEHFrameSectionWrapper(const char *Data, uint64_t Size) { + return WrapperFunction<SPSError(SPSExecutorAddress, uint64_t)>::handle( + Data, Size, registerEHFrameWrapper) + .release(); +} - if (auto Err = deregisterEHFrameSection( - jitTargetAddressToPointer<void *>(EHFrameSectionAddr), - EHFrameSectionSize)) { - auto ErrMsg = toString(std::move(Err)); - return WrapperFunctionResult::from(ErrMsg).release(); - } - return WrapperFunctionResult().release(); +extern "C" orc::shared::detail::CWrapperFunctionResult +llvm_orc_deregisterEHFrameSectionWrapper(const char *Data, uint64_t Size) { + return WrapperFunction<SPSError(SPSExecutorAddress, uint64_t)>::handle( + Data, Size, deregisterEHFrameWrapper) + .release(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp index d4c715cc59f6..4a236e183c8b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp @@ -284,6 +284,9 @@ void PerfJITEventListener::notifyObjectLoaded( NotifyCode(Name, *AddrOrErr, Size); } + // avoid races with writes + std::lock_guard<sys::Mutex> Guard(Mutex); + Dumpstream->flush(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index e49e6e541f15..687fd839805f 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -146,8 +146,8 @@ void RuntimeDyldImpl::resolveLocalRelocations() { // The Section here (Sections[i]) refers to the section in which the // symbol for the relocation is located. The SectionID in the relocation // entry provides the section to which the relocation will be applied. - int Idx = it->first; - uint64_t Addr = Sections[Idx].getLoadAddress(); + unsigned Idx = it->first; + uint64_t Addr = getSectionLoadAddress(Idx); LLVM_DEBUG(dbgs() << "Resolving relocations Section #" << Idx << "\t" << format("%p", (uintptr_t)Addr) << "\n"); resolveRelocationList(it->second, Addr); @@ -655,6 +655,10 @@ unsigned RuntimeDyldImpl::computeGOTSize(const ObjectFile &Obj) { // compute stub buffer size for the given section unsigned RuntimeDyldImpl::computeSectionStubBufSize(const ObjectFile &Obj, const SectionRef &Section) { + if (!MemMgr.allowStubAllocation()) { + return 0; + } + unsigned StubSize = getMaxStubSize(); if (StubSize == 0) { return 0; @@ -1077,7 +1081,8 @@ void RuntimeDyldImpl::resolveRelocationList(const RelocationList &Relocs, for (unsigned i = 0, e = Relocs.size(); i != e; ++i) { const RelocationEntry &RE = Relocs[i]; // Ignore relocations for sections that were not loaded - if (Sections[RE.SectionID].getAddress() == nullptr) + if (RE.SectionID != AbsoluteSymbolSection && + Sections[RE.SectionID].getAddress() == nullptr) continue; resolveRelocation(RE, Value); } @@ -1085,16 +1090,13 @@ void RuntimeDyldImpl::resolveRelocationList(const RelocationList &Relocs, void RuntimeDyldImpl::applyExternalSymbolRelocations( const StringMap<JITEvaluatedSymbol> ExternalSymbolMap) { - while (!ExternalSymbolRelocations.empty()) { - - StringMap<RelocationList>::iterator i = ExternalSymbolRelocations.begin(); - - StringRef Name = i->first(); + for (auto &RelocKV : ExternalSymbolRelocations) { + StringRef Name = RelocKV.first(); + RelocationList &Relocs = RelocKV.second; if (Name.size() == 0) { // This is an absolute symbol, use an address of zero. LLVM_DEBUG(dbgs() << "Resolving absolute relocations." << "\n"); - RelocationList &Relocs = i->second; resolveRelocationList(Relocs, 0); } else { uint64_t Addr = 0; @@ -1105,13 +1107,6 @@ void RuntimeDyldImpl::applyExternalSymbolRelocations( assert(RRI != ExternalSymbolMap.end() && "No result for symbol"); Addr = RRI->second.getAddress(); Flags = RRI->second.getFlags(); - // The call to getSymbolAddress may have caused additional modules to - // be loaded, which may have added new entries to the - // ExternalSymbolRelocations map. Consquently, we need to update our - // iterator. This is also why retrieval of the relocation list - // associated with this symbol is deferred until below this point. - // New entries may have been added to the relocation list. - i = ExternalSymbolRelocations.find(Name); } else { // We found the symbol in our global table. It was probably in a // Module that we loaded previously. @@ -1122,7 +1117,7 @@ void RuntimeDyldImpl::applyExternalSymbolRelocations( } // FIXME: Implement error handling that doesn't kill the host program! - if (!Addr) + if (!Addr && !Resolver.allowsZeroSymbols()) report_fatal_error("Program used external function '" + Name + "' which could not be resolved!"); @@ -1137,15 +1132,11 @@ void RuntimeDyldImpl::applyExternalSymbolRelocations( LLVM_DEBUG(dbgs() << "Resolving relocations Name: " << Name << "\t" << format("0x%lx", Addr) << "\n"); - // This list may have been updated when we called getSymbolAddress, so - // don't change this code to get the list earlier. - RelocationList &Relocs = i->second; resolveRelocationList(Relocs, Addr); } } - - ExternalSymbolRelocations.erase(i); } + ExternalSymbolRelocations.clear(); } Error RuntimeDyldImpl::resolveExternalSymbols() { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index 2fbe707ce8df..a3005f786cf9 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -381,7 +381,9 @@ private: RemainingExpr = RemainingExpr.substr(1).ltrim(); StringRef SectionName; - std::tie(SectionName, RemainingExpr) = parseSymbol(RemainingExpr); + size_t CloseParensIdx = RemainingExpr.find(')'); + SectionName = RemainingExpr.substr(0, CloseParensIdx).rtrim(); + RemainingExpr = RemainingExpr.substr(CloseParensIdx).ltrim(); if (!RemainingExpr.startswith(")")) return std::make_pair( @@ -794,7 +796,7 @@ StringRef RuntimeDyldCheckerImpl::getSymbolContent(StringRef Symbol) const { logAllUnhandledErrors(SymInfo.takeError(), errs(), "RTDyldChecker: "); return StringRef(); } - return SymInfo->getContent(); + return {SymInfo->getContent().data(), SymInfo->getContent().size()}; } std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getSectionAddr( diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 28e1faab5ac7..efe0b9cd61cd 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -266,6 +266,25 @@ void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section, break; case ELF::R_X86_64_NONE: break; + case ELF::R_X86_64_8: { + Value += Addend; + assert((int64_t)Value <= INT8_MAX && (int64_t)Value >= INT8_MIN); + uint8_t TruncatedAddr = (Value & 0xFF); + *Section.getAddressWithOffset(Offset) = TruncatedAddr; + LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at " + << format("%p\n", Section.getAddressWithOffset(Offset))); + break; + } + case ELF::R_X86_64_16: { + Value += Addend; + assert((int64_t)Value <= INT16_MAX && (int64_t)Value >= INT16_MIN); + uint16_t TruncatedAddr = (Value & 0xFFFF); + support::ulittle16_t::ref(Section.getAddressWithOffset(Offset)) = + TruncatedAddr; + LLVM_DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr) << " at " + << format("%p\n", Section.getAddressWithOffset(Offset))); + break; + } case ELF::R_X86_64_64: { support::ulittle64_t::ref(Section.getAddressWithOffset(Offset)) = Value + Addend; @@ -409,6 +428,25 @@ void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section, case ELF::R_AARCH64_PREL64: write(isBE, TargetPtr, Value + Addend - FinalAddress); break; + case ELF::R_AARCH64_CONDBR19: { + uint64_t BranchImm = Value + Addend - FinalAddress; + + assert(isInt<21>(BranchImm)); + *TargetPtr &= 0xff00001fU; + // Immediate:20:2 goes in bits 23:5 of Bcc, CBZ, CBNZ + or32le(TargetPtr, (BranchImm & 0x001FFFFC) << 3); + break; + } + case ELF::R_AARCH64_TSTBR14: { + uint64_t BranchImm = Value + Addend - FinalAddress; + + assert(isInt<16>(BranchImm)); + + *TargetPtr &= 0xfff8001fU; + // Immediate:15:2 goes in bits 18:5 of TBZ, TBNZ + or32le(TargetPtr, (BranchImm & 0x0FFFFFFC) << 3); + break; + } case ELF::R_AARCH64_CALL26: // fallthrough case ELF::R_AARCH64_JUMP26: { // Operation: S+A-P. Set Call or B immediate value to bits fff_fffc of the @@ -481,6 +519,33 @@ void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section, // from bits 11:4 of X or32AArch64Imm(TargetPtr, getBits(Value + Addend, 4, 11)); break; + case ELF::R_AARCH64_LD_PREL_LO19: { + // Operation: S + A - P + uint64_t Result = Value + Addend - FinalAddress; + + // "Check that -2^20 <= result < 2^20". + assert(isInt<21>(Result)); + + *TargetPtr &= 0xff00001fU; + // Immediate goes in bits 23:5 of LD imm instruction, taken + // from bits 20:2 of X + *TargetPtr |= ((Result & 0xffc) << (5 - 2)); + break; + } + case ELF::R_AARCH64_ADR_PREL_LO21: { + // Operation: S + A - P + uint64_t Result = Value + Addend - FinalAddress; + + // "Check that -2^20 <= result < 2^20". + assert(isInt<21>(Result)); + + *TargetPtr &= 0x9f00001fU; + // Immediate goes in bits 23:5, 30:29 of ADR imm instruction, taken + // from bits 20:0 of X + *TargetPtr |= ((Result & 0xffc) << (5 - 2)); + *TargetPtr |= (Result & 0x3) << 29; + break; + } } } @@ -886,14 +951,17 @@ void RuntimeDyldELF::resolveBPFRelocation(const SectionEntry &Section, report_fatal_error("Relocation type not implemented yet!"); break; case ELF::R_BPF_NONE: + case ELF::R_BPF_64_64: + case ELF::R_BPF_64_32: + case ELF::R_BPF_64_NODYLD32: break; - case ELF::R_BPF_64_64: { + case ELF::R_BPF_64_ABS64: { write(isBE, Section.getAddressWithOffset(Offset), Value + Addend); LLVM_DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend)) << " at " << format("%p\n", Section.getAddressWithOffset(Offset))); break; } - case ELF::R_BPF_64_32: { + case ELF::R_BPF_64_ABS32: { Value += Addend; assert(Value <= UINT32_MAX); write(isBE, Section.getAddressWithOffset(Offset), static_cast<uint32_t>(Value)); @@ -1204,7 +1272,9 @@ RuntimeDyldELF::processRelocationRef( LLVM_DEBUG(dbgs() << "\t\tSectionID: " << SectionID << " Offset: " << Offset << "\n"); if ((Arch == Triple::aarch64 || Arch == Triple::aarch64_be)) { - if (RelType == ELF::R_AARCH64_CALL26 || RelType == ELF::R_AARCH64_JUMP26) { + if ((RelType == ELF::R_AARCH64_CALL26 || + RelType == ELF::R_AARCH64_JUMP26) && + MemMgr.allowStubAllocation()) { resolveAArch64Branch(SectionID, Value, RelI, Stubs); } else if (RelType == ELF::R_AARCH64_ADR_GOT_PAGE) { // Craete new GOT entry or find existing one. If GOT entry is @@ -1667,7 +1737,7 @@ RuntimeDyldELF::processRelocationRef( // equivalent to the usual PLT implementation except that we use the stub // mechanism in RuntimeDyld (which puts stubs at the end of the section) // rather than allocating a PLT section. - if (Value.SymbolName) { + if (Value.SymbolName && MemMgr.allowStubAllocation()) { // This is a call to an external function. // Look for an existing stub. SectionEntry *Section = &Sections[SectionID]; @@ -1712,9 +1782,9 @@ RuntimeDyldELF::processRelocationRef( resolveRelocation(*Section, Offset, StubAddress, ELF::R_X86_64_PC32, Addend); } else { - RelocationEntry RE(SectionID, Offset, ELF::R_X86_64_PC32, Value.Addend, - Value.Offset); - addRelocationForSection(RE, Value.SectionID); + Value.Addend += support::ulittle32_t::ref( + computePlaceholderAddress(SectionID, Offset)); + processSimpleRelocation(SectionID, Offset, ELF::R_X86_64_PC32, Value); } } else if (RelType == ELF::R_X86_64_GOTPCREL || RelType == ELF::R_X86_64_GOTPCRELX || diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index d34fae9aaf0c..a5bc181f8af9 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -462,16 +462,26 @@ public: loadObject(const object::ObjectFile &Obj) = 0; uint64_t getSectionLoadAddress(unsigned SectionID) const { - return Sections[SectionID].getLoadAddress(); + if (SectionID == AbsoluteSymbolSection) + return 0; + else + return Sections[SectionID].getLoadAddress(); } uint8_t *getSectionAddress(unsigned SectionID) const { - return Sections[SectionID].getAddress(); + if (SectionID == AbsoluteSymbolSection) + return nullptr; + else + return Sections[SectionID].getAddress(); } StringRef getSectionContent(unsigned SectionID) const { - return StringRef(reinterpret_cast<char *>(Sections[SectionID].getAddress()), - Sections[SectionID].getStubOffset() + getMaxStubSize()); + if (SectionID == AbsoluteSymbolSection) + return {}; + else + return StringRef( + reinterpret_cast<char *>(Sections[SectionID].getAddress()), + Sections[SectionID].getStubOffset() + getMaxStubSize()); } uint8_t* getSymbolLocalAddress(StringRef Name) const { @@ -519,9 +529,7 @@ public: for (auto &KV : GlobalSymbolTable) { auto SectionID = KV.second.getSectionID(); - uint64_t SectionAddr = 0; - if (SectionID != AbsoluteSymbolSection) - SectionAddr = getSectionLoadAddress(SectionID); + uint64_t SectionAddr = getSectionLoadAddress(SectionID); Result[KV.first()] = JITEvaluatedSymbol(SectionAddr + KV.second.getOffset(), KV.second.getFlags()); } |