diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine/JITLink')
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h | 30 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/ELF.cpp | 51 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp | 463 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/JITLink.cpp | 17 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp | 119 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h | 109 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp | 2 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachO.cpp | 26 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp | 82 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h | 28 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp | 34 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp | 131 |
12 files changed, 936 insertions, 156 deletions
diff --git a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h index b47a798c7603..82258a35a675 100644 --- a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h @@ -15,6 +15,8 @@ #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#define DEBUG_TYPE "jitlink" + namespace llvm { namespace jitlink { @@ -27,12 +29,25 @@ public: // the newly added ones, so just copy the existing blocks out. std::vector<Block *> Blocks(G.blocks().begin(), G.blocks().end()); + LLVM_DEBUG(dbgs() << "Creating GOT entries and stubs:\n"); + for (auto *B : Blocks) for (auto &E : B->edges()) - if (impl().isGOTEdge(E)) + if (impl().isGOTEdge(E)) { + LLVM_DEBUG({ + dbgs() << " Updating GOT edge "; + printEdge(dbgs(), *B, E, "<target GOT>"); + dbgs() << "\n"; + }); impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget())); - else if (impl().isExternalBranchEdge(E)) + } else if (impl().isExternalBranchEdge(E)) { + LLVM_DEBUG({ + dbgs() << " Updating external branch edge "; + printEdge(dbgs(), *B, E, "<target PC-rel>"); + dbgs() << "\n"; + }); impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget())); + } } protected: @@ -44,11 +59,17 @@ protected: // Build the entry if it doesn't exist. if (GOTEntryI == GOTEntries.end()) { auto &GOTEntry = impl().createGOTEntry(Target); + LLVM_DEBUG({ + dbgs() << " Created GOT entry for " << Target.getName() << ": " + << GOTEntry << "\n"; + }); GOTEntryI = GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; } assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol"); + LLVM_DEBUG( + { dbgs() << " Using GOT entry " << *GOTEntryI->second << "\n"; }); return *GOTEntryI->second; } @@ -59,10 +80,15 @@ protected: if (StubI == Stubs.end()) { auto &StubSymbol = impl().createStub(Target); + LLVM_DEBUG({ + dbgs() << " Created stub for " << Target.getName() << ": " + << StubSymbol << "\n"; + }); StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first; } assert(StubI != Stubs.end() && "Count not get stub symbol"); + LLVM_DEBUG({ dbgs() << " Using stub " << *StubI->second << "\n"; }); return *StubI->second; } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp new file mode 100644 index 000000000000..6160583b13fe --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp @@ -0,0 +1,51 @@ +//===-------------- ELF.cpp - JIT linker function for ELF -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// ELF jit-link function. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ELF.h" + +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" +#include <cstring> + +using namespace llvm; + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +void jitLink_ELF(std::unique_ptr<JITLinkContext> Ctx) { + + // We don't want to do full ELF validation here. We just verify it is elf'ish. + // Probably should parse into an elf header when we support more than x86 :) + + StringRef Data = Ctx->getObjectBuffer().getBuffer(); + if (Data.size() < llvm::ELF::EI_MAG3 + 1) { + Ctx->notifyFailed(make_error<JITLinkError>("Truncated ELF buffer")); + return; + } + + if (!memcmp(Data.data(), llvm::ELF::ElfMagic, strlen(llvm::ELF::ElfMagic))) { + if (Data.data()[llvm::ELF::EI_CLASS] == ELF::ELFCLASS64) { + return jitLink_ELF_x86_64(std::move(Ctx)); + } + } + + Ctx->notifyFailed(make_error<JITLinkError>("ELF magic not valid")); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp new file mode 100644 index 000000000000..505f03590b6b --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -0,0 +1,463 @@ +//===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ELF/x86-64 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" +#include "JITLinkGeneric.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Object/ELFObjectFile.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; + +static const char *CommonSectionName = "__common"; + +namespace llvm { +namespace jitlink { + +// This should become a template as the ELFFile is so a lot of this could become +// generic +class ELFLinkGraphBuilder_x86_64 { + +private: + Section *CommonSection = nullptr; + // TODO hack to get this working + // Find a better way + using SymbolTable = object::ELFFile<object::ELF64LE>::Elf_Shdr; + // For now we just assume + std::map<int32_t, Symbol *> JITSymbolTable; + + Section &getCommonSection() { + if (!CommonSection) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + CommonSection = &G->createSection(CommonSectionName, Prot); + } + return *CommonSection; + } + + static Expected<ELF_x86_64_Edges::ELFX86RelocationKind> + getRelocationKind(const uint32_t Type) { + switch (Type) { + case ELF::R_X86_64_PC32: + return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32; + } + return make_error<JITLinkError>("Unsupported x86-64 relocation:" + + formatv("{0:d}", Type)); + } + + std::unique_ptr<LinkGraph> G; + // This could be a template + const object::ELFFile<object::ELF64LE> &Obj; + object::ELFFile<object::ELF64LE>::Elf_Shdr_Range sections; + SymbolTable SymTab; + + bool isRelocatable() { return Obj.getHeader()->e_type == llvm::ELF::ET_REL; } + + support::endianness + getEndianness(const object::ELFFile<object::ELF64LE> &Obj) { + return Obj.isLE() ? support::little : support::big; + } + + // This could also just become part of a template + unsigned getPointerSize(const object::ELFFile<object::ELF64LE> &Obj) { + return Obj.getHeader()->getFileClass() == ELF::ELFCLASS64 ? 8 : 4; + } + + // We don't technically need this right now + // But for now going to keep it as it helps me to debug things + + Error createNormalizedSymbols() { + LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n"); + + for (auto SecRef : sections) { + if (SecRef.sh_type != ELF::SHT_SYMTAB && + SecRef.sh_type != ELF::SHT_DYNSYM) + continue; + + auto Symbols = Obj.symbols(&SecRef); + // TODO: Currently I use this function to test things + // I also want to leave it to see if its common between MACH and elf + // so for now I just want to continue even if there is an error + if (errorToBool(Symbols.takeError())) + continue; + + auto StrTabSec = Obj.getSection(SecRef.sh_link); + if (!StrTabSec) + return StrTabSec.takeError(); + auto StringTable = Obj.getStringTable(*StrTabSec); + if (!StringTable) + return StringTable.takeError(); + + for (auto SymRef : *Symbols) { + Optional<StringRef> Name; + uint64_t Size = 0; + + // FIXME: Read size. + (void)Size; + + if (auto NameOrErr = SymRef.getName(*StringTable)) + Name = *NameOrErr; + else + return NameOrErr.takeError(); + + LLVM_DEBUG({ + dbgs() << " "; + if (!Name) + dbgs() << "<anonymous symbol>"; + else + dbgs() << *Name; + dbgs() << ": value = " << formatv("{0:x16}", SymRef.getValue()) + << ", type = " << formatv("{0:x2}", SymRef.getType()) + << ", binding = " << SymRef.getBinding() + << ", size =" << Size; + dbgs() << "\n"; + }); + } + } + return Error::success(); + } + + Error createNormalizedSections() { + LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); + for (auto &SecRef : sections) { + auto Name = Obj.getSectionName(&SecRef); + if (!Name) + return Name.takeError(); + sys::Memory::ProtectionFlags Prot; + if (SecRef.sh_flags & ELF::SHF_EXECINSTR) { + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + } else { + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + } + uint64_t Address = SecRef.sh_addr; + uint64_t Size = SecRef.sh_size; + uint64_t Flags = SecRef.sh_flags; + uint64_t Alignment = SecRef.sh_addralign; + const char *Data = nullptr; + // TODO: figure out what it is that has 0 size no name and address + // 0000-0000 + if (Size == 0) + continue; + + // FIXME: Use flags. + (void)Flags; + + LLVM_DEBUG({ + dbgs() << " " << *Name << ": " << formatv("{0:x16}", Address) << " -- " + << formatv("{0:x16}", Address + Size) << ", align: " << Alignment + << " Flags:" << Flags << "\n"; + }); + + if (SecRef.sh_type != ELF::SHT_NOBITS) { + // .sections() already checks that the data is not beyond the end of + // file + auto contents = Obj.getSectionContentsAsArray<char>(&SecRef); + if (!contents) + return contents.takeError(); + + Data = contents->data(); + // TODO protection flags. + // for now everything is + auto §ion = G->createSection(*Name, Prot); + // Do this here because we have it, but move it into graphify later + G->createContentBlock(section, StringRef(Data, Size), Address, + Alignment, 0); + if (SecRef.sh_type == ELF::SHT_SYMTAB) + // TODO: Dynamic? + SymTab = SecRef; + } + } + + return Error::success(); + } + + Error addRelocations() { + LLVM_DEBUG(dbgs() << "Adding relocations\n"); + // TODO a partern is forming of iterate some sections but only give me + // ones I am interested, i should abstract that concept some where + for (auto &SecRef : sections) { + if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL) + continue; + // TODO can the elf obj file do this for me? + if (SecRef.sh_type == ELF::SHT_REL) + return make_error<llvm::StringError>("Shouldn't have REL in x64", + llvm::inconvertibleErrorCode()); + + auto RelSectName = Obj.getSectionName(&SecRef); + if (!RelSectName) + return RelSectName.takeError(); + // Deal with .eh_frame later + if (*RelSectName == StringRef(".rela.eh_frame")) + continue; + + auto UpdateSection = Obj.getSection(SecRef.sh_info); + if (!UpdateSection) + return UpdateSection.takeError(); + + auto UpdateSectionName = Obj.getSectionName(*UpdateSection); + if (!UpdateSectionName) + return UpdateSectionName.takeError(); + + auto JITSection = G->findSectionByName(*UpdateSectionName); + if (!JITSection) + return make_error<llvm::StringError>( + "Refencing a a section that wasn't added to graph" + + *UpdateSectionName, + llvm::inconvertibleErrorCode()); + + auto Relocations = Obj.relas(&SecRef); + if (!Relocations) + return Relocations.takeError(); + + for (const auto &Rela : *Relocations) { + auto Type = Rela.getType(false); + + LLVM_DEBUG({ + dbgs() << "Relocation Type: " << Type << "\n" + << "Name: " << Obj.getRelocationTypeName(Type) << "\n"; + }); + + auto Symbol = Obj.getRelocationSymbol(&Rela, &SymTab); + if (!Symbol) + return Symbol.takeError(); + + auto BlockToFix = *(JITSection->blocks().begin()); + auto TargetSymbol = JITSymbolTable[(*Symbol)->st_shndx]; + uint64_t Addend = Rela.r_addend; + JITTargetAddress FixupAddress = + (*UpdateSection)->sh_addr + Rela.r_offset; + + LLVM_DEBUG({ + dbgs() << "Processing relocation at " + << format("0x%016" PRIx64, FixupAddress) << "\n"; + }); + auto Kind = getRelocationKind(Type); + if (!Kind) + return Kind.takeError(); + + LLVM_DEBUG({ + Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, + Addend); + // TODO a mapping of KIND => type then call getRelocationTypeName4 + printEdge(dbgs(), *BlockToFix, GE, StringRef("")); + dbgs() << "\n"; + }); + BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + *TargetSymbol, Addend); + } + } + return Error::success(); + } + + Error graphifyRegularSymbols() { + + // TODO: ELF supports beyond SHN_LORESERVE, + // need to perf test how a vector vs map handles those cases + + std::vector<std::vector<object::ELFFile<object::ELF64LE>::Elf_Shdr_Range *>> + SecIndexToSymbols; + + LLVM_DEBUG(dbgs() << "Creating graph symbols...\n"); + + for (auto SecRef : sections) { + + if (SecRef.sh_type != ELF::SHT_SYMTAB && + SecRef.sh_type != ELF::SHT_DYNSYM) + continue; + auto Symbols = Obj.symbols(&SecRef); + if (!Symbols) + return Symbols.takeError(); + + auto StrTabSec = Obj.getSection(SecRef.sh_link); + if (!StrTabSec) + return StrTabSec.takeError(); + auto StringTable = Obj.getStringTable(*StrTabSec); + if (!StringTable) + return StringTable.takeError(); + auto Name = Obj.getSectionName(&SecRef); + if (!Name) + return Name.takeError(); + auto Section = G->findSectionByName(*Name); + if (!Section) + return make_error<llvm::StringError>("Could not find a section", + llvm::inconvertibleErrorCode()); + // we only have one for now + auto blocks = Section->blocks(); + if (blocks.empty()) + return make_error<llvm::StringError>("Section has no block", + llvm::inconvertibleErrorCode()); + + for (auto SymRef : *Symbols) { + auto Type = SymRef.getType(); + if (Type == ELF::STT_NOTYPE || Type == ELF::STT_FILE) + continue; + // these should do it for now + // if(Type != ELF::STT_NOTYPE && + // Type != ELF::STT_OBJECT && + // Type != ELF::STT_FUNC && + // Type != ELF::STT_SECTION && + // Type != ELF::STT_COMMON) { + // continue; + // } + std::pair<Linkage, Scope> bindings; + auto Name = SymRef.getName(*StringTable); + // I am not sure on If this is going to hold as an invariant. Revisit. + if (!Name) + return Name.takeError(); + // TODO: weak and hidden + if (SymRef.isExternal()) + bindings = {Linkage::Strong, Scope::Default}; + else + bindings = {Linkage::Strong, Scope::Local}; + + if (SymRef.isDefined() && + (Type == ELF::STT_FUNC || Type == ELF::STT_OBJECT)) { + + auto DefinedSection = Obj.getSection(SymRef.st_shndx); + if (!DefinedSection) + return DefinedSection.takeError(); + auto sectName = Obj.getSectionName(*DefinedSection); + if (!sectName) + return Name.takeError(); + + auto JitSection = G->findSectionByName(*sectName); + if (!JitSection) + return make_error<llvm::StringError>( + "Could not find a section", llvm::inconvertibleErrorCode()); + auto bs = JitSection->blocks(); + if (bs.empty()) + return make_error<llvm::StringError>( + "Section has no block", llvm::inconvertibleErrorCode()); + + auto B = *bs.begin(); + LLVM_DEBUG({ dbgs() << " " << *Name << ": "; }); + + auto &S = G->addDefinedSymbol( + *B, SymRef.getValue(), *Name, SymRef.st_size, bindings.first, + bindings.second, SymRef.getType() == ELF::STT_FUNC, false); + JITSymbolTable[SymRef.st_shndx] = &S; + } + //TODO: The following has to be implmented. + // leaving commented out to save time for future patchs + /* + G->addAbsoluteSymbol(*Name, SymRef.getValue(), SymRef.st_size, + Linkage::Strong, Scope::Default, false); + + if(SymRef.isCommon()) { + G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, 0, + SymRef.getValue(), false); + } + + + //G->addExternalSymbol(*Name, SymRef.st_size, Linkage::Strong); + */ + } + } + return Error::success(); + } + +public: + ELFLinkGraphBuilder_x86_64(std::string filename, + const object::ELFFile<object::ELF64LE> &Obj) + : G(std::make_unique<LinkGraph>(filename, getPointerSize(Obj), + getEndianness(Obj))), + Obj(Obj) {} + + Expected<std::unique_ptr<LinkGraph>> buildGraph() { + // Sanity check: we only operate on relocatable objects. + if (!isRelocatable()) + return make_error<JITLinkError>("Object is not a relocatable ELF"); + + auto Secs = Obj.sections(); + + if (!Secs) { + return Secs.takeError(); + } + sections = *Secs; + + if (auto Err = createNormalizedSections()) + return std::move(Err); + + if (auto Err = createNormalizedSymbols()) + return std::move(Err); + + if (auto Err = graphifyRegularSymbols()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); + } +}; + +class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> { + friend class JITLinker<ELFJITLinker_x86_64>; + +public: + ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(PassConfig)) {} + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { return StringRef(); } + + Expected<std::unique_ptr<LinkGraph>> + buildGraph(MemoryBufferRef ObjBuffer) override { + auto ELFObj = object::ObjectFile::createELFObjectFile(ObjBuffer); + if (!ELFObj) + return ELFObj.takeError(); + + auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); + std::string fileName(ELFObj->get()->getFileName()); + return ELFLinkGraphBuilder_x86_64(std::move(fileName), + *ELFObjFile.getELFFile()) + .buildGraph(); + } + + Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + using namespace ELF_x86_64_Edges; + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); + switch (E.getKind()) { + + case ELFX86RelocationKind::PCRel32: + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + // verify + *(support::little32_t *)FixupPtr = Value; + break; + } + return Error::success(); + } +}; + +void jitLink_ELF_x86_64(std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; + Triple TT("x86_64-linux"); + // Construct a JITLinker and run the link function. + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + + if (auto Err = Ctx->modifyPassConfig(TT, Config)) + return Ctx->notifyFailed(std::move(Err)); + + ELFJITLinker_x86_64::link(std::move(Ctx), std::move(Config)); +} +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index 6c924f889577..5105ec495148 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -10,6 +10,7 @@ #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/BinaryFormat/Magic.h" +#include "llvm/ExecutionEngine/JITLink/ELF.h" #include "llvm/ExecutionEngine/JITLink/MachO.h" #include "llvm/Support/Format.h" #include "llvm/Support/ManagedStatic.h" @@ -180,18 +181,14 @@ Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex, // Copy edges to NewBlock (recording their iterators so that we can remove // them from B), and update of Edges remaining on B. std::vector<Block::edge_iterator> EdgesToRemove; - for (auto I = B.edges().begin(), E = B.edges().end(); I != E; ++I) { + for (auto I = B.edges().begin(); I != B.edges().end();) { if (I->getOffset() < SplitIndex) { NewBlock.addEdge(*I); - EdgesToRemove.push_back(I); - } else + I = B.removeEdge(I); + } else { I->setOffset(I->getOffset() - SplitIndex); - } - - // Remove edges that were transfered to NewBlock from B. - while (!EdgesToRemove.empty()) { - B.removeEdge(EdgesToRemove.back()); - EdgesToRemove.pop_back(); + ++I; + } } } @@ -304,6 +301,8 @@ void jitLink(std::unique_ptr<JITLinkContext> Ctx) { switch (Magic) { case file_magic::macho_object: return jitLink_MachO(std::move(Ctx)); + case file_magic::elf_relocatable: + return jitLink_ELF(std::move(Ctx)); default: Ctx->notifyFailed(make_error<JITLinkError>("Unsupported file format")); }; diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 7b594fd2c0ea..1d76a49939dc 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -24,6 +24,8 @@ JITLinkerBase::~JITLinkerBase() {} void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { + LLVM_DEBUG({ dbgs() << "Building jitlink graph for new input...\n"; }); + // Build the link graph. if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) G = std::move(*GraphOrErr); @@ -31,6 +33,10 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { return Ctx->notifyFailed(GraphOrErr.takeError()); assert(G && "Graph should have been created by buildGraph above"); + LLVM_DEBUG({ + dbgs() << "Starting link phase 1 for graph " << G->getName() << "\n"; + }); + // Prune and optimize the graph. if (auto Err = runPasses(Passes.PrePrunePasses)) return Ctx->notifyFailed(std::move(Err)); @@ -59,10 +65,17 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { return Ctx->notifyFailed(std::move(Err)); // Notify client that the defined symbols have been assigned addresses. + LLVM_DEBUG( + { dbgs() << "Resolving symbols defined in " << G->getName() << "\n"; }); Ctx->notifyResolved(*G); auto ExternalSymbols = getExternalSymbolNames(); + LLVM_DEBUG({ + dbgs() << "Issuing lookup for external symbols for " << G->getName() + << " (may trigger materialization/linking of other graphs)...\n"; + }); + // We're about to hand off ownership of ourself to the continuation. Grab a // pointer to the context so that we can call it to initiate the lookup. // @@ -87,6 +100,11 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, Expected<AsyncLookupResult> LR, SegmentLayoutMap Layout) { + + LLVM_DEBUG({ + dbgs() << "Starting link phase 2 for graph " << G->getName() << "\n"; + }); + // If the lookup failed, bail out. if (!LR) return deallocateAndBailOut(LR.takeError()); @@ -94,13 +112,25 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, // Assign addresses to external addressables. applyLookupResult(*LR); + // Copy block content to working memory. + copyBlockContentToWorkingMemory(Layout, *Alloc); + + LLVM_DEBUG({ + dbgs() << "Link graph \"" << G->getName() + << "\" before post-allocation passes:\n"; + dumpGraph(dbgs()); + }); + + if (auto Err = runPasses(Passes.PostAllocationPasses)) + return deallocateAndBailOut(std::move(Err)); + LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n"; dumpGraph(dbgs()); }); - // Copy block content to working memory and fix up. - if (auto Err = copyAndFixUpBlocks(Layout, *Alloc)) + // Fix up block content. + if (auto Err = fixUpBlocks(*G)) return deallocateAndBailOut(std::move(Err)); LLVM_DEBUG({ @@ -122,9 +152,16 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, } void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err) { + + LLVM_DEBUG({ + dbgs() << "Starting link phase 3 for graph " << G->getName() << "\n"; + }); + if (Err) return deallocateAndBailOut(std::move(Err)); Ctx->notifyFinalized(std::move(Alloc)); + + LLVM_DEBUG({ dbgs() << "Link of graph " << G->getName() << " complete\n"; }); } Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) { @@ -165,7 +202,7 @@ JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() { } LLVM_DEBUG({ - dbgs() << "Segment ordering:\n"; + dbgs() << "Computed segment ordering:\n"; for (auto &KV : Layout) { dbgs() << " Segment " << static_cast<sys::Memory::ProtectionFlags>(KV.first) << ":\n"; @@ -230,11 +267,12 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { return AllocOrErr.takeError(); LLVM_DEBUG({ - dbgs() << "JIT linker got working memory:\n"; + dbgs() << "JIT linker got memory (working -> target):\n"; for (auto &KV : Layout) { auto Prot = static_cast<sys::Memory::ProtectionFlags>(KV.first); dbgs() << " " << Prot << ": " - << (const void *)Alloc->getWorkingMemory(Prot).data() << "\n"; + << (const void *)Alloc->getWorkingMemory(Prot).data() << " -> " + << formatv("{0:x16}", Alloc->getTargetMemory(Prot)) << "\n"; } }); @@ -302,6 +340,77 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { "All strong external symbols should have been resolved by now"); } +void JITLinkerBase::copyBlockContentToWorkingMemory( + const SegmentLayoutMap &Layout, JITLinkMemoryManager::Allocation &Alloc) { + + LLVM_DEBUG(dbgs() << "Copying block content:\n"); + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + auto SegMem = + Alloc.getWorkingMemory(static_cast<sys::Memory::ProtectionFlags>(Prot)); + char *LastBlockEnd = SegMem.data(); + char *BlockDataPtr = LastBlockEnd; + + LLVM_DEBUG({ + dbgs() << " Processing segment " + << static_cast<sys::Memory::ProtectionFlags>(Prot) << " [ " + << (const void *)SegMem.data() << " .. " + << (const void *)((char *)SegMem.data() + SegMem.size()) + << " ]\n Processing content sections:\n"; + }); + + for (auto *B : SegLayout.ContentBlocks) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); + + // Pad to alignment/alignment-offset. + BlockDataPtr = alignToBlock(BlockDataPtr, *B); + + LLVM_DEBUG({ + dbgs() << " Bumped block pointer to " << (const void *)BlockDataPtr + << " to meet block alignment " << B->getAlignment() + << " and alignment offset " << B->getAlignmentOffset() << "\n"; + }); + + // Zero pad up to alignment. + LLVM_DEBUG({ + if (LastBlockEnd != BlockDataPtr) + dbgs() << " Zero padding from " << (const void *)LastBlockEnd + << " to " << (const void *)BlockDataPtr << "\n"; + }); + + while (LastBlockEnd != BlockDataPtr) + *LastBlockEnd++ = 0; + + // Copy initial block content. + LLVM_DEBUG({ + dbgs() << " Copying block " << *B << " content, " + << B->getContent().size() << " bytes, from " + << (const void *)B->getContent().data() << " to " + << (const void *)BlockDataPtr << "\n"; + }); + memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size()); + + // Point the block's content to the fixed up buffer. + B->setContent(StringRef(BlockDataPtr, B->getContent().size())); + + // Update block end pointer. + LastBlockEnd = BlockDataPtr + B->getContent().size(); + BlockDataPtr = LastBlockEnd; + } + + // Zero pad the rest of the segment. + LLVM_DEBUG({ + dbgs() << " Zero padding end of segment from " + << (const void *)LastBlockEnd << " to " + << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; + }); + while (LastBlockEnd != SegMem.data() + SegMem.size()) + *LastBlockEnd++ = 0; + } +} + void JITLinkerBase::deallocateAndBailOut(Error Err) { assert(Err && "Should not be bailing out on success value"); assert(Alloc && "can not call deallocateAndBailOut before allocation"); diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index d5687b7afc96..87e5e8bbc98d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -80,13 +80,13 @@ protected: // For debug dumping of the link graph. virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; - // Alight a JITTargetAddress to conform with block alignment requirements. + // Align a JITTargetAddress to conform with block alignment requirements. static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) { uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment(); return Addr + Delta; } - // Alight a pointer to conform with block alignment requirements. + // Align a pointer to conform with block alignment requirements. static char *alignToBlock(char *P, Block &B) { uint64_t PAddr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(P)); uint64_t Delta = (B.getAlignmentOffset() - PAddr) % B.getAlignment(); @@ -100,14 +100,14 @@ private: // Copy block contents and apply relocations. // Implemented in JITLinker. - virtual Error - copyAndFixUpBlocks(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const = 0; + virtual Error fixUpBlocks(LinkGraph &G) const = 0; SegmentLayoutMap layOutBlocks(); Error allocateSegments(const SegmentLayoutMap &Layout); JITLinkContext::LookupMap getExternalSymbolNames() const; void applyLookupResult(AsyncLookupResult LR); + void copyBlockContentToWorkingMemory(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc); void deallocateAndBailOut(Error Err); void dumpGraph(raw_ostream &OS); @@ -144,88 +144,25 @@ private: return static_cast<const LinkerImpl &>(*this); } - Error - copyAndFixUpBlocks(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const override { - LLVM_DEBUG(dbgs() << "Copying and fixing up blocks:\n"); - for (auto &KV : Layout) { - auto &Prot = KV.first; - auto &SegLayout = KV.second; - - auto SegMem = Alloc.getWorkingMemory( - static_cast<sys::Memory::ProtectionFlags>(Prot)); - char *LastBlockEnd = SegMem.data(); - char *BlockDataPtr = LastBlockEnd; - - LLVM_DEBUG({ - dbgs() << " Processing segment " - << static_cast<sys::Memory::ProtectionFlags>(Prot) << " [ " - << (const void *)SegMem.data() << " .. " - << (const void *)((char *)SegMem.data() + SegMem.size()) - << " ]\n Processing content sections:\n"; - }); - - for (auto *B : SegLayout.ContentBlocks) { - LLVM_DEBUG(dbgs() << " " << *B << ":\n"); - - // Pad to alignment/alignment-offset. - BlockDataPtr = alignToBlock(BlockDataPtr, *B); - - LLVM_DEBUG({ - dbgs() << " Bumped block pointer to " - << (const void *)BlockDataPtr << " to meet block alignment " - << B->getAlignment() << " and alignment offset " - << B->getAlignmentOffset() << "\n"; - }); - - // Zero pad up to alignment. - LLVM_DEBUG({ - if (LastBlockEnd != BlockDataPtr) - dbgs() << " Zero padding from " << (const void *)LastBlockEnd - << " to " << (const void *)BlockDataPtr << "\n"; - }); - - while (LastBlockEnd != BlockDataPtr) - *LastBlockEnd++ = 0; - - // Copy initial block content. - LLVM_DEBUG({ - dbgs() << " Copying block " << *B << " content, " - << B->getContent().size() << " bytes, from " - << (const void *)B->getContent().data() << " to " - << (const void *)BlockDataPtr << "\n"; - }); - memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size()); - - // Copy Block data and apply fixups. - LLVM_DEBUG(dbgs() << " Applying fixups.\n"); - for (auto &E : B->edges()) { - - // Skip non-relocation edges. - if (!E.isRelocation()) - continue; - - // Dispatch to LinkerImpl for fixup. - if (auto Err = impl().applyFixup(*B, E, BlockDataPtr)) - return Err; - } - - // Point the block's content to the fixed up buffer. - B->setContent(StringRef(BlockDataPtr, B->getContent().size())); - - // Update block end pointer. - LastBlockEnd = BlockDataPtr + B->getContent().size(); - BlockDataPtr = LastBlockEnd; - } + Error fixUpBlocks(LinkGraph &G) const override { + LLVM_DEBUG(dbgs() << "Fixing up blocks:\n"); + + for (auto *B : G.blocks()) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); + + // Copy Block data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : B->edges()) { - // Zero pad the rest of the segment. - LLVM_DEBUG({ - dbgs() << " Zero padding end of segment from " - << (const void *)LastBlockEnd << " to " - << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; - }); - while (LastBlockEnd != SegMem.data() + SegMem.size()) - *LastBlockEnd++ = 0; + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // Dispatch to LinkerImpl for fixup. + auto *BlockData = const_cast<char *>(B->getContent().data()); + if (auto Err = impl().applyFixup(*B, E, BlockData)) + return Err; + } } return Error::success(); diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index 9e0d207e8bdb..68ec9d79af9b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -32,7 +32,7 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { } JITTargetAddress getTargetMemory(ProtectionFlags Seg) override { assert(SegBlocks.count(Seg) && "No allocation for segment"); - return reinterpret_cast<JITTargetAddress>(SegBlocks[Seg].base()); + return pointerToJITTargetAddress(SegBlocks[Seg].base()); } void finalizeAsync(FinalizeContinuation OnFinalize) override { OnFinalize(applyProtections()); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO.cpp index 58bc0f56e155..b3e45868ab22 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO.cpp @@ -16,9 +16,9 @@ #include "llvm/BinaryFormat/MachO.h" #include "llvm/ExecutionEngine/JITLink/MachO_arm64.h" #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" -#include "llvm/Support/Endian.h" #include "llvm/Support/Format.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SwapByteOrder.h" using namespace llvm; @@ -34,7 +34,9 @@ void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) { StringRef Data = Ctx->getObjectBuffer().getBuffer(); if (Data.size() < 4) { - Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer")); + StringRef BufferName = Ctx->getObjectBuffer().getBufferIdentifier(); + Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer \"" + + BufferName + "\"")); return; } @@ -51,20 +53,26 @@ void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) { make_error<JITLinkError>("MachO 32-bit platforms not supported")); return; } else if (Magic == MachO::MH_MAGIC_64 || Magic == MachO::MH_CIGAM_64) { - MachO::mach_header_64 Header; - memcpy(&Header, Data.data(), sizeof(MachO::mach_header_64)); + if (Data.size() < sizeof(MachO::mach_header_64)) { + StringRef BufferName = Ctx->getObjectBuffer().getBufferIdentifier(); + Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer \"" + + BufferName + "\"")); + return; + } + + // Read the CPU type from the header. + uint32_t CPUType; + memcpy(&CPUType, Data.data() + 4, sizeof(uint32_t)); if (Magic == MachO::MH_CIGAM_64) - swapStruct(Header); + CPUType = ByteSwap_32(CPUType); LLVM_DEBUG({ - dbgs() << "jitLink_MachO: cputype = " - << format("0x%08" PRIx32, Header.cputype) - << ", cpusubtype = " << format("0x%08" PRIx32, Header.cpusubtype) + dbgs() << "jitLink_MachO: cputype = " << format("0x%08" PRIx32, CPUType) << "\n"; }); - switch (Header.cputype) { + switch (CPUType) { case MachO::CPU_TYPE_ARM64: return jitLink_MachO_arm64(std::move(Ctx)); case MachO::CPU_TYPE_X86_64: diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp index 701f108a9a21..fa3f403b717c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -47,8 +47,8 @@ Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() { MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj) : Obj(Obj), - G(std::make_unique<LinkGraph>(Obj.getFileName(), getPointerSize(Obj), - getEndianness(Obj))) {} + G(std::make_unique<LinkGraph>(std::string(Obj.getFileName()), + getPointerSize(Obj), getEndianness(Obj))) {} void MachOLinkGraphBuilder::addCustomSectionParser( StringRef SectionName, SectionParserFunction Parser) { @@ -64,12 +64,14 @@ Linkage MachOLinkGraphBuilder::getLinkage(uint16_t Desc) { } Scope MachOLinkGraphBuilder::getScope(StringRef Name, uint8_t Type) { - if (Name.startswith("l")) - return Scope::Local; if (Type & MachO::N_PEXT) return Scope::Hidden; - if (Type & MachO::N_EXT) - return Scope::Default; + if (Type & MachO::N_EXT) { + if (Name.startswith("l")) + return Scope::Hidden; + else + return Scope::Default; + } return Scope::Local; } @@ -77,6 +79,11 @@ bool MachOLinkGraphBuilder::isAltEntry(const NormalizedSymbol &NSym) { return NSym.Desc & MachO::N_ALT_ENTRY; } +bool MachOLinkGraphBuilder::isDebugSection(const NormalizedSection &NSec) { + return (NSec.Flags & MachO::S_ATTR_DEBUG && + strcmp(NSec.SegName, "__DWARF") == 0); +} + unsigned MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { return Obj.is64Bit() ? 8 : 4; @@ -116,6 +123,11 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { const MachO::section_64 &Sec64 = Obj.getSection64(SecRef.getRawDataRefImpl()); + memcpy(&NSec.SectName, &Sec64.sectname, 16); + NSec.SectName[16] = '\0'; + memcpy(&NSec.SegName, Sec64.segname, 16); + NSec.SegName[16] = '\0'; + NSec.Address = Sec64.addr; NSec.Size = Sec64.size; NSec.Alignment = 1ULL << Sec64.align; @@ -123,6 +135,12 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { DataOffset = Sec64.offset; } else { const MachO::section &Sec32 = Obj.getSection(SecRef.getRawDataRefImpl()); + + memcpy(&NSec.SectName, &Sec32.sectname, 16); + NSec.SectName[16] = '\0'; + memcpy(&NSec.SegName, Sec32.segname, 16); + NSec.SegName[16] = '\0'; + NSec.Address = Sec32.addr; NSec.Size = Sec32.size; NSec.Alignment = 1ULL << Sec32.align; @@ -162,7 +180,14 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | sys::Memory::MF_WRITE); - NSec.GraphSection = &G->createSection(*Name, Prot); + if (!isDebugSection(NSec)) + NSec.GraphSection = &G->createSection(*Name, Prot); + else + LLVM_DEBUG({ + dbgs() << " " << *Name + << " is a debug section: No graph section will be created.\n"; + }); + IndexToSection.insert(std::make_pair(SecIndex, std::move(NSec))); } @@ -189,12 +214,12 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { auto &Next = *Sections[I + 1]; if (Next.Address < Cur.Address + Cur.Size) return make_error<JITLinkError>( - "Address range for section " + Cur.GraphSection->getName() + - formatv(" [ {0:x16} -- {1:x16} ] ", Cur.Address, - Cur.Address + Cur.Size) + - "overlaps " + - formatv(" [ {0:x16} -- {1:x16} ] ", Next.Address, - Next.Address + Next.Size)); + "Address range for section " + + formatv("\"{0}/{1}\" [ {2:x16} -- {3:x16} ] ", Cur.SegName, + Cur.SectName, Cur.Address, Cur.Address + Cur.Size) + + "overlaps section \"" + Next.SegName + "/" + Next.SectName + "\"" + + formatv("\"{0}/{1}\" [ {2:x16} -- {3:x16} ] ", Next.SegName, + Next.SectName, Next.Address, Next.Address + Next.Size)); } return Error::success(); @@ -260,21 +285,28 @@ Error MachOLinkGraphBuilder::createNormalizedSymbols() { }); // If this symbol has a section, sanity check that the addresses line up. - NormalizedSection *NSec = nullptr; if (Sect != 0) { - if (auto NSecOrErr = findSectionByIndex(Sect - 1)) - NSec = &*NSecOrErr; - else - return NSecOrErr.takeError(); + auto NSec = findSectionByIndex(Sect - 1); + if (!NSec) + return NSec.takeError(); if (Value < NSec->Address || Value > NSec->Address + NSec->Size) return make_error<JITLinkError>("Symbol address does not fall within " "section"); + + if (!NSec->GraphSection) { + LLVM_DEBUG({ + dbgs() << " Skipping: Symbol is in section " << NSec->SegName << "/" + << NSec->SectName + << " which has no associated graph section.\n"; + }); + continue; + } } IndexToSymbol[SymbolIndex] = &createNormalizedSymbol(*Name, Value, Type, Sect, Desc, - getLinkage(Type), getScope(*Name, Type)); + getLinkage(Desc), getScope(*Name, Type)); } return Error::success(); @@ -362,6 +394,14 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { auto SecIndex = KV.first; auto &NSec = KV.second; + if (!NSec.GraphSection) { + LLVM_DEBUG({ + dbgs() << " " << NSec.SegName << "/" << NSec.SectName + << " has no graph section. Skipping.\n"; + }); + continue; + } + // Skip sections with custom parsers. if (CustomSectionParserFunctions.count(NSec.GraphSection->getName())) { LLVM_DEBUG({ @@ -524,6 +564,10 @@ Error MachOLinkGraphBuilder::graphifySectionsWithCustomParsers() { for (auto &KV : IndexToSection) { auto &NSec = KV.second; + // Skip non-graph sections. + if (!NSec.GraphSection) + continue; + auto HI = CustomSectionParserFunctions.find(NSec.GraphSection->getName()); if (HI != CustomSectionParserFunctions.end()) { auto &Parse = HI->second; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index 91b1d5a22387..dd3bcf27494c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -13,6 +13,8 @@ #ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H #define LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "EHFrameSupportImpl.h" @@ -58,6 +60,8 @@ protected: Symbol *GraphSymbol = nullptr; }; + // Normalized section representation. Section and segment names are guaranteed + // to be null-terminated, hence the extra bytes on SegName and SectName. class NormalizedSection { friend class MachOLinkGraphBuilder; @@ -65,12 +69,14 @@ protected: NormalizedSection() = default; public: - Section *GraphSection = nullptr; + char SectName[17]; + char SegName[17]; uint64_t Address = 0; uint64_t Size = 0; uint64_t Alignment = 0; uint32_t Flags = 0; const char *Data = nullptr; + Section *GraphSection = nullptr; }; using SectionParserFunction = std::function<Error(NormalizedSection &S)>; @@ -110,7 +116,7 @@ protected: auto I = IndexToSection.find(Index); if (I == IndexToSection.end()) return make_error<JITLinkError>("No section recorded for index " + - formatv("{0:u}", Index)); + formatv("{0:d}", Index)); return I->second; } @@ -123,7 +129,7 @@ protected: auto *Sym = IndexToSymbol[Index]; if (!Sym) return make_error<JITLinkError>("No symbol at index " + - formatv("{0:u}", Index)); + formatv("{0:d}", Index)); return *Sym; } @@ -151,6 +157,22 @@ protected: static Scope getScope(StringRef Name, uint8_t Type); static bool isAltEntry(const NormalizedSymbol &NSym); + static bool isDebugSection(const NormalizedSection &NSec); + + MachO::relocation_info + getRelocationInfo(const object::relocation_iterator RelItr) { + MachO::any_relocation_info ARI = + getObject().getRelocation(RelItr->getRawDataRefImpl()); + MachO::relocation_info RI; + RI.r_address = ARI.r_word0; + RI.r_symbolnum = ARI.r_word1 & 0xffffff; + RI.r_pcrel = (ARI.r_word1 >> 24) & 1; + RI.r_length = (ARI.r_word1 >> 25) & 3; + RI.r_extern = (ARI.r_word1 >> 27) & 1; + RI.r_type = (ARI.r_word1 >> 28); + return RI; + } + private: static unsigned getPointerSize(const object::MachOObjectFile &Obj); static support::endianness getEndianness(const object::MachOObjectFile &Obj); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 944767449ce2..463845a5b8cb 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -92,15 +92,6 @@ private: ", length=" + formatv("{0:d}", RI.r_length)); } - MachO::relocation_info - getRelocationInfo(const object::relocation_iterator RelItr) { - MachO::any_relocation_info ARI = - getObject().getRelocation(RelItr->getRawDataRefImpl()); - MachO::relocation_info RI; - memcpy(&RI, &ARI, sizeof(MachO::relocation_info)); - return RI; - } - using PairRelocInfo = std::tuple<MachOARM64RelocationKind, Symbol *, uint64_t>; @@ -194,6 +185,28 @@ private: JITTargetAddress SectionAddress = S.getAddress(); + // Skip relocations virtual sections. + if (S.isVirtual()) { + if (S.relocation_begin() != S.relocation_end()) + return make_error<JITLinkError>("Virtual section contains " + "relocations"); + continue; + } + + // Skip relocations for debug symbols. + { + auto &NSec = + getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); + if (!NSec.GraphSection) { + LLVM_DEBUG({ + dbgs() << "Skipping relocations for MachO section " << NSec.SegName + << "/" << NSec.SectName + << " which has no associated graph section\n"; + }); + continue; + } + } + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); RelItr != RelEnd; ++RelItr) { @@ -560,7 +573,8 @@ private: *(ulittle32_t *)FixupPtr = Value; break; } - case Pointer64: { + case Pointer64: + case Pointer64Anon: { uint64_t Value = E.getTarget().getAddress() + E.getAddend(); *(ulittle64_t *)FixupPtr = Value; break; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 69ec72aae292..a91bc3b6033c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -95,15 +95,6 @@ private: ", length=" + formatv("{0:d}", RI.r_length)); } - MachO::relocation_info - getRelocationInfo(const object::relocation_iterator RelItr) { - MachO::any_relocation_info ARI = - getObject().getRelocation(RelItr->getRawDataRefImpl()); - MachO::relocation_info RI; - memcpy(&RI, &ARI, sizeof(MachO::relocation_info)); - return RI; - } - using PairRelocInfo = std::tuple<MachOX86RelocationKind, Symbol *, uint64_t>; // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, @@ -196,6 +187,7 @@ private: JITTargetAddress SectionAddress = S.getAddress(); + // Skip relocations virtual sections. if (S.isVirtual()) { if (S.relocation_begin() != S.relocation_end()) return make_error<JITLinkError>("Virtual section contains " @@ -203,6 +195,21 @@ private: continue; } + // Skip relocations for debug symbols. + { + auto &NSec = + getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl())); + if (!NSec.GraphSection) { + LLVM_DEBUG({ + dbgs() << "Skipping relocations for MachO section " << NSec.SegName + << "/" << NSec.SectName + << " which has no associated graph section\n"; + }); + continue; + } + } + + // Add relocations for section. for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); RelItr != RelEnd; ++RelItr) { @@ -350,6 +357,9 @@ private: class MachO_x86_64_GOTAndStubsBuilder : public BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder> { public: + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G) : BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder>(G) {} @@ -367,7 +377,13 @@ public: void fixGOTEdge(Edge &E, Symbol &GOTEntry) { assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && "Not a GOT edge?"); - E.setKind(PCRel32); + // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is + // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to + // check for GOT optimization opportunities in the + // optimizeMachO_x86_64_GOTAndStubs pass below. + if (E.getKind() == PCRel32GOT) + E.setKind(PCRel32); + E.setTarget(GOTEntry); // Leave the edge addend as-is. } @@ -388,6 +404,11 @@ public: void fixExternalBranchEdge(Edge &E, Symbol &Stub) { assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + + // Set the edge kind to Branch32ToStub. We will use this to check for stub + // optimization opportunities in the optimizeMachO_x86_64_GOTAndStubs pass + // below. + E.setKind(Branch32ToStub); E.setTarget(Stub); } @@ -417,8 +438,6 @@ private: sizeof(StubContent)); } - static const uint8_t NullGOTEntryContent[8]; - static const uint8_t StubContent[6]; Section *GOTSection = nullptr; Section *StubsSection = nullptr; }; @@ -429,6 +448,89 @@ const uint8_t MachO_x86_64_GOTAndStubsBuilder::StubContent[6] = { 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; } // namespace +static Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); + + for (auto *B : G.blocks()) + for (auto &E : B->edges()) + if (E.getKind() == PCRel32GOTLoad) { + assert(E.getOffset() >= 3 && "GOT edge occurs too early in block"); + + // Switch the edge kind to PCRel32: Whether we change the edge target + // or not this will be the desired kind. + E.setKind(PCRel32); + + // Optimize GOT references. + auto &GOTBlock = E.getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT entry block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT entry should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + // Check that this is a recognized MOV instruction. + // FIXME: Can we assume this? + constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b}; + if (strncmp(B->getContent().data() + E.getOffset() - 3, + reinterpret_cast<const char *>(MOVQRIPRel), 2) != 0) + continue; + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits<int32_t>::min() && + Displacement <= std::numeric_limits<int32_t>::max()) { + E.setTarget(GOTTarget); + auto *BlockData = reinterpret_cast<uint8_t *>( + const_cast<char *>(B->getContent().data())); + BlockData[E.getOffset() - 2] = 0x8d; + LLVM_DEBUG({ + dbgs() << " Replaced GOT load wih LEA:\n "; + printEdge(dbgs(), *B, E, + getMachOX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } else if (E.getKind() == Branch32ToStub) { + + // Switch the edge kind to PCRel32: Whether we change the edge target + // or not this will be the desired kind. + E.setKind(Branch32); + + auto &StubBlock = E.getTarget().getBlock(); + assert(StubBlock.getSize() == + sizeof(MachO_x86_64_GOTAndStubsBuilder::StubContent) && + "Stub block should be stub sized"); + assert(StubBlock.edges_size() == 1 && + "Stub block should only have one outgoing edge"); + + auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT block should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset(); + JITTargetAddress TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (Displacement >= std::numeric_limits<int32_t>::min() && + Displacement <= std::numeric_limits<int32_t>::max()) { + E.setTarget(GOTTarget); + LLVM_DEBUG({ + dbgs() << " Replaced stub branch with direct branch:\n "; + printEdge(dbgs(), *B, E, + getMachOX86RelocationKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } + + return Error::success(); +} + namespace llvm { namespace jitlink { @@ -570,6 +672,9 @@ void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) { MachO_x86_64_GOTAndStubsBuilder(G).run(); return Error::success(); }); + + // Add GOT/Stubs optimizer pass. + Config.PostAllocationPasses.push_back(optimizeMachO_x86_64_GOTAndStubs); } if (auto Err = Ctx->modifyPassConfig(TT, Config)) @@ -583,6 +688,8 @@ StringRef getMachOX86RelocationKindName(Edge::Kind R) { switch (R) { case Branch32: return "Branch32"; + case Branch32ToStub: + return "Branch32ToStub"; case Pointer32: return "Pointer32"; case Pointer64: |