aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/ExecutionEngine/JITLink
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/ExecutionEngine/JITLink')
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h30
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/ELF.cpp51
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp463
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/JITLink.cpp17
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp119
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h109
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp2
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/MachO.cpp26
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp82
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h28
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp34
-rw-r--r--llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp131
12 files changed, 936 insertions, 156 deletions
diff --git a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h
index b47a798c7603..82258a35a675 100644
--- a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h
@@ -15,6 +15,8 @@
#include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#define DEBUG_TYPE "jitlink"
+
namespace llvm {
namespace jitlink {
@@ -27,12 +29,25 @@ public:
// the newly added ones, so just copy the existing blocks out.
std::vector<Block *> Blocks(G.blocks().begin(), G.blocks().end());
+ LLVM_DEBUG(dbgs() << "Creating GOT entries and stubs:\n");
+
for (auto *B : Blocks)
for (auto &E : B->edges())
- if (impl().isGOTEdge(E))
+ if (impl().isGOTEdge(E)) {
+ LLVM_DEBUG({
+ dbgs() << " Updating GOT edge ";
+ printEdge(dbgs(), *B, E, "<target GOT>");
+ dbgs() << "\n";
+ });
impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget()));
- else if (impl().isExternalBranchEdge(E))
+ } else if (impl().isExternalBranchEdge(E)) {
+ LLVM_DEBUG({
+ dbgs() << " Updating external branch edge ";
+ printEdge(dbgs(), *B, E, "<target PC-rel>");
+ dbgs() << "\n";
+ });
impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget()));
+ }
}
protected:
@@ -44,11 +59,17 @@ protected:
// Build the entry if it doesn't exist.
if (GOTEntryI == GOTEntries.end()) {
auto &GOTEntry = impl().createGOTEntry(Target);
+ LLVM_DEBUG({
+ dbgs() << " Created GOT entry for " << Target.getName() << ": "
+ << GOTEntry << "\n";
+ });
GOTEntryI =
GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first;
}
assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol");
+ LLVM_DEBUG(
+ { dbgs() << " Using GOT entry " << *GOTEntryI->second << "\n"; });
return *GOTEntryI->second;
}
@@ -59,10 +80,15 @@ protected:
if (StubI == Stubs.end()) {
auto &StubSymbol = impl().createStub(Target);
+ LLVM_DEBUG({
+ dbgs() << " Created stub for " << Target.getName() << ": "
+ << StubSymbol << "\n";
+ });
StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first;
}
assert(StubI != Stubs.end() && "Count not get stub symbol");
+ LLVM_DEBUG({ dbgs() << " Using stub " << *StubI->second << "\n"; });
return *StubI->second;
}
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
new file mode 100644
index 000000000000..6160583b13fe
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp
@@ -0,0 +1,51 @@
+//===-------------- ELF.cpp - JIT linker function for ELF -------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// ELF jit-link function.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JITLink/ELF.h"
+
+#include "llvm/BinaryFormat/ELF.h"
+#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include <cstring>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "jitlink"
+
+namespace llvm {
+namespace jitlink {
+
+void jitLink_ELF(std::unique_ptr<JITLinkContext> Ctx) {
+
+ // We don't want to do full ELF validation here. We just verify it is elf'ish.
+ // Probably should parse into an elf header when we support more than x86 :)
+
+ StringRef Data = Ctx->getObjectBuffer().getBuffer();
+ if (Data.size() < llvm::ELF::EI_MAG3 + 1) {
+ Ctx->notifyFailed(make_error<JITLinkError>("Truncated ELF buffer"));
+ return;
+ }
+
+ if (!memcmp(Data.data(), llvm::ELF::ElfMagic, strlen(llvm::ELF::ElfMagic))) {
+ if (Data.data()[llvm::ELF::EI_CLASS] == ELF::ELFCLASS64) {
+ return jitLink_ELF_x86_64(std::move(Ctx));
+ }
+ }
+
+ Ctx->notifyFailed(make_error<JITLinkError>("ELF magic not valid"));
+}
+
+} // end namespace jitlink
+} // end namespace llvm
diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
new file mode 100644
index 000000000000..505f03590b6b
--- /dev/null
+++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp
@@ -0,0 +1,463 @@
+//===---- ELF_x86_64.cpp -JIT linker implementation for ELF/x86-64 ----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// ELF/x86-64 jit-link implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h"
+#include "JITLinkGeneric.h"
+#include "llvm/ExecutionEngine/JITLink/JITLink.h"
+#include "llvm/Object/ELFObjectFile.h"
+
+#define DEBUG_TYPE "jitlink"
+
+using namespace llvm;
+using namespace llvm::jitlink;
+
+static const char *CommonSectionName = "__common";
+
+namespace llvm {
+namespace jitlink {
+
+// This should become a template as the ELFFile is so a lot of this could become
+// generic
+class ELFLinkGraphBuilder_x86_64 {
+
+private:
+ Section *CommonSection = nullptr;
+ // TODO hack to get this working
+ // Find a better way
+ using SymbolTable = object::ELFFile<object::ELF64LE>::Elf_Shdr;
+ // For now we just assume
+ std::map<int32_t, Symbol *> JITSymbolTable;
+
+ Section &getCommonSection() {
+ if (!CommonSection) {
+ auto Prot = static_cast<sys::Memory::ProtectionFlags>(
+ sys::Memory::MF_READ | sys::Memory::MF_WRITE);
+ CommonSection = &G->createSection(CommonSectionName, Prot);
+ }
+ return *CommonSection;
+ }
+
+ static Expected<ELF_x86_64_Edges::ELFX86RelocationKind>
+ getRelocationKind(const uint32_t Type) {
+ switch (Type) {
+ case ELF::R_X86_64_PC32:
+ return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32;
+ }
+ return make_error<JITLinkError>("Unsupported x86-64 relocation:" +
+ formatv("{0:d}", Type));
+ }
+
+ std::unique_ptr<LinkGraph> G;
+ // This could be a template
+ const object::ELFFile<object::ELF64LE> &Obj;
+ object::ELFFile<object::ELF64LE>::Elf_Shdr_Range sections;
+ SymbolTable SymTab;
+
+ bool isRelocatable() { return Obj.getHeader()->e_type == llvm::ELF::ET_REL; }
+
+ support::endianness
+ getEndianness(const object::ELFFile<object::ELF64LE> &Obj) {
+ return Obj.isLE() ? support::little : support::big;
+ }
+
+ // This could also just become part of a template
+ unsigned getPointerSize(const object::ELFFile<object::ELF64LE> &Obj) {
+ return Obj.getHeader()->getFileClass() == ELF::ELFCLASS64 ? 8 : 4;
+ }
+
+ // We don't technically need this right now
+ // But for now going to keep it as it helps me to debug things
+
+ Error createNormalizedSymbols() {
+ LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n");
+
+ for (auto SecRef : sections) {
+ if (SecRef.sh_type != ELF::SHT_SYMTAB &&
+ SecRef.sh_type != ELF::SHT_DYNSYM)
+ continue;
+
+ auto Symbols = Obj.symbols(&SecRef);
+ // TODO: Currently I use this function to test things
+ // I also want to leave it to see if its common between MACH and elf
+ // so for now I just want to continue even if there is an error
+ if (errorToBool(Symbols.takeError()))
+ continue;
+
+ auto StrTabSec = Obj.getSection(SecRef.sh_link);
+ if (!StrTabSec)
+ return StrTabSec.takeError();
+ auto StringTable = Obj.getStringTable(*StrTabSec);
+ if (!StringTable)
+ return StringTable.takeError();
+
+ for (auto SymRef : *Symbols) {
+ Optional<StringRef> Name;
+ uint64_t Size = 0;
+
+ // FIXME: Read size.
+ (void)Size;
+
+ if (auto NameOrErr = SymRef.getName(*StringTable))
+ Name = *NameOrErr;
+ else
+ return NameOrErr.takeError();
+
+ LLVM_DEBUG({
+ dbgs() << " ";
+ if (!Name)
+ dbgs() << "<anonymous symbol>";
+ else
+ dbgs() << *Name;
+ dbgs() << ": value = " << formatv("{0:x16}", SymRef.getValue())
+ << ", type = " << formatv("{0:x2}", SymRef.getType())
+ << ", binding = " << SymRef.getBinding()
+ << ", size =" << Size;
+ dbgs() << "\n";
+ });
+ }
+ }
+ return Error::success();
+ }
+
+ Error createNormalizedSections() {
+ LLVM_DEBUG(dbgs() << "Creating normalized sections...\n");
+ for (auto &SecRef : sections) {
+ auto Name = Obj.getSectionName(&SecRef);
+ if (!Name)
+ return Name.takeError();
+ sys::Memory::ProtectionFlags Prot;
+ if (SecRef.sh_flags & ELF::SHF_EXECINSTR) {
+ Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_EXEC);
+ } else {
+ Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_WRITE);
+ }
+ uint64_t Address = SecRef.sh_addr;
+ uint64_t Size = SecRef.sh_size;
+ uint64_t Flags = SecRef.sh_flags;
+ uint64_t Alignment = SecRef.sh_addralign;
+ const char *Data = nullptr;
+ // TODO: figure out what it is that has 0 size no name and address
+ // 0000-0000
+ if (Size == 0)
+ continue;
+
+ // FIXME: Use flags.
+ (void)Flags;
+
+ LLVM_DEBUG({
+ dbgs() << " " << *Name << ": " << formatv("{0:x16}", Address) << " -- "
+ << formatv("{0:x16}", Address + Size) << ", align: " << Alignment
+ << " Flags:" << Flags << "\n";
+ });
+
+ if (SecRef.sh_type != ELF::SHT_NOBITS) {
+ // .sections() already checks that the data is not beyond the end of
+ // file
+ auto contents = Obj.getSectionContentsAsArray<char>(&SecRef);
+ if (!contents)
+ return contents.takeError();
+
+ Data = contents->data();
+ // TODO protection flags.
+ // for now everything is
+ auto &section = G->createSection(*Name, Prot);
+ // Do this here because we have it, but move it into graphify later
+ G->createContentBlock(section, StringRef(Data, Size), Address,
+ Alignment, 0);
+ if (SecRef.sh_type == ELF::SHT_SYMTAB)
+ // TODO: Dynamic?
+ SymTab = SecRef;
+ }
+ }
+
+ return Error::success();
+ }
+
+ Error addRelocations() {
+ LLVM_DEBUG(dbgs() << "Adding relocations\n");
+ // TODO a partern is forming of iterate some sections but only give me
+ // ones I am interested, i should abstract that concept some where
+ for (auto &SecRef : sections) {
+ if (SecRef.sh_type != ELF::SHT_RELA && SecRef.sh_type != ELF::SHT_REL)
+ continue;
+ // TODO can the elf obj file do this for me?
+ if (SecRef.sh_type == ELF::SHT_REL)
+ return make_error<llvm::StringError>("Shouldn't have REL in x64",
+ llvm::inconvertibleErrorCode());
+
+ auto RelSectName = Obj.getSectionName(&SecRef);
+ if (!RelSectName)
+ return RelSectName.takeError();
+ // Deal with .eh_frame later
+ if (*RelSectName == StringRef(".rela.eh_frame"))
+ continue;
+
+ auto UpdateSection = Obj.getSection(SecRef.sh_info);
+ if (!UpdateSection)
+ return UpdateSection.takeError();
+
+ auto UpdateSectionName = Obj.getSectionName(*UpdateSection);
+ if (!UpdateSectionName)
+ return UpdateSectionName.takeError();
+
+ auto JITSection = G->findSectionByName(*UpdateSectionName);
+ if (!JITSection)
+ return make_error<llvm::StringError>(
+ "Refencing a a section that wasn't added to graph" +
+ *UpdateSectionName,
+ llvm::inconvertibleErrorCode());
+
+ auto Relocations = Obj.relas(&SecRef);
+ if (!Relocations)
+ return Relocations.takeError();
+
+ for (const auto &Rela : *Relocations) {
+ auto Type = Rela.getType(false);
+
+ LLVM_DEBUG({
+ dbgs() << "Relocation Type: " << Type << "\n"
+ << "Name: " << Obj.getRelocationTypeName(Type) << "\n";
+ });
+
+ auto Symbol = Obj.getRelocationSymbol(&Rela, &SymTab);
+ if (!Symbol)
+ return Symbol.takeError();
+
+ auto BlockToFix = *(JITSection->blocks().begin());
+ auto TargetSymbol = JITSymbolTable[(*Symbol)->st_shndx];
+ uint64_t Addend = Rela.r_addend;
+ JITTargetAddress FixupAddress =
+ (*UpdateSection)->sh_addr + Rela.r_offset;
+
+ LLVM_DEBUG({
+ dbgs() << "Processing relocation at "
+ << format("0x%016" PRIx64, FixupAddress) << "\n";
+ });
+ auto Kind = getRelocationKind(Type);
+ if (!Kind)
+ return Kind.takeError();
+
+ LLVM_DEBUG({
+ Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol,
+ Addend);
+ // TODO a mapping of KIND => type then call getRelocationTypeName4
+ printEdge(dbgs(), *BlockToFix, GE, StringRef(""));
+ dbgs() << "\n";
+ });
+ BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(),
+ *TargetSymbol, Addend);
+ }
+ }
+ return Error::success();
+ }
+
+ Error graphifyRegularSymbols() {
+
+ // TODO: ELF supports beyond SHN_LORESERVE,
+ // need to perf test how a vector vs map handles those cases
+
+ std::vector<std::vector<object::ELFFile<object::ELF64LE>::Elf_Shdr_Range *>>
+ SecIndexToSymbols;
+
+ LLVM_DEBUG(dbgs() << "Creating graph symbols...\n");
+
+ for (auto SecRef : sections) {
+
+ if (SecRef.sh_type != ELF::SHT_SYMTAB &&
+ SecRef.sh_type != ELF::SHT_DYNSYM)
+ continue;
+ auto Symbols = Obj.symbols(&SecRef);
+ if (!Symbols)
+ return Symbols.takeError();
+
+ auto StrTabSec = Obj.getSection(SecRef.sh_link);
+ if (!StrTabSec)
+ return StrTabSec.takeError();
+ auto StringTable = Obj.getStringTable(*StrTabSec);
+ if (!StringTable)
+ return StringTable.takeError();
+ auto Name = Obj.getSectionName(&SecRef);
+ if (!Name)
+ return Name.takeError();
+ auto Section = G->findSectionByName(*Name);
+ if (!Section)
+ return make_error<llvm::StringError>("Could not find a section",
+ llvm::inconvertibleErrorCode());
+ // we only have one for now
+ auto blocks = Section->blocks();
+ if (blocks.empty())
+ return make_error<llvm::StringError>("Section has no block",
+ llvm::inconvertibleErrorCode());
+
+ for (auto SymRef : *Symbols) {
+ auto Type = SymRef.getType();
+ if (Type == ELF::STT_NOTYPE || Type == ELF::STT_FILE)
+ continue;
+ // these should do it for now
+ // if(Type != ELF::STT_NOTYPE &&
+ // Type != ELF::STT_OBJECT &&
+ // Type != ELF::STT_FUNC &&
+ // Type != ELF::STT_SECTION &&
+ // Type != ELF::STT_COMMON) {
+ // continue;
+ // }
+ std::pair<Linkage, Scope> bindings;
+ auto Name = SymRef.getName(*StringTable);
+ // I am not sure on If this is going to hold as an invariant. Revisit.
+ if (!Name)
+ return Name.takeError();
+ // TODO: weak and hidden
+ if (SymRef.isExternal())
+ bindings = {Linkage::Strong, Scope::Default};
+ else
+ bindings = {Linkage::Strong, Scope::Local};
+
+ if (SymRef.isDefined() &&
+ (Type == ELF::STT_FUNC || Type == ELF::STT_OBJECT)) {
+
+ auto DefinedSection = Obj.getSection(SymRef.st_shndx);
+ if (!DefinedSection)
+ return DefinedSection.takeError();
+ auto sectName = Obj.getSectionName(*DefinedSection);
+ if (!sectName)
+ return Name.takeError();
+
+ auto JitSection = G->findSectionByName(*sectName);
+ if (!JitSection)
+ return make_error<llvm::StringError>(
+ "Could not find a section", llvm::inconvertibleErrorCode());
+ auto bs = JitSection->blocks();
+ if (bs.empty())
+ return make_error<llvm::StringError>(
+ "Section has no block", llvm::inconvertibleErrorCode());
+
+ auto B = *bs.begin();
+ LLVM_DEBUG({ dbgs() << " " << *Name << ": "; });
+
+ auto &S = G->addDefinedSymbol(
+ *B, SymRef.getValue(), *Name, SymRef.st_size, bindings.first,
+ bindings.second, SymRef.getType() == ELF::STT_FUNC, false);
+ JITSymbolTable[SymRef.st_shndx] = &S;
+ }
+ //TODO: The following has to be implmented.
+ // leaving commented out to save time for future patchs
+ /*
+ G->addAbsoluteSymbol(*Name, SymRef.getValue(), SymRef.st_size,
+ Linkage::Strong, Scope::Default, false);
+
+ if(SymRef.isCommon()) {
+ G->addCommonSymbol(*Name, Scope::Default, getCommonSection(), 0, 0,
+ SymRef.getValue(), false);
+ }
+
+
+ //G->addExternalSymbol(*Name, SymRef.st_size, Linkage::Strong);
+ */
+ }
+ }
+ return Error::success();
+ }
+
+public:
+ ELFLinkGraphBuilder_x86_64(std::string filename,
+ const object::ELFFile<object::ELF64LE> &Obj)
+ : G(std::make_unique<LinkGraph>(filename, getPointerSize(Obj),
+ getEndianness(Obj))),
+ Obj(Obj) {}
+
+ Expected<std::unique_ptr<LinkGraph>> buildGraph() {
+ // Sanity check: we only operate on relocatable objects.
+ if (!isRelocatable())
+ return make_error<JITLinkError>("Object is not a relocatable ELF");
+
+ auto Secs = Obj.sections();
+
+ if (!Secs) {
+ return Secs.takeError();
+ }
+ sections = *Secs;
+
+ if (auto Err = createNormalizedSections())
+ return std::move(Err);
+
+ if (auto Err = createNormalizedSymbols())
+ return std::move(Err);
+
+ if (auto Err = graphifyRegularSymbols())
+ return std::move(Err);
+
+ if (auto Err = addRelocations())
+ return std::move(Err);
+
+ return std::move(G);
+ }
+};
+
+class ELFJITLinker_x86_64 : public JITLinker<ELFJITLinker_x86_64> {
+ friend class JITLinker<ELFJITLinker_x86_64>;
+
+public:
+ ELFJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx,
+ PassConfiguration PassConfig)
+ : JITLinker(std::move(Ctx), std::move(PassConfig)) {}
+
+private:
+ StringRef getEdgeKindName(Edge::Kind R) const override { return StringRef(); }
+
+ Expected<std::unique_ptr<LinkGraph>>
+ buildGraph(MemoryBufferRef ObjBuffer) override {
+ auto ELFObj = object::ObjectFile::createELFObjectFile(ObjBuffer);
+ if (!ELFObj)
+ return ELFObj.takeError();
+
+ auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj);
+ std::string fileName(ELFObj->get()->getFileName());
+ return ELFLinkGraphBuilder_x86_64(std::move(fileName),
+ *ELFObjFile.getELFFile())
+ .buildGraph();
+ }
+
+ Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const {
+ using namespace ELF_x86_64_Edges;
+ char *FixupPtr = BlockWorkingMem + E.getOffset();
+ JITTargetAddress FixupAddress = B.getAddress() + E.getOffset();
+ switch (E.getKind()) {
+
+ case ELFX86RelocationKind::PCRel32:
+ int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress;
+ // verify
+ *(support::little32_t *)FixupPtr = Value;
+ break;
+ }
+ return Error::success();
+ }
+};
+
+void jitLink_ELF_x86_64(std::unique_ptr<JITLinkContext> Ctx) {
+ PassConfiguration Config;
+ Triple TT("x86_64-linux");
+ // Construct a JITLinker and run the link function.
+ // Add a mark-live pass.
+ if (auto MarkLive = Ctx->getMarkLivePass(TT))
+ Config.PrePrunePasses.push_back(std::move(MarkLive));
+ else
+ Config.PrePrunePasses.push_back(markAllSymbolsLive);
+
+ if (auto Err = Ctx->modifyPassConfig(TT, Config))
+ return Ctx->notifyFailed(std::move(Err));
+
+ ELFJITLinker_x86_64::link(std::move(Ctx), std::move(Config));
+}
+} // end namespace jitlink
+} // end namespace llvm
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
index 6c924f889577..5105ec495148 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp
@@ -10,6 +10,7 @@
#include "llvm/ExecutionEngine/JITLink/JITLink.h"
#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/ExecutionEngine/JITLink/ELF.h"
#include "llvm/ExecutionEngine/JITLink/MachO.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/ManagedStatic.h"
@@ -180,18 +181,14 @@ Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex,
// Copy edges to NewBlock (recording their iterators so that we can remove
// them from B), and update of Edges remaining on B.
std::vector<Block::edge_iterator> EdgesToRemove;
- for (auto I = B.edges().begin(), E = B.edges().end(); I != E; ++I) {
+ for (auto I = B.edges().begin(); I != B.edges().end();) {
if (I->getOffset() < SplitIndex) {
NewBlock.addEdge(*I);
- EdgesToRemove.push_back(I);
- } else
+ I = B.removeEdge(I);
+ } else {
I->setOffset(I->getOffset() - SplitIndex);
- }
-
- // Remove edges that were transfered to NewBlock from B.
- while (!EdgesToRemove.empty()) {
- B.removeEdge(EdgesToRemove.back());
- EdgesToRemove.pop_back();
+ ++I;
+ }
}
}
@@ -304,6 +301,8 @@ void jitLink(std::unique_ptr<JITLinkContext> Ctx) {
switch (Magic) {
case file_magic::macho_object:
return jitLink_MachO(std::move(Ctx));
+ case file_magic::elf_relocatable:
+ return jitLink_ELF(std::move(Ctx));
default:
Ctx->notifyFailed(make_error<JITLinkError>("Unsupported file format"));
};
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
index 7b594fd2c0ea..1d76a49939dc 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
@@ -24,6 +24,8 @@ JITLinkerBase::~JITLinkerBase() {}
void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
+ LLVM_DEBUG({ dbgs() << "Building jitlink graph for new input...\n"; });
+
// Build the link graph.
if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer()))
G = std::move(*GraphOrErr);
@@ -31,6 +33,10 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
return Ctx->notifyFailed(GraphOrErr.takeError());
assert(G && "Graph should have been created by buildGraph above");
+ LLVM_DEBUG({
+ dbgs() << "Starting link phase 1 for graph " << G->getName() << "\n";
+ });
+
// Prune and optimize the graph.
if (auto Err = runPasses(Passes.PrePrunePasses))
return Ctx->notifyFailed(std::move(Err));
@@ -59,10 +65,17 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
return Ctx->notifyFailed(std::move(Err));
// Notify client that the defined symbols have been assigned addresses.
+ LLVM_DEBUG(
+ { dbgs() << "Resolving symbols defined in " << G->getName() << "\n"; });
Ctx->notifyResolved(*G);
auto ExternalSymbols = getExternalSymbolNames();
+ LLVM_DEBUG({
+ dbgs() << "Issuing lookup for external symbols for " << G->getName()
+ << " (may trigger materialization/linking of other graphs)...\n";
+ });
+
// We're about to hand off ownership of ourself to the continuation. Grab a
// pointer to the context so that we can call it to initiate the lookup.
//
@@ -87,6 +100,11 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
Expected<AsyncLookupResult> LR,
SegmentLayoutMap Layout) {
+
+ LLVM_DEBUG({
+ dbgs() << "Starting link phase 2 for graph " << G->getName() << "\n";
+ });
+
// If the lookup failed, bail out.
if (!LR)
return deallocateAndBailOut(LR.takeError());
@@ -94,13 +112,25 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
// Assign addresses to external addressables.
applyLookupResult(*LR);
+ // Copy block content to working memory.
+ copyBlockContentToWorkingMemory(Layout, *Alloc);
+
+ LLVM_DEBUG({
+ dbgs() << "Link graph \"" << G->getName()
+ << "\" before post-allocation passes:\n";
+ dumpGraph(dbgs());
+ });
+
+ if (auto Err = runPasses(Passes.PostAllocationPasses))
+ return deallocateAndBailOut(std::move(Err));
+
LLVM_DEBUG({
dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n";
dumpGraph(dbgs());
});
- // Copy block content to working memory and fix up.
- if (auto Err = copyAndFixUpBlocks(Layout, *Alloc))
+ // Fix up block content.
+ if (auto Err = fixUpBlocks(*G))
return deallocateAndBailOut(std::move(Err));
LLVM_DEBUG({
@@ -122,9 +152,16 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
}
void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err) {
+
+ LLVM_DEBUG({
+ dbgs() << "Starting link phase 3 for graph " << G->getName() << "\n";
+ });
+
if (Err)
return deallocateAndBailOut(std::move(Err));
Ctx->notifyFinalized(std::move(Alloc));
+
+ LLVM_DEBUG({ dbgs() << "Link of graph " << G->getName() << " complete\n"; });
}
Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) {
@@ -165,7 +202,7 @@ JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() {
}
LLVM_DEBUG({
- dbgs() << "Segment ordering:\n";
+ dbgs() << "Computed segment ordering:\n";
for (auto &KV : Layout) {
dbgs() << " Segment "
<< static_cast<sys::Memory::ProtectionFlags>(KV.first) << ":\n";
@@ -230,11 +267,12 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) {
return AllocOrErr.takeError();
LLVM_DEBUG({
- dbgs() << "JIT linker got working memory:\n";
+ dbgs() << "JIT linker got memory (working -> target):\n";
for (auto &KV : Layout) {
auto Prot = static_cast<sys::Memory::ProtectionFlags>(KV.first);
dbgs() << " " << Prot << ": "
- << (const void *)Alloc->getWorkingMemory(Prot).data() << "\n";
+ << (const void *)Alloc->getWorkingMemory(Prot).data() << " -> "
+ << formatv("{0:x16}", Alloc->getTargetMemory(Prot)) << "\n";
}
});
@@ -302,6 +340,77 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) {
"All strong external symbols should have been resolved by now");
}
+void JITLinkerBase::copyBlockContentToWorkingMemory(
+ const SegmentLayoutMap &Layout, JITLinkMemoryManager::Allocation &Alloc) {
+
+ LLVM_DEBUG(dbgs() << "Copying block content:\n");
+ for (auto &KV : Layout) {
+ auto &Prot = KV.first;
+ auto &SegLayout = KV.second;
+
+ auto SegMem =
+ Alloc.getWorkingMemory(static_cast<sys::Memory::ProtectionFlags>(Prot));
+ char *LastBlockEnd = SegMem.data();
+ char *BlockDataPtr = LastBlockEnd;
+
+ LLVM_DEBUG({
+ dbgs() << " Processing segment "
+ << static_cast<sys::Memory::ProtectionFlags>(Prot) << " [ "
+ << (const void *)SegMem.data() << " .. "
+ << (const void *)((char *)SegMem.data() + SegMem.size())
+ << " ]\n Processing content sections:\n";
+ });
+
+ for (auto *B : SegLayout.ContentBlocks) {
+ LLVM_DEBUG(dbgs() << " " << *B << ":\n");
+
+ // Pad to alignment/alignment-offset.
+ BlockDataPtr = alignToBlock(BlockDataPtr, *B);
+
+ LLVM_DEBUG({
+ dbgs() << " Bumped block pointer to " << (const void *)BlockDataPtr
+ << " to meet block alignment " << B->getAlignment()
+ << " and alignment offset " << B->getAlignmentOffset() << "\n";
+ });
+
+ // Zero pad up to alignment.
+ LLVM_DEBUG({
+ if (LastBlockEnd != BlockDataPtr)
+ dbgs() << " Zero padding from " << (const void *)LastBlockEnd
+ << " to " << (const void *)BlockDataPtr << "\n";
+ });
+
+ while (LastBlockEnd != BlockDataPtr)
+ *LastBlockEnd++ = 0;
+
+ // Copy initial block content.
+ LLVM_DEBUG({
+ dbgs() << " Copying block " << *B << " content, "
+ << B->getContent().size() << " bytes, from "
+ << (const void *)B->getContent().data() << " to "
+ << (const void *)BlockDataPtr << "\n";
+ });
+ memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size());
+
+ // Point the block's content to the fixed up buffer.
+ B->setContent(StringRef(BlockDataPtr, B->getContent().size()));
+
+ // Update block end pointer.
+ LastBlockEnd = BlockDataPtr + B->getContent().size();
+ BlockDataPtr = LastBlockEnd;
+ }
+
+ // Zero pad the rest of the segment.
+ LLVM_DEBUG({
+ dbgs() << " Zero padding end of segment from "
+ << (const void *)LastBlockEnd << " to "
+ << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n";
+ });
+ while (LastBlockEnd != SegMem.data() + SegMem.size())
+ *LastBlockEnd++ = 0;
+ }
+}
+
void JITLinkerBase::deallocateAndBailOut(Error Err) {
assert(Err && "Should not be bailing out on success value");
assert(Alloc && "can not call deallocateAndBailOut before allocation");
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
index d5687b7afc96..87e5e8bbc98d 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
@@ -80,13 +80,13 @@ protected:
// For debug dumping of the link graph.
virtual StringRef getEdgeKindName(Edge::Kind K) const = 0;
- // Alight a JITTargetAddress to conform with block alignment requirements.
+ // Align a JITTargetAddress to conform with block alignment requirements.
static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) {
uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment();
return Addr + Delta;
}
- // Alight a pointer to conform with block alignment requirements.
+ // Align a pointer to conform with block alignment requirements.
static char *alignToBlock(char *P, Block &B) {
uint64_t PAddr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(P));
uint64_t Delta = (B.getAlignmentOffset() - PAddr) % B.getAlignment();
@@ -100,14 +100,14 @@ private:
// Copy block contents and apply relocations.
// Implemented in JITLinker.
- virtual Error
- copyAndFixUpBlocks(const SegmentLayoutMap &Layout,
- JITLinkMemoryManager::Allocation &Alloc) const = 0;
+ virtual Error fixUpBlocks(LinkGraph &G) const = 0;
SegmentLayoutMap layOutBlocks();
Error allocateSegments(const SegmentLayoutMap &Layout);
JITLinkContext::LookupMap getExternalSymbolNames() const;
void applyLookupResult(AsyncLookupResult LR);
+ void copyBlockContentToWorkingMemory(const SegmentLayoutMap &Layout,
+ JITLinkMemoryManager::Allocation &Alloc);
void deallocateAndBailOut(Error Err);
void dumpGraph(raw_ostream &OS);
@@ -144,88 +144,25 @@ private:
return static_cast<const LinkerImpl &>(*this);
}
- Error
- copyAndFixUpBlocks(const SegmentLayoutMap &Layout,
- JITLinkMemoryManager::Allocation &Alloc) const override {
- LLVM_DEBUG(dbgs() << "Copying and fixing up blocks:\n");
- for (auto &KV : Layout) {
- auto &Prot = KV.first;
- auto &SegLayout = KV.second;
-
- auto SegMem = Alloc.getWorkingMemory(
- static_cast<sys::Memory::ProtectionFlags>(Prot));
- char *LastBlockEnd = SegMem.data();
- char *BlockDataPtr = LastBlockEnd;
-
- LLVM_DEBUG({
- dbgs() << " Processing segment "
- << static_cast<sys::Memory::ProtectionFlags>(Prot) << " [ "
- << (const void *)SegMem.data() << " .. "
- << (const void *)((char *)SegMem.data() + SegMem.size())
- << " ]\n Processing content sections:\n";
- });
-
- for (auto *B : SegLayout.ContentBlocks) {
- LLVM_DEBUG(dbgs() << " " << *B << ":\n");
-
- // Pad to alignment/alignment-offset.
- BlockDataPtr = alignToBlock(BlockDataPtr, *B);
-
- LLVM_DEBUG({
- dbgs() << " Bumped block pointer to "
- << (const void *)BlockDataPtr << " to meet block alignment "
- << B->getAlignment() << " and alignment offset "
- << B->getAlignmentOffset() << "\n";
- });
-
- // Zero pad up to alignment.
- LLVM_DEBUG({
- if (LastBlockEnd != BlockDataPtr)
- dbgs() << " Zero padding from " << (const void *)LastBlockEnd
- << " to " << (const void *)BlockDataPtr << "\n";
- });
-
- while (LastBlockEnd != BlockDataPtr)
- *LastBlockEnd++ = 0;
-
- // Copy initial block content.
- LLVM_DEBUG({
- dbgs() << " Copying block " << *B << " content, "
- << B->getContent().size() << " bytes, from "
- << (const void *)B->getContent().data() << " to "
- << (const void *)BlockDataPtr << "\n";
- });
- memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size());
-
- // Copy Block data and apply fixups.
- LLVM_DEBUG(dbgs() << " Applying fixups.\n");
- for (auto &E : B->edges()) {
-
- // Skip non-relocation edges.
- if (!E.isRelocation())
- continue;
-
- // Dispatch to LinkerImpl for fixup.
- if (auto Err = impl().applyFixup(*B, E, BlockDataPtr))
- return Err;
- }
-
- // Point the block's content to the fixed up buffer.
- B->setContent(StringRef(BlockDataPtr, B->getContent().size()));
-
- // Update block end pointer.
- LastBlockEnd = BlockDataPtr + B->getContent().size();
- BlockDataPtr = LastBlockEnd;
- }
+ Error fixUpBlocks(LinkGraph &G) const override {
+ LLVM_DEBUG(dbgs() << "Fixing up blocks:\n");
+
+ for (auto *B : G.blocks()) {
+ LLVM_DEBUG(dbgs() << " " << *B << ":\n");
+
+ // Copy Block data and apply fixups.
+ LLVM_DEBUG(dbgs() << " Applying fixups.\n");
+ for (auto &E : B->edges()) {
- // Zero pad the rest of the segment.
- LLVM_DEBUG({
- dbgs() << " Zero padding end of segment from "
- << (const void *)LastBlockEnd << " to "
- << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n";
- });
- while (LastBlockEnd != SegMem.data() + SegMem.size())
- *LastBlockEnd++ = 0;
+ // Skip non-relocation edges.
+ if (!E.isRelocation())
+ continue;
+
+ // Dispatch to LinkerImpl for fixup.
+ auto *BlockData = const_cast<char *>(B->getContent().data());
+ if (auto Err = impl().applyFixup(*B, E, BlockData))
+ return Err;
+ }
}
return Error::success();
diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
index 9e0d207e8bdb..68ec9d79af9b 100644
--- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
@@ -32,7 +32,7 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) {
}
JITTargetAddress getTargetMemory(ProtectionFlags Seg) override {
assert(SegBlocks.count(Seg) && "No allocation for segment");
- return reinterpret_cast<JITTargetAddress>(SegBlocks[Seg].base());
+ return pointerToJITTargetAddress(SegBlocks[Seg].base());
}
void finalizeAsync(FinalizeContinuation OnFinalize) override {
OnFinalize(applyProtections());
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO.cpp
index 58bc0f56e155..b3e45868ab22 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO.cpp
@@ -16,9 +16,9 @@
#include "llvm/BinaryFormat/MachO.h"
#include "llvm/ExecutionEngine/JITLink/MachO_arm64.h"
#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h"
-#include "llvm/Support/Endian.h"
#include "llvm/Support/Format.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/SwapByteOrder.h"
using namespace llvm;
@@ -34,7 +34,9 @@ void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) {
StringRef Data = Ctx->getObjectBuffer().getBuffer();
if (Data.size() < 4) {
- Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer"));
+ StringRef BufferName = Ctx->getObjectBuffer().getBufferIdentifier();
+ Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer \"" +
+ BufferName + "\""));
return;
}
@@ -51,20 +53,26 @@ void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) {
make_error<JITLinkError>("MachO 32-bit platforms not supported"));
return;
} else if (Magic == MachO::MH_MAGIC_64 || Magic == MachO::MH_CIGAM_64) {
- MachO::mach_header_64 Header;
- memcpy(&Header, Data.data(), sizeof(MachO::mach_header_64));
+ if (Data.size() < sizeof(MachO::mach_header_64)) {
+ StringRef BufferName = Ctx->getObjectBuffer().getBufferIdentifier();
+ Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer \"" +
+ BufferName + "\""));
+ return;
+ }
+
+ // Read the CPU type from the header.
+ uint32_t CPUType;
+ memcpy(&CPUType, Data.data() + 4, sizeof(uint32_t));
if (Magic == MachO::MH_CIGAM_64)
- swapStruct(Header);
+ CPUType = ByteSwap_32(CPUType);
LLVM_DEBUG({
- dbgs() << "jitLink_MachO: cputype = "
- << format("0x%08" PRIx32, Header.cputype)
- << ", cpusubtype = " << format("0x%08" PRIx32, Header.cpusubtype)
+ dbgs() << "jitLink_MachO: cputype = " << format("0x%08" PRIx32, CPUType)
<< "\n";
});
- switch (Header.cputype) {
+ switch (CPUType) {
case MachO::CPU_TYPE_ARM64:
return jitLink_MachO_arm64(std::move(Ctx));
case MachO::CPU_TYPE_X86_64:
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
index 701f108a9a21..fa3f403b717c 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
@@ -47,8 +47,8 @@ Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() {
MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj)
: Obj(Obj),
- G(std::make_unique<LinkGraph>(Obj.getFileName(), getPointerSize(Obj),
- getEndianness(Obj))) {}
+ G(std::make_unique<LinkGraph>(std::string(Obj.getFileName()),
+ getPointerSize(Obj), getEndianness(Obj))) {}
void MachOLinkGraphBuilder::addCustomSectionParser(
StringRef SectionName, SectionParserFunction Parser) {
@@ -64,12 +64,14 @@ Linkage MachOLinkGraphBuilder::getLinkage(uint16_t Desc) {
}
Scope MachOLinkGraphBuilder::getScope(StringRef Name, uint8_t Type) {
- if (Name.startswith("l"))
- return Scope::Local;
if (Type & MachO::N_PEXT)
return Scope::Hidden;
- if (Type & MachO::N_EXT)
- return Scope::Default;
+ if (Type & MachO::N_EXT) {
+ if (Name.startswith("l"))
+ return Scope::Hidden;
+ else
+ return Scope::Default;
+ }
return Scope::Local;
}
@@ -77,6 +79,11 @@ bool MachOLinkGraphBuilder::isAltEntry(const NormalizedSymbol &NSym) {
return NSym.Desc & MachO::N_ALT_ENTRY;
}
+bool MachOLinkGraphBuilder::isDebugSection(const NormalizedSection &NSec) {
+ return (NSec.Flags & MachO::S_ATTR_DEBUG &&
+ strcmp(NSec.SegName, "__DWARF") == 0);
+}
+
unsigned
MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) {
return Obj.is64Bit() ? 8 : 4;
@@ -116,6 +123,11 @@ Error MachOLinkGraphBuilder::createNormalizedSections() {
const MachO::section_64 &Sec64 =
Obj.getSection64(SecRef.getRawDataRefImpl());
+ memcpy(&NSec.SectName, &Sec64.sectname, 16);
+ NSec.SectName[16] = '\0';
+ memcpy(&NSec.SegName, Sec64.segname, 16);
+ NSec.SegName[16] = '\0';
+
NSec.Address = Sec64.addr;
NSec.Size = Sec64.size;
NSec.Alignment = 1ULL << Sec64.align;
@@ -123,6 +135,12 @@ Error MachOLinkGraphBuilder::createNormalizedSections() {
DataOffset = Sec64.offset;
} else {
const MachO::section &Sec32 = Obj.getSection(SecRef.getRawDataRefImpl());
+
+ memcpy(&NSec.SectName, &Sec32.sectname, 16);
+ NSec.SectName[16] = '\0';
+ memcpy(&NSec.SegName, Sec32.segname, 16);
+ NSec.SegName[16] = '\0';
+
NSec.Address = Sec32.addr;
NSec.Size = Sec32.size;
NSec.Alignment = 1ULL << Sec32.align;
@@ -162,7 +180,14 @@ Error MachOLinkGraphBuilder::createNormalizedSections() {
Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
sys::Memory::MF_WRITE);
- NSec.GraphSection = &G->createSection(*Name, Prot);
+ if (!isDebugSection(NSec))
+ NSec.GraphSection = &G->createSection(*Name, Prot);
+ else
+ LLVM_DEBUG({
+ dbgs() << " " << *Name
+ << " is a debug section: No graph section will be created.\n";
+ });
+
IndexToSection.insert(std::make_pair(SecIndex, std::move(NSec)));
}
@@ -189,12 +214,12 @@ Error MachOLinkGraphBuilder::createNormalizedSections() {
auto &Next = *Sections[I + 1];
if (Next.Address < Cur.Address + Cur.Size)
return make_error<JITLinkError>(
- "Address range for section " + Cur.GraphSection->getName() +
- formatv(" [ {0:x16} -- {1:x16} ] ", Cur.Address,
- Cur.Address + Cur.Size) +
- "overlaps " +
- formatv(" [ {0:x16} -- {1:x16} ] ", Next.Address,
- Next.Address + Next.Size));
+ "Address range for section " +
+ formatv("\"{0}/{1}\" [ {2:x16} -- {3:x16} ] ", Cur.SegName,
+ Cur.SectName, Cur.Address, Cur.Address + Cur.Size) +
+ "overlaps section \"" + Next.SegName + "/" + Next.SectName + "\"" +
+ formatv("\"{0}/{1}\" [ {2:x16} -- {3:x16} ] ", Next.SegName,
+ Next.SectName, Next.Address, Next.Address + Next.Size));
}
return Error::success();
@@ -260,21 +285,28 @@ Error MachOLinkGraphBuilder::createNormalizedSymbols() {
});
// If this symbol has a section, sanity check that the addresses line up.
- NormalizedSection *NSec = nullptr;
if (Sect != 0) {
- if (auto NSecOrErr = findSectionByIndex(Sect - 1))
- NSec = &*NSecOrErr;
- else
- return NSecOrErr.takeError();
+ auto NSec = findSectionByIndex(Sect - 1);
+ if (!NSec)
+ return NSec.takeError();
if (Value < NSec->Address || Value > NSec->Address + NSec->Size)
return make_error<JITLinkError>("Symbol address does not fall within "
"section");
+
+ if (!NSec->GraphSection) {
+ LLVM_DEBUG({
+ dbgs() << " Skipping: Symbol is in section " << NSec->SegName << "/"
+ << NSec->SectName
+ << " which has no associated graph section.\n";
+ });
+ continue;
+ }
}
IndexToSymbol[SymbolIndex] =
&createNormalizedSymbol(*Name, Value, Type, Sect, Desc,
- getLinkage(Type), getScope(*Name, Type));
+ getLinkage(Desc), getScope(*Name, Type));
}
return Error::success();
@@ -362,6 +394,14 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() {
auto SecIndex = KV.first;
auto &NSec = KV.second;
+ if (!NSec.GraphSection) {
+ LLVM_DEBUG({
+ dbgs() << " " << NSec.SegName << "/" << NSec.SectName
+ << " has no graph section. Skipping.\n";
+ });
+ continue;
+ }
+
// Skip sections with custom parsers.
if (CustomSectionParserFunctions.count(NSec.GraphSection->getName())) {
LLVM_DEBUG({
@@ -524,6 +564,10 @@ Error MachOLinkGraphBuilder::graphifySectionsWithCustomParsers() {
for (auto &KV : IndexToSection) {
auto &NSec = KV.second;
+ // Skip non-graph sections.
+ if (!NSec.GraphSection)
+ continue;
+
auto HI = CustomSectionParserFunctions.find(NSec.GraphSection->getName());
if (HI != CustomSectionParserFunctions.end()) {
auto &Parse = HI->second;
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
index 91b1d5a22387..dd3bcf27494c 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
+++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
@@ -13,6 +13,8 @@
#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H
#define LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/StringMap.h"
#include "llvm/ExecutionEngine/JITLink/JITLink.h"
#include "EHFrameSupportImpl.h"
@@ -58,6 +60,8 @@ protected:
Symbol *GraphSymbol = nullptr;
};
+ // Normalized section representation. Section and segment names are guaranteed
+ // to be null-terminated, hence the extra bytes on SegName and SectName.
class NormalizedSection {
friend class MachOLinkGraphBuilder;
@@ -65,12 +69,14 @@ protected:
NormalizedSection() = default;
public:
- Section *GraphSection = nullptr;
+ char SectName[17];
+ char SegName[17];
uint64_t Address = 0;
uint64_t Size = 0;
uint64_t Alignment = 0;
uint32_t Flags = 0;
const char *Data = nullptr;
+ Section *GraphSection = nullptr;
};
using SectionParserFunction = std::function<Error(NormalizedSection &S)>;
@@ -110,7 +116,7 @@ protected:
auto I = IndexToSection.find(Index);
if (I == IndexToSection.end())
return make_error<JITLinkError>("No section recorded for index " +
- formatv("{0:u}", Index));
+ formatv("{0:d}", Index));
return I->second;
}
@@ -123,7 +129,7 @@ protected:
auto *Sym = IndexToSymbol[Index];
if (!Sym)
return make_error<JITLinkError>("No symbol at index " +
- formatv("{0:u}", Index));
+ formatv("{0:d}", Index));
return *Sym;
}
@@ -151,6 +157,22 @@ protected:
static Scope getScope(StringRef Name, uint8_t Type);
static bool isAltEntry(const NormalizedSymbol &NSym);
+ static bool isDebugSection(const NormalizedSection &NSec);
+
+ MachO::relocation_info
+ getRelocationInfo(const object::relocation_iterator RelItr) {
+ MachO::any_relocation_info ARI =
+ getObject().getRelocation(RelItr->getRawDataRefImpl());
+ MachO::relocation_info RI;
+ RI.r_address = ARI.r_word0;
+ RI.r_symbolnum = ARI.r_word1 & 0xffffff;
+ RI.r_pcrel = (ARI.r_word1 >> 24) & 1;
+ RI.r_length = (ARI.r_word1 >> 25) & 3;
+ RI.r_extern = (ARI.r_word1 >> 27) & 1;
+ RI.r_type = (ARI.r_word1 >> 28);
+ return RI;
+ }
+
private:
static unsigned getPointerSize(const object::MachOObjectFile &Obj);
static support::endianness getEndianness(const object::MachOObjectFile &Obj);
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
index 944767449ce2..463845a5b8cb 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -92,15 +92,6 @@ private:
", length=" + formatv("{0:d}", RI.r_length));
}
- MachO::relocation_info
- getRelocationInfo(const object::relocation_iterator RelItr) {
- MachO::any_relocation_info ARI =
- getObject().getRelocation(RelItr->getRawDataRefImpl());
- MachO::relocation_info RI;
- memcpy(&RI, &ARI, sizeof(MachO::relocation_info));
- return RI;
- }
-
using PairRelocInfo =
std::tuple<MachOARM64RelocationKind, Symbol *, uint64_t>;
@@ -194,6 +185,28 @@ private:
JITTargetAddress SectionAddress = S.getAddress();
+ // Skip relocations virtual sections.
+ if (S.isVirtual()) {
+ if (S.relocation_begin() != S.relocation_end())
+ return make_error<JITLinkError>("Virtual section contains "
+ "relocations");
+ continue;
+ }
+
+ // Skip relocations for debug symbols.
+ {
+ auto &NSec =
+ getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl()));
+ if (!NSec.GraphSection) {
+ LLVM_DEBUG({
+ dbgs() << "Skipping relocations for MachO section " << NSec.SegName
+ << "/" << NSec.SectName
+ << " which has no associated graph section\n";
+ });
+ continue;
+ }
+ }
+
for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end();
RelItr != RelEnd; ++RelItr) {
@@ -560,7 +573,8 @@ private:
*(ulittle32_t *)FixupPtr = Value;
break;
}
- case Pointer64: {
+ case Pointer64:
+ case Pointer64Anon: {
uint64_t Value = E.getTarget().getAddress() + E.getAddend();
*(ulittle64_t *)FixupPtr = Value;
break;
diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
index 69ec72aae292..a91bc3b6033c 100644
--- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
+++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
@@ -95,15 +95,6 @@ private:
", length=" + formatv("{0:d}", RI.r_length));
}
- MachO::relocation_info
- getRelocationInfo(const object::relocation_iterator RelItr) {
- MachO::any_relocation_info ARI =
- getObject().getRelocation(RelItr->getRawDataRefImpl());
- MachO::relocation_info RI;
- memcpy(&RI, &ARI, sizeof(MachO::relocation_info));
- return RI;
- }
-
using PairRelocInfo = std::tuple<MachOX86RelocationKind, Symbol *, uint64_t>;
// Parses paired SUBTRACTOR/UNSIGNED relocations and, on success,
@@ -196,6 +187,7 @@ private:
JITTargetAddress SectionAddress = S.getAddress();
+ // Skip relocations virtual sections.
if (S.isVirtual()) {
if (S.relocation_begin() != S.relocation_end())
return make_error<JITLinkError>("Virtual section contains "
@@ -203,6 +195,21 @@ private:
continue;
}
+ // Skip relocations for debug symbols.
+ {
+ auto &NSec =
+ getSectionByIndex(Obj.getSectionIndex(S.getRawDataRefImpl()));
+ if (!NSec.GraphSection) {
+ LLVM_DEBUG({
+ dbgs() << "Skipping relocations for MachO section " << NSec.SegName
+ << "/" << NSec.SectName
+ << " which has no associated graph section\n";
+ });
+ continue;
+ }
+ }
+
+ // Add relocations for section.
for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end();
RelItr != RelEnd; ++RelItr) {
@@ -350,6 +357,9 @@ private:
class MachO_x86_64_GOTAndStubsBuilder
: public BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder> {
public:
+ static const uint8_t NullGOTEntryContent[8];
+ static const uint8_t StubContent[6];
+
MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G)
: BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder>(G) {}
@@ -367,7 +377,13 @@ public:
void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) &&
"Not a GOT edge?");
- E.setKind(PCRel32);
+ // If this is a PCRel32GOT then change it to an ordinary PCRel32. If it is
+ // a PCRel32GOTLoad then leave it as-is for now. We will use the kind to
+ // check for GOT optimization opportunities in the
+ // optimizeMachO_x86_64_GOTAndStubs pass below.
+ if (E.getKind() == PCRel32GOT)
+ E.setKind(PCRel32);
+
E.setTarget(GOTEntry);
// Leave the edge addend as-is.
}
@@ -388,6 +404,11 @@ public:
void fixExternalBranchEdge(Edge &E, Symbol &Stub) {
assert(E.getKind() == Branch32 && "Not a Branch32 edge?");
assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?");
+
+ // Set the edge kind to Branch32ToStub. We will use this to check for stub
+ // optimization opportunities in the optimizeMachO_x86_64_GOTAndStubs pass
+ // below.
+ E.setKind(Branch32ToStub);
E.setTarget(Stub);
}
@@ -417,8 +438,6 @@ private:
sizeof(StubContent));
}
- static const uint8_t NullGOTEntryContent[8];
- static const uint8_t StubContent[6];
Section *GOTSection = nullptr;
Section *StubsSection = nullptr;
};
@@ -429,6 +448,89 @@ const uint8_t MachO_x86_64_GOTAndStubsBuilder::StubContent[6] = {
0xFF, 0x25, 0x00, 0x00, 0x00, 0x00};
} // namespace
+static Error optimizeMachO_x86_64_GOTAndStubs(LinkGraph &G) {
+ LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n");
+
+ for (auto *B : G.blocks())
+ for (auto &E : B->edges())
+ if (E.getKind() == PCRel32GOTLoad) {
+ assert(E.getOffset() >= 3 && "GOT edge occurs too early in block");
+
+ // Switch the edge kind to PCRel32: Whether we change the edge target
+ // or not this will be the desired kind.
+ E.setKind(PCRel32);
+
+ // Optimize GOT references.
+ auto &GOTBlock = E.getTarget().getBlock();
+ assert(GOTBlock.getSize() == G.getPointerSize() &&
+ "GOT entry block should be pointer sized");
+ assert(GOTBlock.edges_size() == 1 &&
+ "GOT entry should only have one outgoing edge");
+
+ auto &GOTTarget = GOTBlock.edges().begin()->getTarget();
+ JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset();
+ JITTargetAddress TargetAddr = GOTTarget.getAddress();
+
+ // Check that this is a recognized MOV instruction.
+ // FIXME: Can we assume this?
+ constexpr uint8_t MOVQRIPRel[] = {0x48, 0x8b};
+ if (strncmp(B->getContent().data() + E.getOffset() - 3,
+ reinterpret_cast<const char *>(MOVQRIPRel), 2) != 0)
+ continue;
+
+ int64_t Displacement = TargetAddr - EdgeAddr + 4;
+ if (Displacement >= std::numeric_limits<int32_t>::min() &&
+ Displacement <= std::numeric_limits<int32_t>::max()) {
+ E.setTarget(GOTTarget);
+ auto *BlockData = reinterpret_cast<uint8_t *>(
+ const_cast<char *>(B->getContent().data()));
+ BlockData[E.getOffset() - 2] = 0x8d;
+ LLVM_DEBUG({
+ dbgs() << " Replaced GOT load wih LEA:\n ";
+ printEdge(dbgs(), *B, E,
+ getMachOX86RelocationKindName(E.getKind()));
+ dbgs() << "\n";
+ });
+ }
+ } else if (E.getKind() == Branch32ToStub) {
+
+ // Switch the edge kind to PCRel32: Whether we change the edge target
+ // or not this will be the desired kind.
+ E.setKind(Branch32);
+
+ auto &StubBlock = E.getTarget().getBlock();
+ assert(StubBlock.getSize() ==
+ sizeof(MachO_x86_64_GOTAndStubsBuilder::StubContent) &&
+ "Stub block should be stub sized");
+ assert(StubBlock.edges_size() == 1 &&
+ "Stub block should only have one outgoing edge");
+
+ auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock();
+ assert(GOTBlock.getSize() == G.getPointerSize() &&
+ "GOT block should be pointer sized");
+ assert(GOTBlock.edges_size() == 1 &&
+ "GOT block should only have one outgoing edge");
+
+ auto &GOTTarget = GOTBlock.edges().begin()->getTarget();
+ JITTargetAddress EdgeAddr = B->getAddress() + E.getOffset();
+ JITTargetAddress TargetAddr = GOTTarget.getAddress();
+
+ int64_t Displacement = TargetAddr - EdgeAddr + 4;
+ if (Displacement >= std::numeric_limits<int32_t>::min() &&
+ Displacement <= std::numeric_limits<int32_t>::max()) {
+ E.setTarget(GOTTarget);
+ LLVM_DEBUG({
+ dbgs() << " Replaced stub branch with direct branch:\n ";
+ printEdge(dbgs(), *B, E,
+ getMachOX86RelocationKindName(E.getKind()));
+ dbgs() << "\n";
+ });
+ }
+ }
+
+ return Error::success();
+}
+
namespace llvm {
namespace jitlink {
@@ -570,6 +672,9 @@ void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) {
MachO_x86_64_GOTAndStubsBuilder(G).run();
return Error::success();
});
+
+ // Add GOT/Stubs optimizer pass.
+ Config.PostAllocationPasses.push_back(optimizeMachO_x86_64_GOTAndStubs);
}
if (auto Err = Ctx->modifyPassConfig(TT, Config))
@@ -583,6 +688,8 @@ StringRef getMachOX86RelocationKindName(Edge::Kind R) {
switch (R) {
case Branch32:
return "Branch32";
+ case Branch32ToStub:
+ return "Branch32ToStub";
case Pointer32:
return "Pointer32";
case Pointer64:
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-20 16:27:38 +0000