aboutsummaryrefslogtreecommitdiff
path: root/lib/ExecutionEngine/JITLink
diff options
context:
space:
mode:
Diffstat (limited to 'lib/ExecutionEngine/JITLink')
-rw-r--r--lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h35
-rw-r--r--lib/ExecutionEngine/JITLink/EHFrameSupport.cpp216
-rw-r--r--lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h50
-rw-r--r--lib/ExecutionEngine/JITLink/JITLink.cpp158
-rw-r--r--lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp385
-rw-r--r--lib/ExecutionEngine/JITLink/JITLinkGeneric.h185
-rw-r--r--lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp63
-rw-r--r--lib/ExecutionEngine/JITLink/MachO.cpp3
-rw-r--r--lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp411
-rw-r--r--lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h138
-rw-r--r--lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp535
-rw-r--r--lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h269
-rw-r--r--lib/ExecutionEngine/JITLink/MachO_arm64.cpp736
-rw-r--r--lib/ExecutionEngine/JITLink/MachO_x86_64.cpp279
14 files changed, 2217 insertions, 1246 deletions
diff --git a/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h
index 1271ad962b38..b47a798c7603 100644
--- a/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h
+++ b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h
@@ -20,24 +20,23 @@ namespace jitlink {
template <typename BuilderImpl> class BasicGOTAndStubsBuilder {
public:
- BasicGOTAndStubsBuilder(AtomGraph &G) : G(G) {}
+ BasicGOTAndStubsBuilder(LinkGraph &G) : G(G) {}
void run() {
- // We're going to be adding new atoms, but we don't want to iterate over
- // the newly added ones, so just copy the existing atoms out.
- std::vector<DefinedAtom *> DAs(G.defined_atoms().begin(),
- G.defined_atoms().end());
+ // We're going to be adding new blocks, but we don't want to iterate over
+ // the newly added ones, so just copy the existing blocks out.
+ std::vector<Block *> Blocks(G.blocks().begin(), G.blocks().end());
- for (auto *DA : DAs)
- for (auto &E : DA->edges())
+ for (auto *B : Blocks)
+ for (auto &E : B->edges())
if (impl().isGOTEdge(E))
- impl().fixGOTEdge(E, getGOTEntryAtom(E.getTarget()));
+ impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget()));
else if (impl().isExternalBranchEdge(E))
- impl().fixExternalBranchEdge(E, getStubAtom(E.getTarget()));
+ impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget()));
}
protected:
- Atom &getGOTEntryAtom(Atom &Target) {
+ Symbol &getGOTEntrySymbol(Symbol &Target) {
assert(Target.hasName() && "GOT edge cannot point to anonymous target");
auto GOTEntryI = GOTEntries.find(Target.getName());
@@ -49,31 +48,31 @@ protected:
GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first;
}
- assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom");
+ assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol");
return *GOTEntryI->second;
}
- Atom &getStubAtom(Atom &Target) {
+ Symbol &getStubSymbol(Symbol &Target) {
assert(Target.hasName() &&
"External branch edge can not point to an anonymous target");
auto StubI = Stubs.find(Target.getName());
if (StubI == Stubs.end()) {
- auto &StubAtom = impl().createStub(Target);
- StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first;
+ auto &StubSymbol = impl().createStub(Target);
+ StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first;
}
- assert(StubI != Stubs.end() && "Count not get stub atom");
+ assert(StubI != Stubs.end() && "Count not get stub symbol");
return *StubI->second;
}
- AtomGraph &G;
+ LinkGraph &G;
private:
BuilderImpl &impl() { return static_cast<BuilderImpl &>(*this); }
- DenseMap<StringRef, DefinedAtom *> GOTEntries;
- DenseMap<StringRef, DefinedAtom *> Stubs;
+ DenseMap<StringRef, Symbol *> GOTEntries;
+ DenseMap<StringRef, Symbol *> Stubs;
};
} // end namespace jitlink
diff --git a/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
index 25f0e9040ffe..f80b0e7f8909 100644
--- a/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
+++ b/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp
@@ -17,18 +17,14 @@
namespace llvm {
namespace jitlink {
-EHFrameParser::EHFrameParser(AtomGraph &G, Section &EHFrameSection,
- StringRef EHFrameContent,
- JITTargetAddress EHFrameAddress,
- Edge::Kind FDEToCIERelocKind,
- Edge::Kind FDEToTargetRelocKind)
- : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent),
- EHFrameAddress(EHFrameAddress),
- EHFrameReader(EHFrameContent, G.getEndianness()),
- FDEToCIERelocKind(FDEToCIERelocKind),
- FDEToTargetRelocKind(FDEToTargetRelocKind) {}
-
-Error EHFrameParser::atomize() {
+EHFrameBinaryParser::EHFrameBinaryParser(JITTargetAddress EHFrameAddress,
+ StringRef EHFrameContent,
+ unsigned PointerSize,
+ support::endianness Endianness)
+ : EHFrameAddress(EHFrameAddress), EHFrameContent(EHFrameContent),
+ PointerSize(PointerSize), EHFrameReader(EHFrameContent, Endianness) {}
+
+Error EHFrameBinaryParser::addToGraph() {
while (!EHFrameReader.empty()) {
size_t RecordOffset = EHFrameReader.getOffset();
@@ -38,44 +34,39 @@ Error EHFrameParser::atomize() {
<< " (offset " << RecordOffset << ")\n";
});
- size_t CIELength = 0;
- uint32_t CIELengthField;
- if (auto Err = EHFrameReader.readInteger(CIELengthField))
+ size_t RecordLength = 0;
+ uint32_t RecordLengthField;
+ if (auto Err = EHFrameReader.readInteger(RecordLengthField))
return Err;
- // Process CIE length/extended-length fields to build the atom.
+ // Process CIE/FDE length/extended-length fields to build the blocks.
//
// The value of these fields describe the length of the *rest* of the CIE
// (not including data up to the end of the field itself) so we have to
- // bump CIELength to include the data up to the end of the field: 4 bytes
+ // bump RecordLength to include the data up to the end of the field: 4 bytes
// for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength.
- if (CIELengthField == 0) // Length 0 means end of __eh_frame section.
+ if (RecordLengthField == 0) // Length 0 means end of __eh_frame section.
break;
// If the regular length field's value is 0xffffffff, use extended length.
- if (CIELengthField == 0xffffffff) {
- uint64_t CIEExtendedLengthField;
- if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField))
+ if (RecordLengthField == 0xffffffff) {
+ uint64_t ExtendedLengthField;
+ if (auto Err = EHFrameReader.readInteger(ExtendedLengthField))
return Err;
- if (CIEExtendedLengthField > EHFrameReader.bytesRemaining())
+ if (ExtendedLengthField > EHFrameReader.bytesRemaining())
return make_error<JITLinkError>("CIE record extends past the end of "
"the __eh_frame section");
- if (CIEExtendedLengthField + 12 > std::numeric_limits<size_t>::max())
+ if (ExtendedLengthField + 12 > std::numeric_limits<size_t>::max())
return make_error<JITLinkError>("CIE record too large to process");
- CIELength = CIEExtendedLengthField + 12;
+ RecordLength = ExtendedLengthField + 12;
} else {
- if (CIELengthField > EHFrameReader.bytesRemaining())
+ if (RecordLengthField > EHFrameReader.bytesRemaining())
return make_error<JITLinkError>("CIE record extends past the end of "
"the __eh_frame section");
- CIELength = CIELengthField + 4;
+ RecordLength = RecordLengthField + 4;
}
- LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n");
-
- // Add an atom for this record.
- CurRecordAtom = &G.addAnonymousAtom(
- EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize());
- CurRecordAtom->setContent(EHFrameContent.substr(RecordOffset, CIELength));
+ LLVM_DEBUG(dbgs() << " length: " << RecordLength << "\n");
// Read the CIE Pointer.
size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset();
@@ -85,21 +76,24 @@ Error EHFrameParser::atomize() {
// Based on the CIE pointer value, parse this as a CIE or FDE record.
if (CIEPointer == 0) {
- if (auto Err = processCIE())
+ if (auto Err = processCIE(RecordOffset, RecordLength))
return Err;
} else {
- if (auto Err = processFDE(CIEPointerAddress, CIEPointer))
+ if (auto Err = processFDE(RecordOffset, RecordLength, CIEPointerAddress,
+ CIEPointer))
return Err;
}
- EHFrameReader.setOffset(RecordOffset + CIELength);
+ EHFrameReader.setOffset(RecordOffset + RecordLength);
}
return Error::success();
}
-Expected<EHFrameParser::AugmentationInfo>
-EHFrameParser::parseAugmentationString() {
+void EHFrameBinaryParser::anchor() {}
+
+Expected<EHFrameBinaryParser::AugmentationInfo>
+EHFrameBinaryParser::parseAugmentationString() {
AugmentationInfo AugInfo;
uint8_t NextChar;
uint8_t *NextField = &AugInfo.Fields[0];
@@ -139,14 +133,14 @@ EHFrameParser::parseAugmentationString() {
return std::move(AugInfo);
}
-Expected<JITTargetAddress> EHFrameParser::readAbsolutePointer() {
+Expected<JITTargetAddress> EHFrameBinaryParser::readAbsolutePointer() {
static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t),
"Result must be able to hold a uint64_t");
JITTargetAddress Addr;
- if (G.getPointerSize() == 8) {
+ if (PointerSize == 8) {
if (auto Err = EHFrameReader.readInteger(Addr))
return std::move(Err);
- } else if (G.getPointerSize() == 4) {
+ } else if (PointerSize == 4) {
uint32_t Addr32;
if (auto Err = EHFrameReader.readInteger(Addr32))
return std::move(Err);
@@ -156,14 +150,19 @@ Expected<JITTargetAddress> EHFrameParser::readAbsolutePointer() {
return Addr;
}
-Error EHFrameParser::processCIE() {
+Error EHFrameBinaryParser::processCIE(size_t RecordOffset,
+ size_t RecordLength) {
// Use the dwarf namespace for convenient access to pointer encoding
// constants.
using namespace dwarf;
LLVM_DEBUG(dbgs() << " Record is CIE\n");
- CIEInformation CIEInfo(*CurRecordAtom);
+ auto &CIESymbol =
+ createCIERecord(EHFrameAddress + RecordOffset,
+ EHFrameContent.substr(RecordOffset, RecordLength));
+
+ CIEInformation CIEInfo(CIESymbol);
uint8_t Version = 0;
if (auto Err = EHFrameReader.readInteger(Version))
@@ -179,7 +178,7 @@ Error EHFrameParser::processCIE() {
// Skip the EH Data field if present.
if (AugInfo->EHDataFieldPresent)
- if (auto Err = EHFrameReader.skip(G.getPointerSize()))
+ if (auto Err = EHFrameReader.skip(PointerSize))
return Err;
// Read and sanity check the code alignment factor.
@@ -226,7 +225,7 @@ Error EHFrameParser::processCIE() {
return make_error<JITLinkError>(
"Unsupported LSDA pointer encoding " +
formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " +
- formatv("{0:x16}", CurRecordAtom->getAddress()));
+ formatv("{0:x16}", CIESymbol.getAddress()));
break;
}
case 'P': {
@@ -239,7 +238,7 @@ Error EHFrameParser::processCIE() {
"Unspported personality pointer "
"encoding " +
formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " +
- formatv("{0:x16}", CurRecordAtom->getAddress()));
+ formatv("{0:x16}", CIESymbol.getAddress()));
uint32_t PersonalityPointerAddress;
if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress))
return Err;
@@ -254,7 +253,7 @@ Error EHFrameParser::processCIE() {
"Unsupported FDE address pointer "
"encoding " +
formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " +
- formatv("{0:x16}", CurRecordAtom->getAddress()));
+ formatv("{0:x16}", CIESymbol.getAddress()));
break;
}
default:
@@ -267,15 +266,16 @@ Error EHFrameParser::processCIE() {
return make_error<JITLinkError>("Read past the end of the augmentation "
"data while parsing fields");
- assert(!CIEInfos.count(CurRecordAtom->getAddress()) &&
+ assert(!CIEInfos.count(CIESymbol.getAddress()) &&
"Multiple CIEs recorded at the same address?");
- CIEInfos[CurRecordAtom->getAddress()] = std::move(CIEInfo);
+ CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo);
return Error::success();
}
-Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress,
- uint32_t CIEPointer) {
+Error EHFrameBinaryParser::processFDE(size_t RecordOffset, size_t RecordLength,
+ JITTargetAddress CIEPointerAddress,
+ uint32_t CIEPointer) {
LLVM_DEBUG(dbgs() << " Record is FDE\n");
LLVM_DEBUG({
@@ -286,16 +286,11 @@ Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress,
auto CIEInfoItr = CIEInfos.find(CIEPointerAddress - CIEPointer);
if (CIEInfoItr == CIEInfos.end())
return make_error<JITLinkError>(
- "FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress()) +
+ "FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset) +
" points to non-existant CIE at " +
formatv("{0:x16}", CIEPointerAddress - CIEPointer));
auto &CIEInfo = CIEInfoItr->second;
- // The CIEPointer looks good. Add a relocation.
- CurRecordAtom->addEdge(FDEToCIERelocKind,
- CIEPointerAddress - CurRecordAtom->getAddress(),
- *CIEInfo.CIEAtom, 0);
-
// Read and sanity check the PC-start pointer and size.
JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset();
@@ -305,83 +300,68 @@ Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress,
JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta;
LLVM_DEBUG({
- dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n";
+ dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n";
});
- auto *TargetAtom = G.getAtomByAddress(PCBegin);
+ auto *TargetSymbol = getSymbolAtAddress(PCBegin);
- if (!TargetAtom)
+ if (!TargetSymbol)
return make_error<JITLinkError>("FDE PC-begin " +
formatv("{0:x16}", PCBegin) +
- " does not point at atom");
+ " does not point at symbol");
- if (TargetAtom->getAddress() != PCBegin)
+ if (TargetSymbol->getAddress() != PCBegin)
return make_error<JITLinkError>(
"FDE PC-begin " + formatv("{0:x16}", PCBegin) +
- " does not point to start of atom at " +
- formatv("{0:x16}", TargetAtom->getAddress()));
-
- LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n");
+ " does not point to start of symbol at " +
+ formatv("{0:x16}", TargetSymbol->getAddress()));
- // The PC-start pointer and size look good. Add relocations.
- CurRecordAtom->addEdge(FDEToTargetRelocKind,
- PCBeginAddress - CurRecordAtom->getAddress(),
- *TargetAtom, 0);
-
- // Add a keep-alive relocation from the function to the FDE to ensure it is
- // not dead stripped.
- TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0);
+ LLVM_DEBUG(dbgs() << " FDE target: " << *TargetSymbol << "\n");
// Skip over the PC range size field.
- if (auto Err = EHFrameReader.skip(G.getPointerSize()))
+ if (auto Err = EHFrameReader.skip(PointerSize))
return Err;
+ Symbol *LSDASymbol = nullptr;
+ JITTargetAddress LSDAAddress = 0;
if (CIEInfo.FDEsHaveLSDAField) {
uint64_t AugmentationDataSize;
if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize))
return Err;
- if (AugmentationDataSize != G.getPointerSize())
+ if (AugmentationDataSize != PointerSize)
return make_error<JITLinkError>(
"Unexpected FDE augmentation data size (expected " +
- Twine(G.getPointerSize()) + ", got " + Twine(AugmentationDataSize) +
- ") for FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress()));
- JITTargetAddress LSDAAddress = EHFrameAddress + EHFrameReader.getOffset();
+ Twine(PointerSize) + ", got " + Twine(AugmentationDataSize) +
+ ") for FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset));
+ LSDAAddress = EHFrameAddress + EHFrameReader.getOffset();
auto LSDADelta = readAbsolutePointer();
if (!LSDADelta)
return LSDADelta.takeError();
JITTargetAddress LSDA = LSDAAddress + *LSDADelta;
- auto *LSDAAtom = G.getAtomByAddress(LSDA);
+ LSDASymbol = getSymbolAtAddress(LSDA);
- if (!LSDAAtom)
+ if (!LSDASymbol)
return make_error<JITLinkError>("FDE LSDA " + formatv("{0:x16}", LSDA) +
- " does not point at atom");
+ " does not point at symbol");
- if (LSDAAtom->getAddress() != LSDA)
+ if (LSDASymbol->getAddress() != LSDA)
return make_error<JITLinkError>(
"FDE LSDA " + formatv("{0:x16}", LSDA) +
- " does not point to start of atom at " +
- formatv("{0:x16}", LSDAAtom->getAddress()));
-
- LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDAAtom << "\n");
+ " does not point to start of symbol at " +
+ formatv("{0:x16}", LSDASymbol->getAddress()));
- // LSDA looks good. Add relocations.
- CurRecordAtom->addEdge(FDEToTargetRelocKind,
- LSDAAddress - CurRecordAtom->getAddress(), *LSDAAtom,
- 0);
+ LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDASymbol << "\n");
}
- return Error::success();
-}
+ JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset;
+ auto FDESymbol = createFDERecord(
+ RecordAddress, EHFrameContent.substr(RecordOffset, RecordLength),
+ *CIEInfo.CIESymbol, CIEPointerAddress - RecordAddress, *TargetSymbol,
+ PCBeginAddress - RecordAddress, LSDASymbol, LSDAAddress - RecordAddress);
-Error addEHFrame(AtomGraph &G, Section &EHFrameSection,
- StringRef EHFrameContent, JITTargetAddress EHFrameAddress,
- Edge::Kind FDEToCIERelocKind,
- Edge::Kind FDEToTargetRelocKind) {
- return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress,
- FDEToCIERelocKind, FDEToTargetRelocKind)
- .atomize();
+ return FDESymbol.takeError();
}
// Determine whether we can register EH tables.
@@ -451,11 +431,13 @@ static Error deregisterFrameWrapper(const void *P) {
template <typename HandleFDEFn>
Error walkAppleEHFrameSection(const char *const SectionStart,
+ size_t SectionSize,
HandleFDEFn HandleFDE) {
const char *CurCFIRecord = SectionStart;
+ const char *End = SectionStart + SectionSize;
uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord);
- while (Size != 0) {
+ while (CurCFIRecord != End && Size != 0) {
const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4);
if (Size == 0xffffffff)
Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12;
@@ -484,10 +466,12 @@ Error walkAppleEHFrameSection(const char *const SectionStart,
#endif // __APPLE__
-Error registerEHFrameSection(const void *EHFrameSectionAddr) {
+Error registerEHFrameSection(const void *EHFrameSectionAddr,
+ size_t EHFrameSectionSize) {
#ifdef __APPLE__
// On Darwin __register_frame has to be called for each FDE entry.
return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr),
+ EHFrameSectionSize,
registerFrameWrapper);
#else
// On Linux __register_frame takes a single argument:
@@ -499,9 +483,11 @@ Error registerEHFrameSection(const void *EHFrameSectionAddr) {
#endif
}
-Error deregisterEHFrameSection(const void *EHFrameSectionAddr) {
+Error deregisterEHFrameSection(const void *EHFrameSectionAddr,
+ size_t EHFrameSectionSize) {
#ifdef __APPLE__
return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr),
+ EHFrameSectionSize,
deregisterFrameWrapper);
#else
return deregisterFrameWrapper(EHFrameSectionAddr);
@@ -517,23 +503,31 @@ InProcessEHFrameRegistrar &InProcessEHFrameRegistrar::getInstance() {
InProcessEHFrameRegistrar::InProcessEHFrameRegistrar() {}
-AtomGraphPassFunction
+LinkGraphPassFunction
createEHFrameRecorderPass(const Triple &TT,
- StoreFrameAddressFunction StoreFrameAddress) {
+ StoreFrameRangeFunction StoreRangeAddress) {
const char *EHFrameSectionName = nullptr;
if (TT.getObjectFormat() == Triple::MachO)
EHFrameSectionName = "__eh_frame";
else
EHFrameSectionName = ".eh_frame";
- auto RecordEHFrame = [EHFrameSectionName,
- StoreFrameAddress](AtomGraph &G) -> Error {
- // Search for a non-empty eh-frame and record the address of the first atom
- // in it.
+ auto RecordEHFrame =
+ [EHFrameSectionName,
+ StoreFrameRange = std::move(StoreRangeAddress)](LinkGraph &G) -> Error {
+ // Search for a non-empty eh-frame and record the address of the first
+ // symbol in it.
JITTargetAddress Addr = 0;
- if (auto *S = G.findSectionByName(EHFrameSectionName))
- Addr = S->getRange().getStart();
- StoreFrameAddress(Addr);
+ size_t Size = 0;
+ if (auto *S = G.findSectionByName(EHFrameSectionName)) {
+ auto R = SectionRange(*S);
+ Addr = R.getStart();
+ Size = R.getSize();
+ }
+ if (Addr == 0 && Size != 0)
+ return make_error<JITLinkError>("__eh_frame section can not have zero "
+ "address with non-zero size");
+ StoreFrameRange(Addr, Size);
return Error::success();
};
diff --git a/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h
index d679edef7ea6..6f9f68ad8382 100644
--- a/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h
+++ b/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h
@@ -21,18 +21,31 @@
namespace llvm {
namespace jitlink {
-/// A generic parser for eh-frame sections.
+/// A generic binary parser for eh-frame sections.
///
-/// Adds atoms representing CIE and FDE entries, using the given FDE-to-CIE and
-/// FDEToTarget relocation kinds.
-class EHFrameParser {
+/// Adds blocks and symbols representing CIE and FDE entries to a JITLink graph.
+///
+/// This parser assumes that the user has already verified that the EH-frame's
+/// address range does not overlap any other section/symbol, so that generated
+/// CIE/FDE records do not overlap other sections/symbols.
+class EHFrameBinaryParser {
public:
- EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent,
- JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind,
- Edge::Kind FDEToTargetRelocKind);
- Error atomize();
+ EHFrameBinaryParser(JITTargetAddress EHFrameAddress, StringRef EHFrameContent,
+ unsigned PointerSize, support::endianness Endianness);
+ virtual ~EHFrameBinaryParser() {}
+
+ Error addToGraph();
private:
+ virtual void anchor();
+ virtual Symbol *getSymbolAtAddress(JITTargetAddress Addr) = 0;
+ virtual Symbol &createCIERecord(JITTargetAddress RecordAddr,
+ StringRef RecordContent) = 0;
+ virtual Expected<Symbol &>
+ createFDERecord(JITTargetAddress RecordAddr, StringRef RecordContent,
+ Symbol &CIE, size_t CIEOffset, Symbol &Func,
+ size_t FuncOffset, Symbol *LSDA, size_t LSDAOffset) = 0;
+
struct AugmentationInfo {
bool AugmentationDataPresent = false;
bool EHDataFieldPresent = false;
@@ -41,31 +54,24 @@ private:
Expected<AugmentationInfo> parseAugmentationString();
Expected<JITTargetAddress> readAbsolutePointer();
- Error processCIE();
- Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer);
+ Error processCIE(size_t RecordOffset, size_t RecordLength);
+ Error processFDE(size_t RecordOffset, size_t RecordLength,
+ JITTargetAddress CIEPointerOffset, uint32_t CIEPointer);
struct CIEInformation {
CIEInformation() = default;
- CIEInformation(DefinedAtom &CIEAtom) : CIEAtom(&CIEAtom) {}
- DefinedAtom *CIEAtom = nullptr;
+ CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {}
+ Symbol *CIESymbol = nullptr;
bool FDEsHaveLSDAField = false;
};
- AtomGraph &G;
- Section &EHFrameSection;
- StringRef EHFrameContent;
JITTargetAddress EHFrameAddress;
+ StringRef EHFrameContent;
+ unsigned PointerSize;
BinaryStreamReader EHFrameReader;
- DefinedAtom *CurRecordAtom = nullptr;
DenseMap<JITTargetAddress, CIEInformation> CIEInfos;
- Edge::Kind FDEToCIERelocKind;
- Edge::Kind FDEToTargetRelocKind;
};
-Error addEHFrame(AtomGraph &G, Section &EHFrameSection,
- StringRef EHFrameContent, JITTargetAddress EHFrameAddress,
- Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind);
-
} // end namespace jitlink
} // end namespace llvm
diff --git a/lib/ExecutionEngine/JITLink/JITLink.cpp b/lib/ExecutionEngine/JITLink/JITLink.cpp
index 9d0a7459dc09..1e19038951ac 100644
--- a/lib/ExecutionEngine/JITLink/JITLink.cpp
+++ b/lib/ExecutionEngine/JITLink/JITLink.cpp
@@ -56,95 +56,151 @@ std::error_code JITLinkError::convertToErrorCode() const {
return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory);
}
-const StringRef getGenericEdgeKindName(Edge::Kind K) {
+const char *getGenericEdgeKindName(Edge::Kind K) {
switch (K) {
case Edge::Invalid:
return "INVALID RELOCATION";
case Edge::KeepAlive:
return "Keep-Alive";
- case Edge::LayoutNext:
- return "Layout-Next";
default:
llvm_unreachable("Unrecognized relocation kind");
}
}
-raw_ostream &operator<<(raw_ostream &OS, const Atom &A) {
+const char *getLinkageName(Linkage L) {
+ switch (L) {
+ case Linkage::Strong:
+ return "strong";
+ case Linkage::Weak:
+ return "weak";
+ }
+ llvm_unreachable("Unrecognized llvm.jitlink.Linkage enum");
+}
+
+const char *getScopeName(Scope S) {
+ switch (S) {
+ case Scope::Default:
+ return "default";
+ case Scope::Hidden:
+ return "hidden";
+ case Scope::Local:
+ return "local";
+ }
+ llvm_unreachable("Unrecognized llvm.jitlink.Scope enum");
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const Block &B) {
+ return OS << formatv("{0:x16}", B.getAddress()) << " -- "
+ << formatv("{0:x16}", B.getAddress() + B.getSize()) << ": "
+ << (B.isZeroFill() ? "zero-fill" : "content")
+ << ", align = " << B.getAlignment()
+ << ", align-ofs = " << B.getAlignmentOffset()
+ << ", section = " << B.getSection().getName();
+}
+
+raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) {
OS << "<";
- if (A.getName().empty())
- OS << "anon@" << format("0x%016" PRIx64, A.getAddress());
+ if (Sym.getName().empty())
+ OS << "*anon*";
else
- OS << A.getName();
- OS << " [";
- if (A.isDefined()) {
- auto &DA = static_cast<const DefinedAtom &>(A);
- OS << " section=" << DA.getSection().getName();
- if (DA.isLive())
- OS << " live";
- if (DA.shouldDiscard())
- OS << " should-discard";
- } else
- OS << " external";
- OS << " ]>";
+ OS << Sym.getName();
+ OS << ": flags = ";
+ switch (Sym.getLinkage()) {
+ case Linkage::Strong:
+ OS << 'S';
+ break;
+ case Linkage::Weak:
+ OS << 'W';
+ break;
+ }
+ switch (Sym.getScope()) {
+ case Scope::Default:
+ OS << 'D';
+ break;
+ case Scope::Hidden:
+ OS << 'H';
+ break;
+ case Scope::Local:
+ OS << 'L';
+ break;
+ }
+ OS << (Sym.isLive() ? '+' : '-')
+ << ", size = " << formatv("{0:x8}", Sym.getSize())
+ << ", addr = " << formatv("{0:x16}", Sym.getAddress()) << " ("
+ << formatv("{0:x16}", Sym.getAddressable().getAddress()) << " + "
+ << formatv("{0:x8}", Sym.getOffset());
+ if (Sym.isDefined())
+ OS << " " << Sym.getBlock().getSection().getName();
+ OS << ")>";
return OS;
}
-void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E,
+void printEdge(raw_ostream &OS, const Block &B, const Edge &E,
StringRef EdgeKindName) {
- OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset())
- << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName
- << " -> " << E.getTarget() << " + " << E.getAddend();
+ OS << "edge@" << formatv("{0:x16}", B.getAddress() + E.getOffset()) << ": "
+ << formatv("{0:x16}", B.getAddress()) << " + " << E.getOffset() << " -- "
+ << EdgeKindName << " -> " << E.getTarget() << " + " << E.getAddend();
}
Section::~Section() {
- for (auto *DA : DefinedAtoms)
- DA->~DefinedAtom();
+ for (auto *Sym : Symbols)
+ Sym->~Symbol();
}
-void AtomGraph::dump(raw_ostream &OS,
+LinkGraph::~LinkGraph() {
+ // Destroy blocks.
+ for (auto *B : Blocks)
+ B->~Block();
+}
+
+void LinkGraph::dump(raw_ostream &OS,
std::function<StringRef(Edge::Kind)> EdgeKindToName) {
if (!EdgeKindToName)
EdgeKindToName = [](Edge::Kind K) { return StringRef(); };
- OS << "Defined atoms:\n";
- for (auto *DA : defined_atoms()) {
- OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA
+ OS << "Symbols:\n";
+ for (auto *Sym : defined_symbols()) {
+ OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym
<< "\n";
- for (auto &E : DA->edges()) {
- OS << " ";
- StringRef EdgeName = (E.getKind() < Edge::FirstRelocation
- ? getGenericEdgeKindName(E.getKind())
- : EdgeKindToName(E.getKind()));
-
- if (!EdgeName.empty())
- printEdge(OS, *DA, E, EdgeName);
- else {
- auto EdgeNumberString = std::to_string(E.getKind());
- printEdge(OS, *DA, E, EdgeNumberString);
+ if (Sym->isDefined()) {
+ for (auto &E : Sym->getBlock().edges()) {
+ OS << " ";
+ StringRef EdgeName = (E.getKind() < Edge::FirstRelocation
+ ? getGenericEdgeKindName(E.getKind())
+ : EdgeKindToName(E.getKind()));
+
+ if (!EdgeName.empty())
+ printEdge(OS, Sym->getBlock(), E, EdgeName);
+ else {
+ auto EdgeNumberString = std::to_string(E.getKind());
+ printEdge(OS, Sym->getBlock(), E, EdgeNumberString);
+ }
+ OS << "\n";
}
- OS << "\n";
}
}
- OS << "Absolute atoms:\n";
- for (auto *A : absolute_atoms())
- OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A
+ OS << "Absolute symbols:\n";
+ for (auto *Sym : absolute_symbols())
+ OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym
<< "\n";
- OS << "External atoms:\n";
- for (auto *A : external_atoms())
- OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A
+ OS << "External symbols:\n";
+ for (auto *Sym : external_symbols())
+ OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym
<< "\n";
}
+void JITLinkAsyncLookupContinuation::anchor() {}
+
JITLinkContext::~JITLinkContext() {}
bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const {
return true;
}
-AtomGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
- return AtomGraphPassFunction();
+LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const {
+ return LinkGraphPassFunction();
}
Error JITLinkContext::modifyPassConfig(const Triple &TT,
@@ -152,9 +208,9 @@ Error JITLinkContext::modifyPassConfig(const Triple &TT,
return Error::success();
}
-Error markAllAtomsLive(AtomGraph &G) {
- for (auto *DA : G.defined_atoms())
- DA->setLive(true);
+Error markAllSymbolsLive(LinkGraph &G) {
+ for (auto *Sym : G.defined_symbols())
+ Sym->setLive(true);
return Error::success();
}
diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
index 96e074da122b..d4270b5aa796 100644
--- a/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
+++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp
@@ -11,7 +11,6 @@
//===----------------------------------------------------------------------===//
#include "JITLinkGeneric.h"
-#include "EHFrameSupportImpl.h"
#include "llvm/Support/BinaryStreamReader.h"
#include "llvm/Support/MemoryBuffer.h"
@@ -25,7 +24,7 @@ JITLinkerBase::~JITLinkerBase() {}
void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
- // Build the atom graph.
+ // Build the link graph.
if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer()))
G = std::move(*GraphOrErr);
else
@@ -33,33 +32,33 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
assert(G && "Graph should have been created by buildGraph above");
// Prune and optimize the graph.
- if (auto Err = runPasses(Passes.PrePrunePasses, *G))
+ if (auto Err = runPasses(Passes.PrePrunePasses))
return Ctx->notifyFailed(std::move(Err));
LLVM_DEBUG({
- dbgs() << "Atom graph \"" << G->getName() << "\" pre-pruning:\n";
+ dbgs() << "Link graph \"" << G->getName() << "\" pre-pruning:\n";
dumpGraph(dbgs());
});
prune(*G);
LLVM_DEBUG({
- dbgs() << "Atom graph \"" << G->getName() << "\" post-pruning:\n";
+ dbgs() << "Link graph \"" << G->getName() << "\" post-pruning:\n";
dumpGraph(dbgs());
});
// Run post-pruning passes.
- if (auto Err = runPasses(Passes.PostPrunePasses, *G))
+ if (auto Err = runPasses(Passes.PostPrunePasses))
return Ctx->notifyFailed(std::move(Err));
- // Sort atoms into segments.
- layOutAtoms();
+ // Sort blocks into segments.
+ auto Layout = layOutBlocks();
// Allocate memory for segments.
if (auto Err = allocateSegments(Layout))
return Ctx->notifyFailed(std::move(Err));
- // Notify client that the defined atoms have been assigned addresses.
+ // Notify client that the defined symbols have been assigned addresses.
Ctx->notifyResolved(*G);
auto ExternalSymbols = getExternalSymbolNames();
@@ -74,42 +73,42 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) {
// [Self=std::move(Self)](Expected<AsyncLookupResult> Result) {
// Self->linkPhase2(std::move(Self), std::move(Result));
// });
- //
- // FIXME: Use move capture once we have c++14.
auto *TmpCtx = Ctx.get();
- auto *UnownedSelf = Self.release();
- auto Phase2Continuation =
- [UnownedSelf](Expected<AsyncLookupResult> LookupResult) {
- std::unique_ptr<JITLinkerBase> Self(UnownedSelf);
- UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult));
- };
- TmpCtx->lookup(std::move(ExternalSymbols), std::move(Phase2Continuation));
+ TmpCtx->lookup(std::move(ExternalSymbols),
+ createLookupContinuation(
+ [S = std::move(Self), L = std::move(Layout)](
+ Expected<AsyncLookupResult> LookupResult) mutable {
+ auto &TmpSelf = *S;
+ TmpSelf.linkPhase2(std::move(S), std::move(LookupResult),
+ std::move(L));
+ }));
}
void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self,
- Expected<AsyncLookupResult> LR) {
+ Expected<AsyncLookupResult> LR,
+ SegmentLayoutMap Layout) {
// If the lookup failed, bail out.
if (!LR)
return deallocateAndBailOut(LR.takeError());
- // Assign addresses to external atoms.
+ // Assign addresses to external addressables.
applyLookupResult(*LR);
LLVM_DEBUG({
- dbgs() << "Atom graph \"" << G->getName() << "\" before copy-and-fixup:\n";
+ dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n";
dumpGraph(dbgs());
});
- // Copy atom content to working memory and fix up.
- if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc))
+ // Copy block content to working memory and fix up.
+ if (auto Err = copyAndFixUpBlocks(Layout, *Alloc))
return deallocateAndBailOut(std::move(Err));
LLVM_DEBUG({
- dbgs() << "Atom graph \"" << G->getName() << "\" after copy-and-fixup:\n";
+ dbgs() << "Link graph \"" << G->getName() << "\" after copy-and-fixup:\n";
dumpGraph(dbgs());
});
- if (auto Err = runPasses(Passes.PostFixupPasses, *G))
+ if (auto Err = runPasses(Passes.PostFixupPasses))
return deallocateAndBailOut(std::move(Err));
// FIXME: Use move capture once we have c++14.
@@ -128,82 +127,38 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err) {
Ctx->notifyFinalized(std::move(Alloc));
}
-Error JITLinkerBase::runPasses(AtomGraphPassList &Passes, AtomGraph &G) {
+Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) {
for (auto &P : Passes)
- if (auto Err = P(G))
+ if (auto Err = P(*G))
return Err;
return Error::success();
}
-void JITLinkerBase::layOutAtoms() {
- // Group sections by protections, and whether or not they're zero-fill.
- for (auto &S : G->sections()) {
+JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() {
- // Skip empty sections.
- if (S.atoms_empty())
- continue;
+ SegmentLayoutMap Layout;
- auto &SL = Layout[S.getProtectionFlags()];
- if (S.isZeroFill())
- SL.ZeroFillSections.push_back(SegmentLayout::SectionLayout(S));
+ /// Partition blocks based on permissions and content vs. zero-fill.
+ for (auto *B : G->blocks()) {
+ auto &SegLists = Layout[B->getSection().getProtectionFlags()];
+ if (!B->isZeroFill())
+ SegLists.ContentBlocks.push_back(B);
else
- SL.ContentSections.push_back(SegmentLayout::SectionLayout(S));
+ SegLists.ZeroFillBlocks.push_back(B);
}
- // Sort sections within the layout by ordinal.
- {
- auto CompareByOrdinal = [](const SegmentLayout::SectionLayout &LHS,
- const SegmentLayout::SectionLayout &RHS) {
- return LHS.S->getSectionOrdinal() < RHS.S->getSectionOrdinal();
+ /// Sort blocks within each list.
+ for (auto &KV : Layout) {
+
+ auto CompareBlocks = [](const Block *LHS, const Block *RHS) {
+ if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal())
+ return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal();
+ return LHS->getOrdinal() < RHS->getOrdinal();
};
- for (auto &KV : Layout) {
- auto &SL = KV.second;
- std::sort(SL.ContentSections.begin(), SL.ContentSections.end(),
- CompareByOrdinal);
- std::sort(SL.ZeroFillSections.begin(), SL.ZeroFillSections.end(),
- CompareByOrdinal);
- }
- }
- // Add atoms to the sections.
- for (auto &KV : Layout) {
- auto &SL = KV.second;
- for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) {
- for (auto &SI : *SIList) {
- // First build the set of layout-heads (i.e. "heads" of layout-next
- // chains) by copying the section atoms, then eliminating any that
- // appear as layout-next targets.
- DenseSet<DefinedAtom *> LayoutHeads;
- for (auto *DA : SI.S->atoms())
- LayoutHeads.insert(DA);
-
- for (auto *DA : SI.S->atoms())
- if (DA->hasLayoutNext())
- LayoutHeads.erase(&DA->getLayoutNext());
-
- // Next, sort the layout heads by address order.
- std::vector<DefinedAtom *> OrderedLayoutHeads;
- OrderedLayoutHeads.reserve(LayoutHeads.size());
- for (auto *DA : LayoutHeads)
- OrderedLayoutHeads.push_back(DA);
-
- // Now sort the list of layout heads by address.
- std::sort(OrderedLayoutHeads.begin(), OrderedLayoutHeads.end(),
- [](const DefinedAtom *LHS, const DefinedAtom *RHS) {
- return LHS->getAddress() < RHS->getAddress();
- });
-
- // Now populate the SI.Atoms field by appending each of the chains.
- for (auto *DA : OrderedLayoutHeads) {
- SI.Atoms.push_back(DA);
- while (DA->hasLayoutNext()) {
- auto &Next = DA->getLayoutNext();
- SI.Atoms.push_back(&Next);
- DA = &Next;
- }
- }
- }
- }
+ auto &SegLists = KV.second;
+ llvm::sort(SegLists.ContentBlocks, CompareBlocks);
+ llvm::sort(SegLists.ZeroFillBlocks, CompareBlocks);
}
LLVM_DEBUG({
@@ -213,18 +168,16 @@ void JITLinkerBase::layOutAtoms() {
<< static_cast<sys::Memory::ProtectionFlags>(KV.first) << ":\n";
auto &SL = KV.second;
for (auto &SIEntry :
- {std::make_pair(&SL.ContentSections, "content sections"),
- std::make_pair(&SL.ZeroFillSections, "zero-fill sections")}) {
- auto &SIList = *SIEntry.first;
+ {std::make_pair(&SL.ContentBlocks, "content block"),
+ std::make_pair(&SL.ZeroFillBlocks, "zero-fill block")}) {
dbgs() << " " << SIEntry.second << ":\n";
- for (auto &SI : SIList) {
- dbgs() << " " << SI.S->getName() << ":\n";
- for (auto *DA : SI.Atoms)
- dbgs() << " " << *DA << "\n";
- }
+ for (auto *B : *SIEntry.first)
+ dbgs() << " " << *B << "\n";
}
}
});
+
+ return Layout;
}
Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) {
@@ -234,74 +187,36 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) {
JITLinkMemoryManager::SegmentsRequestMap Segments;
for (auto &KV : Layout) {
auto &Prot = KV.first;
- auto &SegLayout = KV.second;
+ auto &SegLists = KV.second;
+
+ uint64_t SegAlign = 1;
// Calculate segment content size.
size_t SegContentSize = 0;
- for (auto &SI : SegLayout.ContentSections) {
- assert(!SI.S->atoms_empty() && "Sections in layout must not be empty");
- assert(!SI.Atoms.empty() && "Section layouts must not be empty");
-
- // Bump to section alignment before processing atoms.
- SegContentSize = alignTo(SegContentSize, SI.S->getAlignment());
-
- for (auto *DA : SI.Atoms) {
- SegContentSize = alignTo(SegContentSize, DA->getAlignment());
- SegContentSize += DA->getSize();
- }
+ for (auto *B : SegLists.ContentBlocks) {
+ SegAlign = std::max(SegAlign, B->getAlignment());
+ SegContentSize = alignToBlock(SegContentSize, *B);
+ SegContentSize += B->getSize();
}
- // Get segment content alignment.
- unsigned SegContentAlign = 1;
- if (!SegLayout.ContentSections.empty()) {
- auto &FirstContentSection = SegLayout.ContentSections.front();
- SegContentAlign =
- std::max(FirstContentSection.S->getAlignment(),
- FirstContentSection.Atoms.front()->getAlignment());
- }
+ uint64_t SegZeroFillStart = SegContentSize;
+ uint64_t SegZeroFillEnd = SegZeroFillStart;
- // Calculate segment zero-fill size.
- uint64_t SegZeroFillSize = 0;
- for (auto &SI : SegLayout.ZeroFillSections) {
- assert(!SI.S->atoms_empty() && "Sections in layout must not be empty");
- assert(!SI.Atoms.empty() && "Section layouts must not be empty");
-
- // Bump to section alignment before processing atoms.
- SegZeroFillSize = alignTo(SegZeroFillSize, SI.S->getAlignment());
-
- for (auto *DA : SI.Atoms) {
- SegZeroFillSize = alignTo(SegZeroFillSize, DA->getAlignment());
- SegZeroFillSize += DA->getSize();
- }
- }
-
- // Calculate segment zero-fill alignment.
- uint32_t SegZeroFillAlign = 1;
-
- if (!SegLayout.ZeroFillSections.empty()) {
- auto &FirstZeroFillSection = SegLayout.ZeroFillSections.front();
- SegZeroFillAlign =
- std::max(FirstZeroFillSection.S->getAlignment(),
- FirstZeroFillSection.Atoms.front()->getAlignment());
+ for (auto *B : SegLists.ZeroFillBlocks) {
+ SegAlign = std::max(SegAlign, B->getAlignment());
+ SegZeroFillEnd = alignToBlock(SegZeroFillEnd, *B);
+ SegZeroFillEnd += B->getSize();
}
- if (SegContentSize == 0)
- SegContentAlign = SegZeroFillAlign;
-
- if (SegContentAlign % SegZeroFillAlign != 0)
- return make_error<JITLinkError>("First content atom alignment does not "
- "accommodate first zero-fill atom "
- "alignment");
-
- Segments[Prot] = {SegContentSize, SegContentAlign, SegZeroFillSize,
- SegZeroFillAlign};
+ Segments[Prot] = {SegAlign, SegContentSize,
+ SegZeroFillEnd - SegZeroFillStart};
LLVM_DEBUG({
dbgs() << (&KV == &*Layout.begin() ? "" : "; ")
- << static_cast<sys::Memory::ProtectionFlags>(Prot) << ": "
- << SegContentSize << " content bytes (alignment "
- << SegContentAlign << ") + " << SegZeroFillSize
- << " zero-fill bytes (alignment " << SegZeroFillAlign << ")";
+ << static_cast<sys::Memory::ProtectionFlags>(Prot)
+ << ": alignment = " << SegAlign
+ << ", content size = " << SegContentSize
+ << ", zero-fill size = " << (SegZeroFillEnd - SegZeroFillStart);
});
}
LLVM_DEBUG(dbgs() << " }\n");
@@ -320,22 +235,19 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) {
}
});
- // Update atom target addresses.
+ // Update block target addresses.
for (auto &KV : Layout) {
auto &Prot = KV.first;
auto &SL = KV.second;
- JITTargetAddress AtomTargetAddr =
+ JITTargetAddress NextBlockAddr =
Alloc->getTargetMemory(static_cast<sys::Memory::ProtectionFlags>(Prot));
- for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections})
- for (auto &SI : *SIList) {
- AtomTargetAddr = alignTo(AtomTargetAddr, SI.S->getAlignment());
- for (auto *DA : SI.Atoms) {
- AtomTargetAddr = alignTo(AtomTargetAddr, DA->getAlignment());
- DA->setAddress(AtomTargetAddr);
- AtomTargetAddr += DA->getSize();
- }
+ for (auto *SIList : {&SL.ContentBlocks, &SL.ZeroFillBlocks})
+ for (auto *B : *SIList) {
+ NextBlockAddr = alignToBlock(NextBlockAddr, *B);
+ B->setAddress(NextBlockAddr);
+ NextBlockAddr += B->getSize();
}
}
@@ -343,34 +255,35 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) {
}
DenseSet<StringRef> JITLinkerBase::getExternalSymbolNames() const {
- // Identify unresolved external atoms.
+ // Identify unresolved external symbols.
DenseSet<StringRef> UnresolvedExternals;
- for (auto *DA : G->external_atoms()) {
- assert(DA->getAddress() == 0 &&
+ for (auto *Sym : G->external_symbols()) {
+ assert(Sym->getAddress() == 0 &&
"External has already been assigned an address");
- assert(DA->getName() != StringRef() && DA->getName() != "" &&
+ assert(Sym->getName() != StringRef() && Sym->getName() != "" &&
"Externals must be named");
- UnresolvedExternals.insert(DA->getName());
+ UnresolvedExternals.insert(Sym->getName());
}
return UnresolvedExternals;
}
void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) {
- for (auto &KV : Result) {
- Atom &A = G->getAtomByName(KV.first);
- assert(A.getAddress() == 0 && "Atom already resolved");
- A.setAddress(KV.second.getAddress());
+ for (auto *Sym : G->external_symbols()) {
+ assert(Sym->getAddress() == 0 && "Symbol already resolved");
+ assert(!Sym->isDefined() && "Symbol being resolved is already defined");
+ assert(Result.count(Sym->getName()) && "Missing resolution for symbol");
+ Sym->getAddressable().setAddress(Result[Sym->getName()].getAddress());
}
LLVM_DEBUG({
dbgs() << "Externals after applying lookup result:\n";
- for (auto *A : G->external_atoms())
- dbgs() << " " << A->getName() << ": "
- << formatv("{0:x16}", A->getAddress()) << "\n";
+ for (auto *Sym : G->external_symbols())
+ dbgs() << " " << Sym->getName() << ": "
+ << formatv("{0:x16}", Sym->getAddress()) << "\n";
});
- assert(llvm::all_of(G->external_atoms(),
- [](Atom *A) { return A->getAddress() != 0; }) &&
- "All atoms should have been resolved by this point");
+ assert(llvm::all_of(G->external_symbols(),
+ [](Symbol *Sym) { return Sym->getAddress() != 0; }) &&
+ "All symbols should have been resolved by this point");
}
void JITLinkerBase::deallocateAndBailOut(Error Err) {
@@ -384,96 +297,60 @@ void JITLinkerBase::dumpGraph(raw_ostream &OS) {
G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); });
}
-void prune(AtomGraph &G) {
- std::vector<DefinedAtom *> Worklist;
- DenseMap<DefinedAtom *, std::vector<Edge *>> EdgesToUpdate;
-
- // Build the initial worklist from all atoms initially live.
- for (auto *DA : G.defined_atoms()) {
- if (!DA->isLive() || DA->shouldDiscard())
- continue;
-
- for (auto &E : DA->edges()) {
- if (!E.getTarget().isDefined())
- continue;
+void prune(LinkGraph &G) {
+ std::vector<Symbol *> Worklist;
+ DenseSet<Block *> VisitedBlocks;
- auto &EDT = static_cast<DefinedAtom &>(E.getTarget());
+ // Build the initial worklist from all symbols initially live.
+ for (auto *Sym : G.defined_symbols())
+ if (Sym->isLive())
+ Worklist.push_back(Sym);
- if (EDT.shouldDiscard())
- EdgesToUpdate[&EDT].push_back(&E);
- else if (E.isKeepAlive() && !EDT.isLive())
- Worklist.push_back(&EDT);
- }
- }
-
- // Propagate live flags to all atoms reachable from the initial live set.
+ // Propagate live flags to all symbols reachable from the initial live set.
while (!Worklist.empty()) {
- DefinedAtom &NextLive = *Worklist.back();
+ auto *Sym = Worklist.back();
Worklist.pop_back();
- assert(!NextLive.shouldDiscard() &&
- "should-discard nodes should never make it into the worklist");
+ auto &B = Sym->getBlock();
- // If this atom has already been marked as live, or is marked to be
- // discarded, then skip it.
- if (NextLive.isLive())
+ // Skip addressables that we've visited before.
+ if (VisitedBlocks.count(&B))
continue;
- // Otherwise set it as live and add any non-live atoms that it points to
- // to the worklist.
- NextLive.setLive(true);
-
- for (auto &E : NextLive.edges()) {
- if (!E.getTarget().isDefined())
- continue;
+ VisitedBlocks.insert(&B);
- auto &EDT = static_cast<DefinedAtom &>(E.getTarget());
-
- if (EDT.shouldDiscard())
- EdgesToUpdate[&EDT].push_back(&E);
- else if (E.isKeepAlive() && !EDT.isLive())
- Worklist.push_back(&EDT);
+ for (auto &E : Sym->getBlock().edges()) {
+ if (E.getTarget().isDefined() && !E.getTarget().isLive()) {
+ E.getTarget().setLive(true);
+ Worklist.push_back(&E.getTarget());
+ }
}
}
- // Collect atoms to remove, then remove them from the graph.
- std::vector<DefinedAtom *> AtomsToRemove;
- for (auto *DA : G.defined_atoms())
- if (DA->shouldDiscard() || !DA->isLive())
- AtomsToRemove.push_back(DA);
-
- LLVM_DEBUG(dbgs() << "Pruning atoms:\n");
- for (auto *DA : AtomsToRemove) {
- LLVM_DEBUG(dbgs() << " " << *DA << "... ");
-
- // Check whether we need to replace this atom with an external atom.
- //
- // We replace if all of the following hold:
- // (1) The atom is marked should-discard,
- // (2) it has live edges (i.e. edges from live atoms) pointing to it.
- //
- // Otherwise we simply delete the atom.
-
- G.removeDefinedAtom(*DA);
-
- auto EdgesToUpdateItr = EdgesToUpdate.find(DA);
- if (EdgesToUpdateItr != EdgesToUpdate.end()) {
- auto &ExternalReplacement = G.addExternalAtom(DA->getName());
- for (auto *EdgeToUpdate : EdgesToUpdateItr->second)
- EdgeToUpdate->setTarget(ExternalReplacement);
- LLVM_DEBUG(dbgs() << "replaced with " << ExternalReplacement << "\n");
- } else
- LLVM_DEBUG(dbgs() << "deleted\n");
+ // Collect all the symbols to remove, then remove them.
+ {
+ LLVM_DEBUG(dbgs() << "Dead-stripping symbols:\n");
+ std::vector<Symbol *> SymbolsToRemove;
+ for (auto *Sym : G.defined_symbols())
+ if (!Sym->isLive())
+ SymbolsToRemove.push_back(Sym);
+ for (auto *Sym : SymbolsToRemove) {
+ LLVM_DEBUG(dbgs() << " " << *Sym << "...\n");
+ G.removeDefinedSymbol(*Sym);
+ }
}
- // Finally, discard any absolute symbols that were marked should-discard.
+ // Delete any unused blocks.
{
- std::vector<Atom *> AbsoluteAtomsToRemove;
- for (auto *A : G.absolute_atoms())
- if (A->shouldDiscard() || A->isLive())
- AbsoluteAtomsToRemove.push_back(A);
- for (auto *A : AbsoluteAtomsToRemove)
- G.removeAbsoluteAtom(*A);
+ LLVM_DEBUG(dbgs() << "Dead-stripping blocks:\n");
+ std::vector<Block *> BlocksToRemove;
+ for (auto *B : G.blocks())
+ if (!VisitedBlocks.count(B))
+ BlocksToRemove.push_back(B);
+ for (auto *B : BlocksToRemove) {
+ LLVM_DEBUG(dbgs() << " " << *B << "...\n");
+ G.removeBlock(*B);
+ }
}
}
diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
index e6fd6e38f7a6..07dee6cee200 100644
--- a/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
+++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h
@@ -41,39 +41,32 @@ public:
protected:
struct SegmentLayout {
- using SectionAtomsList = std::vector<DefinedAtom *>;
- struct SectionLayout {
- SectionLayout(Section &S) : S(&S) {}
+ using BlocksList = std::vector<Block *>;
- Section *S;
- SectionAtomsList Atoms;
- };
-
- using SectionLayoutList = std::vector<SectionLayout>;
-
- SectionLayoutList ContentSections;
- SectionLayoutList ZeroFillSections;
+ BlocksList ContentBlocks;
+ BlocksList ZeroFillBlocks;
};
using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>;
// Phase 1:
- // 1.1: Build atom graph
+ // 1.1: Build link graph
// 1.2: Run pre-prune passes
// 1.2: Prune graph
// 1.3: Run post-prune passes
- // 1.4: Sort atoms into segments
+ // 1.4: Sort blocks into segments
// 1.5: Allocate segment memory
// 1.6: Identify externals and make an async call to resolve function
void linkPhase1(std::unique_ptr<JITLinkerBase> Self);
// Phase 2:
// 2.1: Apply resolution results
- // 2.2: Fix up atom contents
+ // 2.2: Fix up block contents
// 2.3: Call OnResolved callback
// 2.3: Make an async call to transfer and finalize memory.
void linkPhase2(std::unique_ptr<JITLinkerBase> Self,
- Expected<AsyncLookupResult> LookupResult);
+ Expected<AsyncLookupResult> LookupResult,
+ SegmentLayoutMap Layout);
// Phase 3:
// 3.1: Call OnFinalized callback, handing off allocation.
@@ -81,24 +74,37 @@ protected:
// Build a graph from the given object buffer.
// To be implemented by the client.
- virtual Expected<std::unique_ptr<AtomGraph>>
+ virtual Expected<std::unique_ptr<LinkGraph>>
buildGraph(MemoryBufferRef ObjBuffer) = 0;
- // For debug dumping of the atom graph.
+ // For debug dumping of the link graph.
virtual StringRef getEdgeKindName(Edge::Kind K) const = 0;
+ // Alight a JITTargetAddress to conform with block alignment requirements.
+ static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) {
+ uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment();
+ return Addr + Delta;
+ }
+
+ // Alight a pointer to conform with block alignment requirements.
+ static char *alignToBlock(char *P, Block &B) {
+ uint64_t PAddr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(P));
+ uint64_t Delta = (B.getAlignmentOffset() - PAddr) % B.getAlignment();
+ return P + Delta;
+ }
+
private:
// Run all passes in the given pass list, bailing out immediately if any pass
// returns an error.
- Error runPasses(AtomGraphPassList &Passes, AtomGraph &G);
+ Error runPasses(LinkGraphPassList &Passes);
- // Copy atom contents and apply relocations.
+ // Copy block contents and apply relocations.
// Implemented in JITLinker.
virtual Error
- copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout,
- JITLinkMemoryManager::Allocation &Alloc) const = 0;
+ copyAndFixUpBlocks(const SegmentLayoutMap &Layout,
+ JITLinkMemoryManager::Allocation &Alloc) const = 0;
- void layOutAtoms();
+ SegmentLayoutMap layOutBlocks();
Error allocateSegments(const SegmentLayoutMap &Layout);
DenseSet<StringRef> getExternalSymbolNames() const;
void applyLookupResult(AsyncLookupResult LR);
@@ -108,8 +114,7 @@ private:
std::unique_ptr<JITLinkContext> Ctx;
PassConfiguration Passes;
- std::unique_ptr<AtomGraph> G;
- SegmentLayoutMap Layout;
+ std::unique_ptr<LinkGraph> G;
std::unique_ptr<JITLinkMemoryManager::Allocation> Alloc;
};
@@ -121,7 +126,7 @@ public:
/// Link should be called with the constructor arguments for LinkerImpl, which
/// will be forwarded to the constructor.
template <typename... ArgTs> static void link(ArgTs &&... Args) {
- auto L = llvm::make_unique<LinkerImpl>(std::forward<ArgTs>(Args)...);
+ auto L = std::make_unique<LinkerImpl>(std::forward<ArgTs>(Args)...);
// Ownership of the linker is passed into the linker's doLink function to
// allow it to be passed on to async continuations.
@@ -140,17 +145,17 @@ private:
}
Error
- copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout,
- JITLinkMemoryManager::Allocation &Alloc) const override {
- LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n");
+ copyAndFixUpBlocks(const SegmentLayoutMap &Layout,
+ JITLinkMemoryManager::Allocation &Alloc) const override {
+ LLVM_DEBUG(dbgs() << "Copying and fixing up blocks:\n");
for (auto &KV : Layout) {
auto &Prot = KV.first;
auto &SegLayout = KV.second;
auto SegMem = Alloc.getWorkingMemory(
static_cast<sys::Memory::ProtectionFlags>(Prot));
- char *LastAtomEnd = SegMem.data();
- char *AtomDataPtr = LastAtomEnd;
+ char *LastBlockEnd = SegMem.data();
+ char *BlockDataPtr = LastBlockEnd;
LLVM_DEBUG({
dbgs() << " Processing segment "
@@ -160,93 +165,79 @@ private:
<< " ]\n Processing content sections:\n";
});
- for (auto &SI : SegLayout.ContentSections) {
- LLVM_DEBUG(dbgs() << " " << SI.S->getName() << ":\n");
+ for (auto *B : SegLayout.ContentBlocks) {
+ LLVM_DEBUG(dbgs() << " " << *B << ":\n");
+
+ // Pad to alignment/alignment-offset.
+ BlockDataPtr = alignToBlock(BlockDataPtr, *B);
- AtomDataPtr += alignmentAdjustment(AtomDataPtr, SI.S->getAlignment());
+ LLVM_DEBUG({
+ dbgs() << " Bumped block pointer to "
+ << (const void *)BlockDataPtr << " to meet block alignment "
+ << B->getAlignment() << " and alignment offset "
+ << B->getAlignmentOffset() << "\n";
+ });
+ // Zero pad up to alignment.
LLVM_DEBUG({
- dbgs() << " Bumped atom pointer to " << (const void *)AtomDataPtr
- << " to meet section alignment "
- << " of " << SI.S->getAlignment() << "\n";
+ if (LastBlockEnd != BlockDataPtr)
+ dbgs() << " Zero padding from " << (const void *)LastBlockEnd
+ << " to " << (const void *)BlockDataPtr << "\n";
});
- for (auto *DA : SI.Atoms) {
-
- // Align.
- AtomDataPtr += alignmentAdjustment(AtomDataPtr, DA->getAlignment());
- LLVM_DEBUG({
- dbgs() << " Bumped atom pointer to "
- << (const void *)AtomDataPtr << " to meet alignment of "
- << DA->getAlignment() << "\n";
- });
-
- // Zero pad up to alignment.
- LLVM_DEBUG({
- if (LastAtomEnd != AtomDataPtr)
- dbgs() << " Zero padding from " << (const void *)LastAtomEnd
- << " to " << (const void *)AtomDataPtr << "\n";
- });
- while (LastAtomEnd != AtomDataPtr)
- *LastAtomEnd++ = 0;
-
- // Copy initial atom content.
- LLVM_DEBUG({
- dbgs() << " Copying atom " << *DA << " content, "
- << DA->getContent().size() << " bytes, from "
- << (const void *)DA->getContent().data() << " to "
- << (const void *)AtomDataPtr << "\n";
- });
- memcpy(AtomDataPtr, DA->getContent().data(), DA->getContent().size());
-
- // Copy atom data and apply fixups.
- LLVM_DEBUG(dbgs() << " Applying fixups.\n");
- for (auto &E : DA->edges()) {
-
- // Skip non-relocation edges.
- if (!E.isRelocation())
- continue;
-
- // Dispatch to LinkerImpl for fixup.
- if (auto Err = impl().applyFixup(*DA, E, AtomDataPtr))
- return Err;
- }
-
- // Point the atom's content to the fixed up buffer.
- DA->setContent(StringRef(AtomDataPtr, DA->getContent().size()));
-
- // Update atom end pointer.
- LastAtomEnd = AtomDataPtr + DA->getContent().size();
- AtomDataPtr = LastAtomEnd;
+ while (LastBlockEnd != BlockDataPtr)
+ *LastBlockEnd++ = 0;
+
+ // Copy initial block content.
+ LLVM_DEBUG({
+ dbgs() << " Copying block " << *B << " content, "
+ << B->getContent().size() << " bytes, from "
+ << (const void *)B->getContent().data() << " to "
+ << (const void *)BlockDataPtr << "\n";
+ });
+ memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size());
+
+ // Copy Block data and apply fixups.
+ LLVM_DEBUG(dbgs() << " Applying fixups.\n");
+ for (auto &E : B->edges()) {
+
+ // Skip non-relocation edges.
+ if (!E.isRelocation())
+ continue;
+
+ // Dispatch to LinkerImpl for fixup.
+ if (auto Err = impl().applyFixup(*B, E, BlockDataPtr))
+ return Err;
}
+
+ // Point the block's content to the fixed up buffer.
+ B->setContent(StringRef(BlockDataPtr, B->getContent().size()));
+
+ // Update block end pointer.
+ LastBlockEnd = BlockDataPtr + B->getContent().size();
+ BlockDataPtr = LastBlockEnd;
}
// Zero pad the rest of the segment.
LLVM_DEBUG({
dbgs() << " Zero padding end of segment from "
- << (const void *)LastAtomEnd << " to "
+ << (const void *)LastBlockEnd << " to "
<< (const void *)((char *)SegMem.data() + SegMem.size()) << "\n";
});
- while (LastAtomEnd != SegMem.data() + SegMem.size())
- *LastAtomEnd++ = 0;
+ while (LastBlockEnd != SegMem.data() + SegMem.size())
+ *LastBlockEnd++ = 0;
}
return Error::success();
}
};
-/// Dead strips and replaces discarded definitions with external atoms.
+/// Removes dead symbols/blocks/addressables.
///
-/// Finds the set of nodes reachable from any node initially marked live
-/// (nodes marked should-discard are treated as not live, even if they are
-/// reachable). All nodes not marked as live at the end of this process,
-/// are deleted. Nodes that are live, but marked should-discard are replaced
-/// with external atoms and all edges to them are re-written.
-void prune(AtomGraph &G);
-
-Error addEHFrame(AtomGraph &G, Section &EHFrameSection,
- StringRef EHFrameContent, JITTargetAddress EHFrameAddress,
- Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind);
+/// Finds the set of symbols and addressables reachable from any symbol
+/// initially marked live. All symbols/addressables not marked live at the end
+/// of this process are removed.
+void prune(LinkGraph &G);
} // end namespace jitlink
} // end namespace llvm
diff --git a/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
index 267307cfde05..9e0d207e8bdb 100644
--- a/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
+++ b/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp
@@ -38,9 +38,21 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) {
OnFinalize(applyProtections());
}
Error deallocate() override {
- for (auto &KV : SegBlocks)
- if (auto EC = sys::Memory::releaseMappedMemory(KV.second))
- return errorCodeToError(EC);
+ if (SegBlocks.empty())
+ return Error::success();
+ void *SlabStart = SegBlocks.begin()->second.base();
+ char *SlabEnd = (char *)SlabStart;
+ for (auto &KV : SegBlocks) {
+ SlabStart = std::min(SlabStart, KV.second.base());
+ SlabEnd = std::max(SlabEnd, (char *)(KV.second.base()) +
+ KV.second.allocatedSize());
+ }
+ size_t SlabSize = SlabEnd - (char *)SlabStart;
+ assert((SlabSize % sys::Process::getPageSizeEstimate()) == 0 &&
+ "Slab size is not a multiple of page size");
+ sys::MemoryBlock Slab(SlabStart, SlabSize);
+ if (auto EC = sys::Memory::releaseMappedMemory(Slab))
+ return errorCodeToError(EC);
return Error::success();
}
@@ -61,37 +73,52 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) {
AllocationMap SegBlocks;
};
+ if (!isPowerOf2_64((uint64_t)sys::Process::getPageSizeEstimate()))
+ return make_error<StringError>("Page size is not a power of 2",
+ inconvertibleErrorCode());
+
AllocationMap Blocks;
const sys::Memory::ProtectionFlags ReadWrite =
static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
sys::Memory::MF_WRITE);
+ // Compute the total number of pages to allocate.
+ size_t TotalSize = 0;
for (auto &KV : Request) {
- auto &Seg = KV.second;
+ const auto &Seg = KV.second;
- if (Seg.getContentAlignment() > sys::Process::getPageSizeEstimate())
+ if (Seg.getAlignment() > sys::Process::getPageSizeEstimate())
return make_error<StringError>("Cannot request higher than page "
"alignment",
inconvertibleErrorCode());
- if (sys::Process::getPageSizeEstimate() % Seg.getContentAlignment() != 0)
- return make_error<StringError>("Page size is not a multiple of "
- "alignment",
- inconvertibleErrorCode());
+ TotalSize = alignTo(TotalSize, sys::Process::getPageSizeEstimate());
+ TotalSize += Seg.getContentSize();
+ TotalSize += Seg.getZeroFillSize();
+ }
+
+ // Allocate one slab to cover all the segments.
+ std::error_code EC;
+ auto SlabRemaining =
+ sys::Memory::allocateMappedMemory(TotalSize, nullptr, ReadWrite, EC);
+
+ if (EC)
+ return errorCodeToError(EC);
+
+ // Allocate segment memory from the slab.
+ for (auto &KV : Request) {
- uint64_t ZeroFillStart =
- alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment());
- uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize();
+ const auto &Seg = KV.second;
- std::error_code EC;
- auto SegMem =
- sys::Memory::allocateMappedMemory(SegmentSize, nullptr, ReadWrite, EC);
+ uint64_t SegmentSize = alignTo(Seg.getContentSize() + Seg.getZeroFillSize(),
+ sys::Process::getPageSizeEstimate());
- if (EC)
- return errorCodeToError(EC);
+ sys::MemoryBlock SegMem(SlabRemaining.base(), SegmentSize);
+ SlabRemaining = sys::MemoryBlock((char *)SlabRemaining.base() + SegmentSize,
+ SegmentSize);
// Zero out the zero-fill memory.
- memset(static_cast<char *>(SegMem.base()) + ZeroFillStart, 0,
+ memset(static_cast<char *>(SegMem.base()) + Seg.getContentSize(), 0,
Seg.getZeroFillSize());
// Record the block for this segment.
diff --git a/lib/ExecutionEngine/JITLink/MachO.cpp b/lib/ExecutionEngine/JITLink/MachO.cpp
index 15995b8ce98f..58bc0f56e155 100644
--- a/lib/ExecutionEngine/JITLink/MachO.cpp
+++ b/lib/ExecutionEngine/JITLink/MachO.cpp
@@ -14,6 +14,7 @@
#include "llvm/ExecutionEngine/JITLink/MachO.h"
#include "llvm/BinaryFormat/MachO.h"
+#include "llvm/ExecutionEngine/JITLink/MachO_arm64.h"
#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h"
#include "llvm/Support/Endian.h"
#include "llvm/Support/Format.h"
@@ -64,6 +65,8 @@ void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) {
});
switch (Header.cputype) {
+ case MachO::CPU_TYPE_ARM64:
+ return jitLink_MachO_arm64(std::move(Ctx));
case MachO::CPU_TYPE_X86_64:
return jitLink_MachO_x86_64(std::move(Ctx));
}
diff --git a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp
deleted file mode 100644
index 1501c7ad0bc5..000000000000
--- a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp
+++ /dev/null
@@ -1,411 +0,0 @@
-//=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Generic MachO AtomGraph buliding code.
-//
-//===----------------------------------------------------------------------===//
-
-#include "MachOAtomGraphBuilder.h"
-
-#define DEBUG_TYPE "jitlink"
-
-namespace llvm {
-namespace jitlink {
-
-MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {}
-
-Expected<std::unique_ptr<AtomGraph>> MachOAtomGraphBuilder::buildGraph() {
- if (auto Err = parseSections())
- return std::move(Err);
-
- if (auto Err = addAtoms())
- return std::move(Err);
-
- if (auto Err = addRelocations())
- return std::move(Err);
-
- return std::move(G);
-}
-
-MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj)
- : Obj(Obj),
- G(llvm::make_unique<AtomGraph>(Obj.getFileName(), getPointerSize(Obj),
- getEndianness(Obj))) {}
-
-void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName,
- CustomAtomizeFunction Atomizer) {
- assert(!CustomAtomizeFunctions.count(SectionName) &&
- "Custom atomizer for this section already exists");
- CustomAtomizeFunctions[SectionName] = std::move(Atomizer);
-}
-
-bool MachOAtomGraphBuilder::areLayoutLocked(const Atom &A, const Atom &B) {
- // If these atoms are the same then they're trivially "locked".
- if (&A == &B)
- return true;
-
- // If A and B are different, check whether either is undefined. (in which
- // case they are not locked).
- if (!A.isDefined() || !B.isDefined())
- return false;
-
- // A and B are different, but they're both defined atoms. We need to check
- // whether they're part of the same alt_entry chain.
- auto &DA = static_cast<const DefinedAtom &>(A);
- auto &DB = static_cast<const DefinedAtom &>(B);
-
- auto AStartItr = AltEntryStarts.find(&DA);
- if (AStartItr == AltEntryStarts.end()) // If A is not in a chain bail out.
- return false;
-
- auto BStartItr = AltEntryStarts.find(&DB);
- if (BStartItr == AltEntryStarts.end()) // If B is not in a chain bail out.
- return false;
-
- // A and B are layout locked if they're in the same chain.
- return AStartItr->second == BStartItr->second;
-}
-
-unsigned
-MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) {
- return Obj.is64Bit() ? 8 : 4;
-}
-
-support::endianness
-MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) {
- return Obj.isLittleEndian() ? support::little : support::big;
-}
-
-MachOAtomGraphBuilder::MachOSection &MachOAtomGraphBuilder::getCommonSection() {
- if (!CommonSymbolsSection) {
- auto Prot = static_cast<sys::Memory::ProtectionFlags>(
- sys::Memory::MF_READ | sys::Memory::MF_WRITE);
- auto &GenericSection = G->createSection("<common>", 1, Prot, true);
- CommonSymbolsSection = MachOSection(GenericSection);
- }
- return *CommonSymbolsSection;
-}
-
-Error MachOAtomGraphBuilder::parseSections() {
- for (auto &SecRef : Obj.sections()) {
- assert((SecRef.getAlignment() <= std::numeric_limits<uint32_t>::max()) &&
- "Section alignment does not fit in 32 bits");
-
- StringRef Name;
- if (auto EC = SecRef.getName(Name))
- return errorCodeToError(EC);
-
- unsigned SectionIndex = SecRef.getIndex() + 1;
-
- uint32_t Align = SecRef.getAlignment();
- if (!isPowerOf2_32(Align))
- return make_error<JITLinkError>("Section " + Name +
- " has non-power-of-2 "
- "alignment");
-
- // FIXME: Get real section permissions
- // How, exactly, on MachO?
- sys::Memory::ProtectionFlags Prot;
- if (SecRef.isText())
- Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
- sys::Memory::MF_EXEC);
- else
- Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
- sys::Memory::MF_WRITE);
-
- auto &GenericSection = G->createSection(Name, Align, Prot, SecRef.isBSS());
-
- LLVM_DEBUG({
- dbgs() << "Adding section " << Name << ": "
- << format("0x%016" PRIx64, SecRef.getAddress())
- << ", align: " << SecRef.getAlignment() << "\n";
- });
-
- assert(!Sections.count(SectionIndex) && "Section index already in use");
-
- auto &MachOSec =
- Sections
- .try_emplace(SectionIndex, GenericSection, SecRef.getAddress(),
- SecRef.getAlignment())
- .first->second;
-
- if (!SecRef.isVirtual()) {
- // If this section has content then record it.
- Expected<StringRef> Content = SecRef.getContents();
- if (!Content)
- return Content.takeError();
- if (Content->size() != SecRef.getSize())
- return make_error<JITLinkError>("Section content size does not match "
- "declared size for " +
- Name);
- MachOSec.setContent(*Content);
- } else {
- // If this is a zero-fill section then just record the size.
- MachOSec.setZeroFill(SecRef.getSize());
- }
-
- uint32_t SectionFlags =
- Obj.is64Bit() ? Obj.getSection64(SecRef.getRawDataRefImpl()).flags
- : Obj.getSection(SecRef.getRawDataRefImpl()).flags;
-
- MachOSec.setNoDeadStrip(SectionFlags & MachO::S_ATTR_NO_DEAD_STRIP);
- }
-
- return Error::success();
-}
-
-// Adds atoms with identified start addresses (but not lengths) for all named
-// atoms.
-// Also, for every section that contains named atoms, but does not have an
-// atom at offset zero of that section, constructs an anonymous atom covering
-// that range.
-Error MachOAtomGraphBuilder::addNonCustomAtoms() {
- using AddrToAtomMap = std::map<JITTargetAddress, DefinedAtom *>;
- DenseMap<MachOSection *, AddrToAtomMap> SecToAtoms;
-
- DenseMap<MachOSection *, unsigned> FirstOrdinal;
- std::vector<DefinedAtom *> AltEntryAtoms;
-
- DenseSet<StringRef> ProcessedSymbols; // Used to check for duplicate defs.
-
- for (auto SymI = Obj.symbol_begin(), SymE = Obj.symbol_end(); SymI != SymE;
- ++SymI) {
- object::SymbolRef Sym(SymI->getRawDataRefImpl(), &Obj);
-
- auto Name = Sym.getName();
- if (!Name)
- return Name.takeError();
-
- // Bail out on duplicate definitions: There should never be more than one
- // definition for a symbol in a given object file.
- if (ProcessedSymbols.count(*Name))
- return make_error<JITLinkError>("Duplicate definition within object: " +
- *Name);
- else
- ProcessedSymbols.insert(*Name);
-
- auto Addr = Sym.getAddress();
- if (!Addr)
- return Addr.takeError();
-
- auto SymType = Sym.getType();
- if (!SymType)
- return SymType.takeError();
-
- auto Flags = Sym.getFlags();
-
- if (Flags & object::SymbolRef::SF_Undefined) {
- LLVM_DEBUG(dbgs() << "Adding undef atom \"" << *Name << "\"\n");
- G->addExternalAtom(*Name);
- continue;
- } else if (Flags & object::SymbolRef::SF_Absolute) {
- LLVM_DEBUG(dbgs() << "Adding absolute \"" << *Name << "\" addr: "
- << format("0x%016" PRIx64, *Addr) << "\n");
- auto &A = G->addAbsoluteAtom(*Name, *Addr);
- A.setGlobal(Flags & object::SymbolRef::SF_Global);
- A.setExported(Flags & object::SymbolRef::SF_Exported);
- A.setWeak(Flags & object::SymbolRef::SF_Weak);
- continue;
- } else if (Flags & object::SymbolRef::SF_Common) {
- LLVM_DEBUG({
- dbgs() << "Adding common \"" << *Name
- << "\" addr: " << format("0x%016" PRIx64, *Addr) << "\n";
- });
- auto &A =
- G->addCommonAtom(getCommonSection().getGenericSection(), *Name, *Addr,
- std::max(Sym.getAlignment(), 1U),
- Obj.getCommonSymbolSize(Sym.getRawDataRefImpl()));
- A.setGlobal(Flags & object::SymbolRef::SF_Global);
- A.setExported(Flags & object::SymbolRef::SF_Exported);
- continue;
- }
-
- LLVM_DEBUG(dbgs() << "Adding defined atom \"" << *Name << "\"\n");
-
- // This atom is neither undefined nor absolute, so it must be defined in
- // this object. Get its section index.
- auto SecItr = Sym.getSection();
- if (!SecItr)
- return SecItr.takeError();
-
- uint64_t SectionIndex = (*SecItr)->getIndex() + 1;
-
- LLVM_DEBUG(dbgs() << " to section index " << SectionIndex << "\n");
-
- auto SecByIndexItr = Sections.find(SectionIndex);
- if (SecByIndexItr == Sections.end())
- return make_error<JITLinkError>("Unrecognized section index in macho");
-
- auto &Sec = SecByIndexItr->second;
-
- auto &DA = G->addDefinedAtom(Sec.getGenericSection(), *Name, *Addr,
- std::max(Sym.getAlignment(), 1U));
-
- DA.setGlobal(Flags & object::SymbolRef::SF_Global);
- DA.setExported(Flags & object::SymbolRef::SF_Exported);
- DA.setWeak(Flags & object::SymbolRef::SF_Weak);
-
- DA.setCallable(*SymType & object::SymbolRef::ST_Function);
-
- // Check NDesc flags.
- {
- uint16_t NDesc = 0;
- if (Obj.is64Bit())
- NDesc = Obj.getSymbol64TableEntry(SymI->getRawDataRefImpl()).n_desc;
- else
- NDesc = Obj.getSymbolTableEntry(SymI->getRawDataRefImpl()).n_desc;
-
- // Record atom for alt-entry post-processing (where the layout-next
- // constraints will be added).
- if (NDesc & MachO::N_ALT_ENTRY)
- AltEntryAtoms.push_back(&DA);
-
- // If this atom has a no-dead-strip attr attached then mark it live.
- if (NDesc & MachO::N_NO_DEAD_STRIP)
- DA.setLive(true);
- }
-
- LLVM_DEBUG({
- dbgs() << " Added " << *Name
- << " addr: " << format("0x%016" PRIx64, *Addr)
- << ", align: " << DA.getAlignment()
- << ", section: " << Sec.getGenericSection().getName() << "\n";
- });
-
- auto &SecAtoms = SecToAtoms[&Sec];
- SecAtoms[DA.getAddress() - Sec.getAddress()] = &DA;
- }
-
- // Add anonymous atoms.
- for (auto &KV : Sections) {
- auto &S = KV.second;
-
- // Skip empty sections.
- if (S.empty())
- continue;
-
- // Skip sections with custom handling.
- if (CustomAtomizeFunctions.count(S.getName()))
- continue;
-
- auto SAI = SecToAtoms.find(&S);
-
- // If S is not in the SecToAtoms map then it contained no named atom. Add
- // one anonymous atom to cover the whole section.
- if (SAI == SecToAtoms.end()) {
- SecToAtoms[&S][0] = &G->addAnonymousAtom(
- S.getGenericSection(), S.getAddress(), S.getAlignment());
- continue;
- }
-
- // Otherwise, check whether this section had an atom covering offset zero.
- // If not, add one.
- auto &SecAtoms = SAI->second;
- if (!SecAtoms.count(0))
- SecAtoms[0] = &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(),
- S.getAlignment());
- }
-
- LLVM_DEBUG(dbgs() << "MachOGraphBuilder setting atom content\n");
-
- // Set atom contents and any section-based flags.
- for (auto &KV : SecToAtoms) {
- auto &S = *KV.first;
- auto &SecAtoms = KV.second;
-
- // Iterate the atoms in reverse order and set up their contents.
- JITTargetAddress LastAtomAddr = S.getSize();
- for (auto I = SecAtoms.rbegin(), E = SecAtoms.rend(); I != E; ++I) {
- auto Offset = I->first;
- auto &A = *I->second;
- LLVM_DEBUG({
- dbgs() << " " << A << " to [ " << S.getAddress() + Offset << " .. "
- << S.getAddress() + LastAtomAddr << " ]\n";
- });
-
- if (S.isZeroFill())
- A.setZeroFill(LastAtomAddr - Offset);
- else
- A.setContent(S.getContent().substr(Offset, LastAtomAddr - Offset));
-
- // If the section has no-dead-strip set then mark the atom as live.
- if (S.isNoDeadStrip())
- A.setLive(true);
-
- LastAtomAddr = Offset;
- }
- }
-
- LLVM_DEBUG(dbgs() << "Adding alt-entry starts\n");
-
- // Sort alt-entry atoms by address in ascending order.
- llvm::sort(AltEntryAtoms.begin(), AltEntryAtoms.end(),
- [](const DefinedAtom *LHS, const DefinedAtom *RHS) {
- return LHS->getAddress() < RHS->getAddress();
- });
-
- // Process alt-entry atoms in address order to build the table of alt-entry
- // atoms to alt-entry chain starts.
- for (auto *DA : AltEntryAtoms) {
- assert(!AltEntryStarts.count(DA) && "Duplicate entry in AltEntryStarts");
-
- // DA is an alt-entry atom. Look for the predecessor atom that it is locked
- // to, bailing out if we do not find one.
- auto AltEntryPred = G->findAtomByAddress(DA->getAddress() - 1);
- if (!AltEntryPred)
- return AltEntryPred.takeError();
-
- // Add a LayoutNext edge from the predecessor to this atom.
- AltEntryPred->setLayoutNext(*DA);
-
- // Check to see whether the predecessor itself is an alt-entry atom.
- auto AltEntryStartItr = AltEntryStarts.find(&*AltEntryPred);
- if (AltEntryStartItr != AltEntryStarts.end()) {
- // If the predecessor was an alt-entry atom then re-use its value.
- LLVM_DEBUG({
- dbgs() << " " << *DA << " -> " << *AltEntryStartItr->second
- << " (based on existing entry for " << *AltEntryPred << ")\n";
- });
- AltEntryStarts[DA] = AltEntryStartItr->second;
- } else {
- // If the predecessor does not have an entry then add an entry for this
- // atom (i.e. the alt_entry atom) and a self-reference entry for the
- /// predecessory atom that is the start of this chain.
- LLVM_DEBUG({
- dbgs() << " " << *AltEntryPred << " -> " << *AltEntryPred << "\n"
- << " " << *DA << " -> " << *AltEntryPred << "\n";
- });
- AltEntryStarts[&*AltEntryPred] = &*AltEntryPred;
- AltEntryStarts[DA] = &*AltEntryPred;
- }
- }
-
- return Error::success();
-}
-
-Error MachOAtomGraphBuilder::addAtoms() {
- // Add all named atoms.
- if (auto Err = addNonCustomAtoms())
- return Err;
-
- // Process special sections.
- for (auto &KV : Sections) {
- auto &S = KV.second;
- auto HI = CustomAtomizeFunctions.find(S.getGenericSection().getName());
- if (HI != CustomAtomizeFunctions.end()) {
- auto &Atomize = HI->second;
- if (auto Err = Atomize(S))
- return Err;
- }
- }
-
- return Error::success();
-}
-
-} // end namespace jitlink
-} // end namespace llvm
diff --git a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h
deleted file mode 100644
index 72d441b24d06..000000000000
--- a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h
+++ /dev/null
@@ -1,138 +0,0 @@
-//===----- MachOAtomGraphBuilder.h - MachO AtomGraph builder ----*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-//
-// Generic MachO AtomGraph building code.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H
-#define LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H
-
-#include "llvm/ExecutionEngine/JITLink/JITLink.h"
-
-#include "JITLinkGeneric.h"
-
-#include "llvm/Object/MachO.h"
-
-namespace llvm {
-namespace jitlink {
-
-class MachOAtomGraphBuilder {
-public:
- virtual ~MachOAtomGraphBuilder();
- Expected<std::unique_ptr<AtomGraph>> buildGraph();
-
-protected:
- using OffsetToAtomMap = std::map<JITTargetAddress, DefinedAtom *>;
-
- class MachOSection {
- public:
- MachOSection() = default;
-
- /// Create a MachO section with the given address and alignment.
- MachOSection(Section &GenericSection, JITTargetAddress Address,
- unsigned Alignment)
- : Address(Address), GenericSection(&GenericSection),
- Alignment(Alignment) {}
-
- /// Create a section without address, content or size (used for common
- /// symbol sections).
- MachOSection(Section &GenericSection) : GenericSection(&GenericSection) {}
-
- Section &getGenericSection() const {
- assert(GenericSection && "Section is null");
- return *GenericSection;
- }
-
- StringRef getName() const {
- assert(GenericSection && "No generic section attached");
- return GenericSection->getName();
- }
-
- MachOSection &setContent(StringRef Content) {
- assert(!ContentPtr && !Size && "Content/zeroFill already set");
- ContentPtr = Content.data();
- Size = Content.size();
- return *this;
- }
-
- MachOSection &setZeroFill(uint64_t Size) {
- assert(!ContentPtr && !this->Size && "Content/zeroFill already set");
- this->Size = Size;
- return *this;
- }
-
- bool isZeroFill() const { return !ContentPtr; }
-
- bool empty() const { return getSize() == 0; }
-
- size_t getSize() const { return Size; }
-
- StringRef getContent() const {
- assert(ContentPtr && "getContent() called on zero-fill section");
- return {ContentPtr, static_cast<size_t>(Size)};
- }
-
- JITTargetAddress getAddress() const { return Address; }
-
- unsigned getAlignment() const { return Alignment; }
-
- MachOSection &setNoDeadStrip(bool NoDeadStrip) {
- this->NoDeadStrip = NoDeadStrip;
- return *this;
- }
-
- bool isNoDeadStrip() const { return NoDeadStrip; }
-
- private:
- JITTargetAddress Address = 0;
- Section *GenericSection = nullptr;
- const char *ContentPtr = nullptr;
- uint64_t Size = 0;
- unsigned Alignment = 0;
- bool NoDeadStrip = false;
- };
-
- using CustomAtomizeFunction = std::function<Error(MachOSection &S)>;
-
- MachOAtomGraphBuilder(const object::MachOObjectFile &Obj);
-
- AtomGraph &getGraph() const { return *G; }
-
- const object::MachOObjectFile &getObject() const { return Obj; }
-
- void addCustomAtomizer(StringRef SectionName, CustomAtomizeFunction Atomizer);
-
- virtual Error addRelocations() = 0;
-
- /// Returns true if Atom A and Atom B are at a fixed offset from one another
- /// (i.e. if they're part of the same alt-entry chain).
- bool areLayoutLocked(const Atom &A, const Atom &B);
-
-private:
- static unsigned getPointerSize(const object::MachOObjectFile &Obj);
- static support::endianness getEndianness(const object::MachOObjectFile &Obj);
-
- MachOSection &getCommonSection();
-
- Error parseSections();
- Error addNonCustomAtoms();
- Error addAtoms();
-
- const object::MachOObjectFile &Obj;
- std::unique_ptr<AtomGraph> G;
- DenseMap<const DefinedAtom *, const DefinedAtom *> AltEntryStarts;
- DenseMap<unsigned, MachOSection> Sections;
- StringMap<CustomAtomizeFunction> CustomAtomizeFunctions;
- Optional<MachOSection> CommonSymbolsSection;
-};
-
-} // end namespace jitlink
-} // end namespace llvm
-
-#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H
diff --git a/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
new file mode 100644
index 000000000000..7366f53ebf36
--- /dev/null
+++ b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp
@@ -0,0 +1,535 @@
+//=--------- MachOLinkGraphBuilder.cpp - MachO LinkGraph builder ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Generic MachO LinkGraph buliding code.
+//
+//===----------------------------------------------------------------------===//
+
+#include "MachOLinkGraphBuilder.h"
+
+#define DEBUG_TYPE "jitlink"
+
+static const char *CommonSectionName = "__common";
+
+namespace llvm {
+namespace jitlink {
+
+MachOLinkGraphBuilder::~MachOLinkGraphBuilder() {}
+
+Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() {
+
+ // Sanity check: we only operate on relocatable objects.
+ if (!Obj.isRelocatableObject())
+ return make_error<JITLinkError>("Object is not a relocatable MachO");
+
+ if (auto Err = createNormalizedSections())
+ return std::move(Err);
+
+ if (auto Err = createNormalizedSymbols())
+ return std::move(Err);
+
+ if (auto Err = graphifyRegularSymbols())
+ return std::move(Err);
+
+ if (auto Err = graphifySectionsWithCustomParsers())
+ return std::move(Err);
+
+ if (auto Err = addRelocations())
+ return std::move(Err);
+
+ return std::move(G);
+}
+
+MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj)
+ : Obj(Obj),
+ G(std::make_unique<LinkGraph>(Obj.getFileName(), getPointerSize(Obj),
+ getEndianness(Obj))) {}
+
+void MachOLinkGraphBuilder::addCustomSectionParser(
+ StringRef SectionName, SectionParserFunction Parser) {
+ assert(!CustomSectionParserFunctions.count(SectionName) &&
+ "Custom parser for this section already exists");
+ CustomSectionParserFunctions[SectionName] = std::move(Parser);
+}
+
+Linkage MachOLinkGraphBuilder::getLinkage(uint16_t Desc) {
+ if ((Desc & MachO::N_WEAK_DEF) || (Desc & MachO::N_WEAK_REF))
+ return Linkage::Weak;
+ return Linkage::Strong;
+}
+
+Scope MachOLinkGraphBuilder::getScope(StringRef Name, uint8_t Type) {
+ if (Name.startswith("l"))
+ return Scope::Local;
+ if (Type & MachO::N_PEXT)
+ return Scope::Hidden;
+ if (Type & MachO::N_EXT)
+ return Scope::Default;
+ return Scope::Local;
+}
+
+bool MachOLinkGraphBuilder::isAltEntry(const NormalizedSymbol &NSym) {
+ return NSym.Desc & MachO::N_ALT_ENTRY;
+}
+
+unsigned
+MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) {
+ return Obj.is64Bit() ? 8 : 4;
+}
+
+support::endianness
+MachOLinkGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) {
+ return Obj.isLittleEndian() ? support::little : support::big;
+}
+
+Section &MachOLinkGraphBuilder::getCommonSection() {
+ if (!CommonSection) {
+ auto Prot = static_cast<sys::Memory::ProtectionFlags>(
+ sys::Memory::MF_READ | sys::Memory::MF_WRITE);
+ CommonSection = &G->createSection(CommonSectionName, Prot);
+ }
+ return *CommonSection;
+}
+
+Error MachOLinkGraphBuilder::createNormalizedSections() {
+ // Build normalized sections. Verifies that section data is in-range (for
+ // sections with content) and that address ranges are non-overlapping.
+
+ LLVM_DEBUG(dbgs() << "Creating normalized sections...\n");
+
+ for (auto &SecRef : Obj.sections()) {
+ NormalizedSection NSec;
+ uint32_t DataOffset = 0;
+
+ auto SecIndex = Obj.getSectionIndex(SecRef.getRawDataRefImpl());
+
+ auto Name = SecRef.getName();
+ if (!Name)
+ return Name.takeError();
+
+ if (Obj.is64Bit()) {
+ const MachO::section_64 &Sec64 =
+ Obj.getSection64(SecRef.getRawDataRefImpl());
+
+ NSec.Address = Sec64.addr;
+ NSec.Size = Sec64.size;
+ NSec.Alignment = 1ULL << Sec64.align;
+ NSec.Flags = Sec64.flags;
+ DataOffset = Sec64.offset;
+ } else {
+ const MachO::section &Sec32 = Obj.getSection(SecRef.getRawDataRefImpl());
+ NSec.Address = Sec32.addr;
+ NSec.Size = Sec32.size;
+ NSec.Alignment = 1ULL << Sec32.align;
+ NSec.Flags = Sec32.flags;
+ DataOffset = Sec32.offset;
+ }
+
+ LLVM_DEBUG({
+ dbgs() << " " << *Name << ": " << formatv("{0:x16}", NSec.Address)
+ << " -- " << formatv("{0:x16}", NSec.Address + NSec.Size)
+ << ", align: " << NSec.Alignment << ", index: " << SecIndex
+ << "\n";
+ });
+
+ // Get the section data if any.
+ {
+ unsigned SectionType = NSec.Flags & MachO::SECTION_TYPE;
+ if (SectionType != MachO::S_ZEROFILL &&
+ SectionType != MachO::S_GB_ZEROFILL) {
+
+ if (DataOffset + NSec.Size > Obj.getData().size())
+ return make_error<JITLinkError>(
+ "Section data extends past end of file");
+
+ NSec.Data = Obj.getData().data() + DataOffset;
+ }
+ }
+
+ // Get prot flags.
+ // FIXME: Make sure this test is correct (it's probably missing cases
+ // as-is).
+ sys::Memory::ProtectionFlags Prot;
+ if (NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS)
+ Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_EXEC);
+ else
+ Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ |
+ sys::Memory::MF_WRITE);
+
+ NSec.GraphSection = &G->createSection(*Name, Prot);
+ IndexToSection.insert(std::make_pair(SecIndex, std::move(NSec)));
+ }
+
+ std::vector<NormalizedSection *> Sections;
+ Sections.reserve(IndexToSection.size());
+ for (auto &KV : IndexToSection)
+ Sections.push_back(&KV.second);
+
+ // If we didn't end up creating any sections then bail out. The code below
+ // assumes that we have at least one section.
+ if (Sections.empty())
+ return Error::success();
+
+ llvm::sort(Sections,
+ [](const NormalizedSection *LHS, const NormalizedSection *RHS) {
+ assert(LHS && RHS && "Null section?");
+ return LHS->Address < RHS->Address;
+ });
+
+ for (unsigned I = 0, E = Sections.size() - 1; I != E; ++I) {
+ auto &Cur = *Sections[I];
+ auto &Next = *Sections[I + 1];
+ if (Next.Address < Cur.Address + Cur.Size)
+ return make_error<JITLinkError>(
+ "Address range for section " + Cur.GraphSection->getName() +
+ formatv(" [ {0:x16} -- {1:x16} ] ", Cur.Address,
+ Cur.Address + Cur.Size) +
+ "overlaps " +
+ formatv(" [ {0:x16} -- {1:x16} ] ", Next.Address,
+ Next.Address + Next.Size));
+ }
+
+ return Error::success();
+}
+
+Error MachOLinkGraphBuilder::createNormalizedSymbols() {
+ LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n");
+
+ for (auto &SymRef : Obj.symbols()) {
+
+ unsigned SymbolIndex = Obj.getSymbolIndex(SymRef.getRawDataRefImpl());
+ uint64_t Value;
+ uint32_t NStrX;
+ uint8_t Type;
+ uint8_t Sect;
+ uint16_t Desc;
+
+ if (Obj.is64Bit()) {
+ const MachO::nlist_64 &NL64 =
+ Obj.getSymbol64TableEntry(SymRef.getRawDataRefImpl());
+ Value = NL64.n_value;
+ NStrX = NL64.n_strx;
+ Type = NL64.n_type;
+ Sect = NL64.n_sect;
+ Desc = NL64.n_desc;
+ } else {
+ const MachO::nlist &NL32 =
+ Obj.getSymbolTableEntry(SymRef.getRawDataRefImpl());
+ Value = NL32.n_value;
+ NStrX = NL32.n_strx;
+ Type = NL32.n_type;
+ Sect = NL32.n_sect;
+ Desc = NL32.n_desc;
+ }
+
+ // Skip stabs.
+ // FIXME: Are there other symbols we should be skipping?
+ if (Type & MachO::N_STAB)
+ continue;
+
+ Optional<StringRef> Name;
+ if (NStrX) {
+ if (auto NameOrErr = SymRef.getName())
+ Name = *NameOrErr;
+ else
+ return NameOrErr.takeError();
+ }
+
+ LLVM_DEBUG({
+ dbgs() << " ";
+ if (!Name)
+ dbgs() << "<anonymous symbol>";
+ else
+ dbgs() << *Name;
+ dbgs() << ": value = " << formatv("{0:x16}", Value)
+ << ", type = " << formatv("{0:x2}", Type)
+ << ", desc = " << formatv("{0:x4}", Desc) << ", sect = ";
+ if (Sect)
+ dbgs() << static_cast<unsigned>(Sect - 1);
+ else
+ dbgs() << "none";
+ dbgs() << "\n";
+ });
+
+ // If this symbol has a section, sanity check that the addresses line up.
+ NormalizedSection *NSec = nullptr;
+ if (Sect != 0) {
+ if (auto NSecOrErr = findSectionByIndex(Sect - 1))
+ NSec = &*NSecOrErr;
+ else
+ return NSecOrErr.takeError();
+
+ if (Value < NSec->Address || Value > NSec->Address + NSec->Size)
+ return make_error<JITLinkError>("Symbol address does not fall within "
+ "section");
+ }
+
+ IndexToSymbol[SymbolIndex] =
+ &createNormalizedSymbol(*Name, Value, Type, Sect, Desc,
+ getLinkage(Type), getScope(*Name, Type));
+ }
+
+ return Error::success();
+}
+
+void MachOLinkGraphBuilder::addSectionStartSymAndBlock(
+ Section &GraphSec, uint64_t Address, const char *Data, uint64_t Size,
+ uint32_t Alignment, bool IsLive) {
+ Block &B =
+ Data ? G->createContentBlock(GraphSec, StringRef(Data, Size), Address,
+ Alignment, 0)
+ : G->createZeroFillBlock(GraphSec, Size, Address, Alignment, 0);
+ auto &Sym = G->addAnonymousSymbol(B, 0, Size, false, IsLive);
+ assert(!AddrToCanonicalSymbol.count(Sym.getAddress()) &&
+ "Anonymous block start symbol clashes with existing symbol address");
+ AddrToCanonicalSymbol[Sym.getAddress()] = &Sym;
+}
+
+Error MachOLinkGraphBuilder::graphifyRegularSymbols() {
+
+ LLVM_DEBUG(dbgs() << "Creating graph symbols...\n");
+
+ /// We only have 256 section indexes: Use a vector rather than a map.
+ std::vector<std::vector<NormalizedSymbol *>> SecIndexToSymbols;
+ SecIndexToSymbols.resize(256);
+
+ // Create commons, externs, and absolutes, and partition all other symbols by
+ // section.
+ for (auto &KV : IndexToSymbol) {
+ auto &NSym = *KV.second;
+
+ switch (NSym.Type & MachO::N_TYPE) {
+ case MachO::N_UNDF:
+ if (NSym.Value) {
+ if (!NSym.Name)
+ return make_error<JITLinkError>("Anonymous common symbol at index " +
+ Twine(KV.first));
+ NSym.GraphSymbol = &G->addCommonSymbol(
+ *NSym.Name, NSym.S, getCommonSection(), NSym.Value, 0,
+ 1ull << MachO::GET_COMM_ALIGN(NSym.Desc),
+ NSym.Desc & MachO::N_NO_DEAD_STRIP);
+ } else {
+ if (!NSym.Name)
+ return make_error<JITLinkError>("Anonymous external symbol at "
+ "index " +
+ Twine(KV.first));
+ NSym.GraphSymbol = &G->addExternalSymbol(*NSym.Name, 0);
+ }
+ break;
+ case MachO::N_ABS:
+ if (!NSym.Name)
+ return make_error<JITLinkError>("Anonymous absolute symbol at index " +
+ Twine(KV.first));
+ NSym.GraphSymbol = &G->addAbsoluteSymbol(
+ *NSym.Name, NSym.Value, 0, Linkage::Strong, Scope::Default,
+ NSym.Desc & MachO::N_NO_DEAD_STRIP);
+ break;
+ case MachO::N_SECT:
+ SecIndexToSymbols[NSym.Sect - 1].push_back(&NSym);
+ break;
+ case MachO::N_PBUD:
+ return make_error<JITLinkError>(
+ "Unupported N_PBUD symbol " +
+ (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) +
+ " at index " + Twine(KV.first));
+ case MachO::N_INDR:
+ return make_error<JITLinkError>(
+ "Unupported N_INDR symbol " +
+ (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) +
+ " at index " + Twine(KV.first));
+ default:
+ return make_error<JITLinkError>(
+ "Unrecognized symbol type " + Twine(NSym.Type & MachO::N_TYPE) +
+ " for symbol " +
+ (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) +
+ " at index " + Twine(KV.first));
+ }
+ }
+
+ // Loop over sections performing regular graphification for those that
+ // don't have custom parsers.
+ for (auto &KV : IndexToSection) {
+ auto SecIndex = KV.first;
+ auto &NSec = KV.second;
+
+ // Skip sections with custom parsers.
+ if (CustomSectionParserFunctions.count(NSec.GraphSection->getName())) {
+ LLVM_DEBUG({
+ dbgs() << " Skipping section " << NSec.GraphSection->getName()
+ << " as it has a custom parser.\n";
+ });
+ continue;
+ } else
+ LLVM_DEBUG({
+ dbgs() << " Processing section " << NSec.GraphSection->getName()
+ << "...\n";
+ });
+
+ bool SectionIsNoDeadStrip = NSec.Flags & MachO::S_ATTR_NO_DEAD_STRIP;
+ bool SectionIsText = NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS;
+
+ auto &SecNSymStack = SecIndexToSymbols[SecIndex];
+
+ // If this section is non-empty but there are no symbols covering it then
+ // create one block and anonymous symbol to cover the entire section.
+ if (SecNSymStack.empty()) {
+ if (NSec.Size > 0) {
+ LLVM_DEBUG({
+ dbgs() << " Section non-empty, but contains no symbols. "
+ "Creating anonymous block to cover "
+ << formatv("{0:x16}", NSec.Address) << " -- "
+ << formatv("{0:x16}", NSec.Address + NSec.Size) << "\n";
+ });
+ addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data,
+ NSec.Size, NSec.Alignment,
+ SectionIsNoDeadStrip);
+ } else
+ LLVM_DEBUG({
+ dbgs() << " Section empty and contains no symbols. Skipping.\n";
+ });
+ continue;
+ }
+
+ // Sort the symbol stack in by address, alt-entry status, scope, and name.
+ // We sort in reverse order so that symbols will be visited in the right
+ // order when we pop off the stack below.
+ llvm::sort(SecNSymStack, [](const NormalizedSymbol *LHS,
+ const NormalizedSymbol *RHS) {
+ if (LHS->Value != RHS->Value)
+ return LHS->Value > RHS->Value;
+ if (isAltEntry(*LHS) != isAltEntry(*RHS))
+ return isAltEntry(*RHS);
+ if (LHS->S != RHS->S)
+ return static_cast<uint8_t>(LHS->S) < static_cast<uint8_t>(RHS->S);
+ return LHS->Name < RHS->Name;
+ });
+
+ // The first symbol in a section can not be an alt-entry symbol.
+ if (!SecNSymStack.empty() && isAltEntry(*SecNSymStack.back()))
+ return make_error<JITLinkError>(
+ "First symbol in " + NSec.GraphSection->getName() + " is alt-entry");
+
+ // If the section is non-empty but there is no symbol covering the start
+ // address then add an anonymous one.
+ if (SecNSymStack.back()->Value != NSec.Address) {
+ auto AnonBlockSize = SecNSymStack.back()->Value - NSec.Address;
+ LLVM_DEBUG({
+ dbgs() << " Section start not covered by symbol. "
+ << "Creating anonymous block to cover [ "
+ << formatv("{0:x16}", NSec.Address) << " -- "
+ << formatv("{0:x16}", NSec.Address + AnonBlockSize) << " ]\n";
+ });
+ addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data,
+ AnonBlockSize, NSec.Alignment,
+ SectionIsNoDeadStrip);
+ }
+
+ // Visit section symbols in order by popping off the reverse-sorted stack,
+ // building blocks for each alt-entry chain and creating symbols as we go.
+ while (!SecNSymStack.empty()) {
+ SmallVector<NormalizedSymbol *, 8> BlockSyms;
+
+ BlockSyms.push_back(SecNSymStack.back());
+ SecNSymStack.pop_back();
+ while (!SecNSymStack.empty() &&
+ (isAltEntry(*SecNSymStack.back()) ||
+ SecNSymStack.back()->Value == BlockSyms.back()->Value)) {
+ BlockSyms.push_back(SecNSymStack.back());
+ SecNSymStack.pop_back();
+ }
+
+ // BlockNSyms now contains the block symbols in reverse canonical order.
+ JITTargetAddress BlockStart = BlockSyms.front()->Value;
+ JITTargetAddress BlockEnd = SecNSymStack.empty()
+ ? NSec.Address + NSec.Size
+ : SecNSymStack.back()->Value;
+ JITTargetAddress BlockOffset = BlockStart - NSec.Address;
+ JITTargetAddress BlockSize = BlockEnd - BlockStart;
+
+ LLVM_DEBUG({
+ dbgs() << " Creating block for " << formatv("{0:x16}", BlockStart)
+ << " -- " << formatv("{0:x16}", BlockEnd) << ": "
+ << NSec.GraphSection->getName() << " + "
+ << formatv("{0:x16}", BlockOffset) << " with "
+ << BlockSyms.size() << " symbol(s)...\n";
+ });
+
+ Block &B =
+ NSec.Data
+ ? G->createContentBlock(
+ *NSec.GraphSection,
+ StringRef(NSec.Data + BlockOffset, BlockSize), BlockStart,
+ NSec.Alignment, BlockStart % NSec.Alignment)
+ : G->createZeroFillBlock(*NSec.GraphSection, BlockSize,
+ BlockStart, NSec.Alignment,
+ BlockStart % NSec.Alignment);
+
+ Optional<JITTargetAddress> LastCanonicalAddr;
+ JITTargetAddress SymEnd = BlockEnd;
+ while (!BlockSyms.empty()) {
+ auto &NSym = *BlockSyms.back();
+ BlockSyms.pop_back();
+
+ bool SymLive =
+ (NSym.Desc & MachO::N_NO_DEAD_STRIP) || SectionIsNoDeadStrip;
+
+ LLVM_DEBUG({
+ dbgs() << " " << formatv("{0:x16}", NSym.Value) << " -- "
+ << formatv("{0:x16}", SymEnd) << ": ";
+ if (!NSym.Name)
+ dbgs() << "<anonymous symbol>";
+ else
+ dbgs() << NSym.Name;
+ if (SymLive)
+ dbgs() << " [no-dead-strip]";
+ if (LastCanonicalAddr == NSym.Value)
+ dbgs() << " [non-canonical]";
+ dbgs() << "\n";
+ });
+
+ auto &Sym =
+ NSym.Name
+ ? G->addDefinedSymbol(B, NSym.Value - BlockStart, *NSym.Name,
+ SymEnd - NSym.Value, NSym.L, NSym.S,
+ SectionIsText, SymLive)
+ : G->addAnonymousSymbol(B, NSym.Value - BlockStart,
+ SymEnd - NSym.Value, SectionIsText,
+ SymLive);
+ NSym.GraphSymbol = &Sym;
+ if (LastCanonicalAddr != Sym.getAddress()) {
+ if (LastCanonicalAddr)
+ SymEnd = *LastCanonicalAddr;
+ LastCanonicalAddr = Sym.getAddress();
+ setCanonicalSymbol(Sym);
+ }
+ }
+ }
+ }
+
+ return Error::success();
+}
+
+Error MachOLinkGraphBuilder::graphifySectionsWithCustomParsers() {
+ // Graphify special sections.
+ for (auto &KV : IndexToSection) {
+ auto &NSec = KV.second;
+
+ auto HI = CustomSectionParserFunctions.find(NSec.GraphSection->getName());
+ if (HI != CustomSectionParserFunctions.end()) {
+ auto &Parse = HI->second;
+ if (auto Err = Parse(NSec))
+ return Err;
+ }
+ }
+
+ return Error::success();
+}
+
+} // end namespace jitlink
+} // end namespace llvm
diff --git a/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
new file mode 100644
index 000000000000..e1123cd11048
--- /dev/null
+++ b/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h
@@ -0,0 +1,269 @@
+//===----- MachOLinkGraphBuilder.h - MachO LinkGraph builder ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Generic MachO LinkGraph building code.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H
+#define LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H
+
+#include "llvm/ExecutionEngine/JITLink/JITLink.h"
+
+#include "EHFrameSupportImpl.h"
+#include "JITLinkGeneric.h"
+#include "llvm/Object/MachO.h"
+
+#include <list>
+
+namespace llvm {
+namespace jitlink {
+
+class MachOLinkGraphBuilder {
+public:
+ virtual ~MachOLinkGraphBuilder();
+ Expected<std::unique_ptr<LinkGraph>> buildGraph();
+
+protected:
+ class MachOEHFrameBinaryParser : public EHFrameBinaryParser {
+ public:
+ MachOEHFrameBinaryParser(MachOLinkGraphBuilder &Builder,
+ JITTargetAddress EHFrameAddress,
+ StringRef EHFrameContent, Section &EHFrameSection,
+ uint64_t CIEAlignment, uint64_t FDEAlignment,
+ Edge::Kind FDEToCIERelocKind,
+ Edge::Kind FDEToTargetRelocKind)
+ : EHFrameBinaryParser(EHFrameAddress, EHFrameContent,
+ Builder.getGraph().getPointerSize(),
+ Builder.getGraph().getEndianness()),
+ Builder(Builder), EHFrameSection(EHFrameSection),
+ CIEAlignment(CIEAlignment), FDEAlignment(FDEAlignment),
+ FDEToCIERelocKind(FDEToCIERelocKind),
+ FDEToTargetRelocKind(FDEToTargetRelocKind) {}
+
+ Symbol *getSymbolAtAddress(JITTargetAddress Address) override {
+ if (auto *Sym = Builder.getSymbolByAddress(Address))
+ if (Sym->getAddress() == Address)
+ return Sym;
+ return nullptr;
+ }
+
+ Symbol &createCIERecord(JITTargetAddress RecordAddr,
+ StringRef RecordContent) override {
+ auto &G = Builder.getGraph();
+ auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr,
+ CIEAlignment, 0);
+ auto &CIESymbol =
+ G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false);
+ Builder.setCanonicalSymbol(CIESymbol);
+ return CIESymbol;
+ }
+
+ Expected<Symbol &> createFDERecord(JITTargetAddress RecordAddr,
+ StringRef RecordContent, Symbol &CIE,
+ size_t CIEOffset, Symbol &Func,
+ size_t FuncOffset, Symbol *LSDA,
+ size_t LSDAOffset) override {
+ auto &G = Builder.getGraph();
+ auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr,
+ FDEAlignment, 0);
+
+ // Add edges to CIE, Func, and (conditionally) LSDA.
+ B.addEdge(FDEToCIERelocKind, CIEOffset, CIE, 0);
+ B.addEdge(FDEToTargetRelocKind, FuncOffset, Func, 0);
+
+ if (LSDA)
+ B.addEdge(FDEToTargetRelocKind, LSDAOffset, *LSDA, 0);
+
+ auto &FDESymbol =
+ G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false);
+
+ // Add a keep-alive relocation from the function to the FDE to ensure it
+ // is not dead stripped.
+ Func.getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0);
+
+ return FDESymbol;
+ }
+
+ private:
+ MachOLinkGraphBuilder &Builder;
+ Section &EHFrameSection;
+ uint64_t CIEAlignment;
+ uint64_t FDEAlignment;
+ Edge::Kind FDEToCIERelocKind;
+ Edge::Kind FDEToTargetRelocKind;
+ };
+
+ struct NormalizedSymbol {
+ friend class MachOLinkGraphBuilder;
+
+ private:
+ NormalizedSymbol(Optional<StringRef> Name, uint64_t Value, uint8_t Type,
+ uint8_t Sect, uint16_t Desc, Linkage L, Scope S)
+ : Name(Name), Value(Value), Type(Type), Sect(Sect), Desc(Desc), L(L),
+ S(S) {
+ assert((!Name || !Name->empty()) && "Name must be none or non-empty");
+ }
+
+ public:
+ NormalizedSymbol(const NormalizedSymbol &) = delete;
+ NormalizedSymbol &operator=(const NormalizedSymbol &) = delete;
+ NormalizedSymbol(NormalizedSymbol &&) = delete;
+ NormalizedSymbol &operator=(NormalizedSymbol &&) = delete;
+
+ Optional<StringRef> Name;
+ uint64_t Value = 0;
+ uint8_t Type = 0;
+ uint8_t Sect = 0;
+ uint16_t Desc = 0;
+ Linkage L = Linkage::Strong;
+ Scope S = Scope::Default;
+ Symbol *GraphSymbol = nullptr;
+ };
+
+ class NormalizedSection {
+ friend class MachOLinkGraphBuilder;
+
+ private:
+ NormalizedSection() = default;
+
+ public:
+ Section *GraphSection = nullptr;
+ uint64_t Address = 0;
+ uint64_t Size = 0;
+ uint64_t Alignment = 0;
+ uint32_t Flags = 0;
+ const char *Data = nullptr;
+ };
+
+ using SectionParserFunction = std::function<Error(NormalizedSection &S)>;
+
+ MachOLinkGraphBuilder(const object::MachOObjectFile &Obj);
+
+ LinkGraph &getGraph() const { return *G; }
+
+ const object::MachOObjectFile &getObject() const { return Obj; }
+
+ void addCustomSectionParser(StringRef SectionName,
+ SectionParserFunction Parse);
+
+ virtual Error addRelocations() = 0;
+
+ /// Create a symbol.
+ template <typename... ArgTs>
+ NormalizedSymbol &createNormalizedSymbol(ArgTs &&... Args) {
+ NormalizedSymbol *Sym = reinterpret_cast<NormalizedSymbol *>(
+ Allocator.Allocate<NormalizedSymbol>());
+ new (Sym) NormalizedSymbol(std::forward<ArgTs>(Args)...);
+ return *Sym;
+ }
+
+ /// Index is zero-based (MachO section indexes are usually one-based) and
+ /// assumed to be in-range. Client is responsible for checking.
+ NormalizedSection &getSectionByIndex(unsigned Index) {
+ auto I = IndexToSection.find(Index);
+ assert(I != IndexToSection.end() && "No section recorded at index");
+ return I->second;
+ }
+
+ /// Try to get the section at the given index. Will return an error if the
+ /// given index is out of range, or if no section has been added for the given
+ /// index.
+ Expected<NormalizedSection &> findSectionByIndex(unsigned Index) {
+ auto I = IndexToSection.find(Index);
+ if (I == IndexToSection.end())
+ return make_error<JITLinkError>("No section recorded for index " +
+ formatv("{0:u}", Index));
+ return I->second;
+ }
+
+ /// Try to get the symbol at the given index. Will return an error if the
+ /// given index is out of range, or if no symbol has been added for the given
+ /// index.
+ Expected<NormalizedSymbol &> findSymbolByIndex(uint64_t Index) {
+ if (Index >= IndexToSymbol.size())
+ return make_error<JITLinkError>("Symbol index out of range");
+ auto *Sym = IndexToSymbol[Index];
+ if (!Sym)
+ return make_error<JITLinkError>("No symbol at index " +
+ formatv("{0:u}", Index));
+ return *Sym;
+ }
+
+ /// Returns the symbol with the highest address not greater than the search
+ /// address, or null if no such symbol exists.
+ Symbol *getSymbolByAddress(JITTargetAddress Address) {
+ auto I = AddrToCanonicalSymbol.upper_bound(Address);
+ if (I == AddrToCanonicalSymbol.begin())
+ return nullptr;
+ return std::prev(I)->second;
+ }
+
+ /// Returns the symbol with the highest address not greater than the search
+ /// address, or an error if no such symbol exists.
+ Expected<Symbol &> findSymbolByAddress(JITTargetAddress Address) {
+ auto *Sym = getSymbolByAddress(Address);
+ if (Sym)
+ if (Address < Sym->getAddress() + Sym->getSize())
+ return *Sym;
+ return make_error<JITLinkError>("No symbol covering address " +
+ formatv("{0:x16}", Address));
+ }
+
+ static Linkage getLinkage(uint16_t Desc);
+ static Scope getScope(StringRef Name, uint8_t Type);
+ static bool isAltEntry(const NormalizedSymbol &NSym);
+
+private:
+ static unsigned getPointerSize(const object::MachOObjectFile &Obj);
+ static support::endianness getEndianness(const object::MachOObjectFile &Obj);
+
+ void setCanonicalSymbol(Symbol &Sym) {
+ auto *&CanonicalSymEntry = AddrToCanonicalSymbol[Sym.getAddress()];
+ // There should be no symbol at this address, or, if there is,
+ // it should be a zero-sized symbol from an empty section (which
+ // we can safely override).
+ assert((!CanonicalSymEntry || CanonicalSymEntry->getSize() == 0) &&
+ "Duplicate canonical symbol at address");
+ CanonicalSymEntry = &Sym;
+ }
+
+ Section &getCommonSection();
+ void addSectionStartSymAndBlock(Section &GraphSec, uint64_t Address,
+ const char *Data, uint64_t Size,
+ uint32_t Alignment, bool IsLive);
+
+ Error createNormalizedSections();
+ Error createNormalizedSymbols();
+
+ /// Create graph blocks and symbols for externals, absolutes, commons and
+ /// all defined symbols in sections without custom parsers.
+ Error graphifyRegularSymbols();
+
+ /// Create graph blocks and symbols for all sections.
+ Error graphifySectionsWithCustomParsers();
+
+ // Put the BumpPtrAllocator first so that we don't free any of the underlying
+ // memory until the Symbol/Addressable destructors have been run.
+ BumpPtrAllocator Allocator;
+
+ const object::MachOObjectFile &Obj;
+ std::unique_ptr<LinkGraph> G;
+
+ DenseMap<unsigned, NormalizedSection> IndexToSection;
+ Section *CommonSection = nullptr;
+
+ DenseMap<uint32_t, NormalizedSymbol *> IndexToSymbol;
+ std::map<JITTargetAddress, Symbol *> AddrToCanonicalSymbol;
+ StringMap<SectionParserFunction> CustomSectionParserFunctions;
+};
+
+} // end namespace jitlink
+} // end namespace llvm
+
+#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H
diff --git a/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
new file mode 100644
index 000000000000..945343bff89d
--- /dev/null
+++ b/lib/ExecutionEngine/JITLink/MachO_arm64.cpp
@@ -0,0 +1,736 @@
+//===---- MachO_arm64.cpp - JIT linker implementation for MachO/arm64 -----===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// MachO/arm64 jit-link implementation.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/JITLink/MachO_arm64.h"
+
+#include "BasicGOTAndStubsBuilder.h"
+#include "MachOLinkGraphBuilder.h"
+
+#define DEBUG_TYPE "jitlink"
+
+using namespace llvm;
+using namespace llvm::jitlink;
+using namespace llvm::jitlink::MachO_arm64_Edges;
+
+namespace {
+
+class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder {
+public:
+ MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj)
+ : MachOLinkGraphBuilder(Obj),
+ NumSymbols(Obj.getSymtabLoadCommand().nsyms) {
+ addCustomSectionParser(
+ "__eh_frame", [this](NormalizedSection &EHFrameSection) {
+ if (!EHFrameSection.Data)
+ return make_error<JITLinkError>(
+ "__eh_frame section is marked zero-fill");
+ return MachOEHFrameBinaryParser(
+ *this, EHFrameSection.Address,
+ StringRef(EHFrameSection.Data, EHFrameSection.Size),
+ *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64)
+ .addToGraph();
+ });
+ }
+
+private:
+ static Expected<MachOARM64RelocationKind>
+ getRelocationKind(const MachO::relocation_info &RI) {
+ switch (RI.r_type) {
+ case MachO::ARM64_RELOC_UNSIGNED:
+ if (!RI.r_pcrel) {
+ if (RI.r_length == 3)
+ return RI.r_extern ? Pointer64 : Pointer64Anon;
+ else if (RI.r_length == 2)
+ return Pointer32;
+ }
+ break;
+ case MachO::ARM64_RELOC_SUBTRACTOR:
+ // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3.
+ // Initially represent SUBTRACTOR relocations with 'Delta<W>'.
+ // They may be turned into NegDelta<W> by parsePairRelocation.
+ if (!RI.r_pcrel && RI.r_extern) {
+ if (RI.r_length == 2)
+ return Delta32;
+ else if (RI.r_length == 3)
+ return Delta64;
+ }
+ break;
+ case MachO::ARM64_RELOC_BRANCH26:
+ if (RI.r_pcrel && RI.r_extern && RI.r_length == 2)
+ return Branch26;
+ break;
+ case MachO::ARM64_RELOC_PAGE21:
+ if (RI.r_pcrel && RI.r_extern && RI.r_length == 2)
+ return Page21;
+ break;
+ case MachO::ARM64_RELOC_PAGEOFF12:
+ if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2)
+ return PageOffset12;
+ break;
+ case MachO::ARM64_RELOC_GOT_LOAD_PAGE21:
+ if (RI.r_pcrel && RI.r_extern && RI.r_length == 2)
+ return GOTPage21;
+ break;
+ case MachO::ARM64_RELOC_GOT_LOAD_PAGEOFF12:
+ if (!RI.r_pcrel && RI.r_extern && RI.r_length == 2)
+ return GOTPageOffset12;
+ break;
+ case MachO::ARM64_RELOC_POINTER_TO_GOT:
+ if (RI.r_pcrel && RI.r_extern && RI.r_length == 2)
+ return PointerToGOT;
+ break;
+ case MachO::ARM64_RELOC_ADDEND:
+ if (!RI.r_pcrel && !RI.r_extern && RI.r_length == 2)
+ return PairedAddend;
+ break;
+ }
+
+ return make_error<JITLinkError>(
+ "Unsupported arm64 relocation: address=" +
+ formatv("{0:x8}", RI.r_address) +
+ ", symbolnum=" + formatv("{0:x6}", RI.r_symbolnum) +
+ ", kind=" + formatv("{0:x1}", RI.r_type) +
+ ", pc_rel=" + (RI.r_pcrel ? "true" : "false") +
+ ", extern=" + (RI.r_extern ? "true" : "false") +
+ ", length=" + formatv("{0:d}", RI.r_length));
+ }
+
+ MachO::relocation_info
+ getRelocationInfo(const object::relocation_iterator RelItr) {
+ MachO::any_relocation_info ARI =
+ getObject().getRelocation(RelItr->getRawDataRefImpl());
+ MachO::relocation_info RI;
+ memcpy(&RI, &ARI, sizeof(MachO::relocation_info));
+ return RI;
+ }
+
+ using PairRelocInfo =
+ std::tuple<MachOARM64RelocationKind, Symbol *, uint64_t>;
+
+ // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success,
+ // returns the edge kind and addend to be used.
+ Expected<PairRelocInfo>
+ parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind,
+ const MachO::relocation_info &SubRI,
+ JITTargetAddress FixupAddress, const char *FixupContent,
+ object::relocation_iterator &UnsignedRelItr,
+ object::relocation_iterator &RelEnd) {
+ using namespace support;
+
+ assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) ||
+ (SubtractorKind == Delta64 && SubRI.r_length == 3)) &&
+ "Subtractor kind should match length");
+ assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern");
+ assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel");
+
+ if (UnsignedRelItr == RelEnd)
+ return make_error<JITLinkError>("arm64 SUBTRACTOR without paired "
+ "UNSIGNED relocation");
+
+ auto UnsignedRI = getRelocationInfo(UnsignedRelItr);
+
+ if (SubRI.r_address != UnsignedRI.r_address)
+ return make_error<JITLinkError>("arm64 SUBTRACTOR and paired UNSIGNED "
+ "point to different addresses");
+
+ if (SubRI.r_length != UnsignedRI.r_length)
+ return make_error<JITLinkError>("length of arm64 SUBTRACTOR and paired "
+ "UNSIGNED reloc must match");
+
+ Symbol *FromSymbol;
+ if (auto FromSymbolOrErr = findSymbolByIndex(SubRI.r_symbolnum))
+ FromSymbol = FromSymbolOrErr->GraphSymbol;
+ else
+ return FromSymbolOrErr.takeError();
+
+ // Read the current fixup value.
+ uint64_t FixupValue = 0;
+ if (SubRI.r_length == 3)
+ FixupValue = *(const little64_t *)FixupContent;
+ else
+ FixupValue = *(const little32_t *)FixupContent;
+
+ // Find 'ToSymbol' using symbol number or address, depending on whether the
+ // paired UNSIGNED relocation is extern.
+ Symbol *ToSymbol = nullptr;
+ if (UnsignedRI.r_extern) {
+ // Find target symbol by symbol index.
+ if (auto ToSymbolOrErr = findSymbolByIndex(UnsignedRI.r_symbolnum))
+ ToSymbol = ToSymbolOrErr->GraphSymbol;
+ else
+ return ToSymbolOrErr.takeError();
+ } else {
+ if (auto ToSymbolOrErr = findSymbolByAddress(FixupValue))
+ ToSymbol = &*ToSymbolOrErr;
+ else
+ return ToSymbolOrErr.takeError();
+ FixupValue -= ToSymbol->getAddress();
+ }
+
+ MachOARM64RelocationKind DeltaKind;
+ Symbol *TargetSymbol;
+ uint64_t Addend;
+ if (&BlockToFix == &FromSymbol->getAddressable()) {
+ TargetSymbol = ToSymbol;
+ DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32;
+ Addend = FixupValue + (FixupAddress - FromSymbol->getAddress());
+ // FIXME: handle extern 'from'.
+ } else if (&BlockToFix == &ToSymbol->getAddressable()) {
+ TargetSymbol = &*FromSymbol;
+ DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32;
+ Addend = FixupValue - (FixupAddress - ToSymbol->getAddress());
+ } else {
+ // BlockToFix was neither FromSymbol nor ToSymbol.
+ return make_error<JITLinkError>("SUBTRACTOR relocation must fix up "
+ "either 'A' or 'B' (or a symbol in one "
+ "of their alt-entry groups)");
+ }
+
+ return PairRelocInfo(DeltaKind, TargetSymbol, Addend);
+ }
+
+ Error addRelocations() override {
+ using namespace support;
+ auto &Obj = getObject();
+
+ for (auto &S : Obj.sections()) {
+
+ JITTargetAddress SectionAddress = S.getAddress();
+
+ for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end();
+ RelItr != RelEnd; ++RelItr) {
+
+ MachO::relocation_info RI = getRelocationInfo(RelItr);
+
+ // Sanity check the relocation kind.
+ auto Kind = getRelocationKind(RI);
+ if (!Kind)
+ return Kind.takeError();
+
+ // Find the address of the value to fix up.
+ JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address;
+
+ LLVM_DEBUG({
+ dbgs() << "Processing " << getMachOARM64RelocationKindName(*Kind)
+ << " relocation at " << format("0x%016" PRIx64, FixupAddress)
+ << "\n";
+ });
+
+ // Find the block that the fixup points to.
+ Block *BlockToFix = nullptr;
+ {
+ auto SymbolToFixOrErr = findSymbolByAddress(FixupAddress);
+ if (!SymbolToFixOrErr)
+ return SymbolToFixOrErr.takeError();
+ BlockToFix = &SymbolToFixOrErr->getBlock();
+ }
+
+ if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) >
+ BlockToFix->getAddress() + BlockToFix->getContent().size())
+ return make_error<JITLinkError>(
+ "Relocation content extends past end of fixup block");
+
+ // Get a pointer to the fixup content.
+ const char *FixupContent = BlockToFix->getContent().data() +
+ (FixupAddress - BlockToFix->getAddress());
+
+ // The target symbol and addend will be populated by the switch below.
+ Symbol *TargetSymbol = nullptr;
+ uint64_t Addend = 0;
+
+ if (*Kind == PairedAddend) {
+ // If this is an Addend relocation then process it and move to the
+ // paired reloc.
+
+ Addend = RI.r_symbolnum;
+
+ if (RelItr == RelEnd)
+ return make_error<JITLinkError>("Unpaired Addend reloc at " +
+ formatv("{0:x16}", FixupAddress));
+ ++RelItr;
+ RI = getRelocationInfo(RelItr);
+
+ Kind = getRelocationKind(RI);
+ if (!Kind)
+ return Kind.takeError();
+
+ if (*Kind != Branch26 && *Kind != Page21 && *Kind != PageOffset12)
+ return make_error<JITLinkError>(
+ "Invalid relocation pair: Addend + " +
+ getMachOARM64RelocationKindName(*Kind));
+ else
+ LLVM_DEBUG({
+ dbgs() << " pair is " << getMachOARM64RelocationKindName(*Kind)
+ << "`\n";
+ });
+
+ // Find the address of the value to fix up.
+ JITTargetAddress PairedFixupAddress =
+ SectionAddress + (uint32_t)RI.r_address;
+ if (PairedFixupAddress != FixupAddress)
+ return make_error<JITLinkError>("Paired relocation points at "
+ "different target");
+ }
+
+ switch (*Kind) {
+ case Branch26: {
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
+ else
+ return TargetSymbolOrErr.takeError();
+ uint32_t Instr = *(const ulittle32_t *)FixupContent;
+ if ((Instr & 0x7fffffff) != 0x14000000)
+ return make_error<JITLinkError>("BRANCH26 target is not a B or BL "
+ "instruction with a zero addend");
+ break;
+ }
+ case Pointer32:
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
+ else
+ return TargetSymbolOrErr.takeError();
+ Addend = *(const ulittle32_t *)FixupContent;
+ break;
+ case Pointer64:
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
+ else
+ return TargetSymbolOrErr.takeError();
+ Addend = *(const ulittle64_t *)FixupContent;
+ break;
+ case Pointer64Anon: {
+ JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent;
+ if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress))
+ TargetSymbol = &*TargetSymbolOrErr;
+ else
+ return TargetSymbolOrErr.takeError();
+ Addend = TargetAddress - TargetSymbol->getAddress();
+ break;
+ }
+ case Page21:
+ case GOTPage21: {
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
+ else
+ return TargetSymbolOrErr.takeError();
+ uint32_t Instr = *(const ulittle32_t *)FixupContent;
+ if ((Instr & 0xffffffe0) != 0x90000000)
+ return make_error<JITLinkError>("PAGE21/GOTPAGE21 target is not an "
+ "ADRP instruction with a zero "
+ "addend");
+ break;
+ }
+ case PageOffset12: {
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
+ else
+ return TargetSymbolOrErr.takeError();
+ break;
+ }
+ case GOTPageOffset12: {
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
+ else
+ return TargetSymbolOrErr.takeError();
+ uint32_t Instr = *(const ulittle32_t *)FixupContent;
+ if ((Instr & 0xfffffc00) != 0xf9400000)
+ return make_error<JITLinkError>("GOTPAGEOFF12 target is not an LDR "
+ "immediate instruction with a zero "
+ "addend");
+ break;
+ }
+ case PointerToGOT:
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
+ else
+ return TargetSymbolOrErr.takeError();
+ break;
+ case Delta32:
+ case Delta64: {
+ // We use Delta32/Delta64 to represent SUBTRACTOR relocations.
+ // parsePairRelocation handles the paired reloc, and returns the
+ // edge kind to be used (either Delta32/Delta64, or
+ // NegDelta32/NegDelta64, depending on the direction of the
+ // subtraction) along with the addend.
+ auto PairInfo =
+ parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress,
+ FixupContent, ++RelItr, RelEnd);
+ if (!PairInfo)
+ return PairInfo.takeError();
+ std::tie(*Kind, TargetSymbol, Addend) = *PairInfo;
+ assert(TargetSymbol && "No target symbol from parsePairRelocation?");
+ break;
+ }
+ default:
+ llvm_unreachable("Special relocation kind should not appear in "
+ "mach-o file");
+ }
+
+ LLVM_DEBUG({
+ Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol,
+ Addend);
+ printEdge(dbgs(), *BlockToFix, GE,
+ getMachOARM64RelocationKindName(*Kind));
+ dbgs() << "\n";
+ });
+ BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(),
+ *TargetSymbol, Addend);
+ }
+ }
+ return Error::success();
+ }
+
+ unsigned NumSymbols = 0;
+};
+
+class MachO_arm64_GOTAndStubsBuilder
+ : public BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder> {
+public:
+ MachO_arm64_GOTAndStubsBuilder(LinkGraph &G)
+ : BasicGOTAndStubsBuilder<MachO_arm64_GOTAndStubsBuilder>(G) {}
+
+ bool isGOTEdge(Edge &E) const {
+ return E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12 ||
+ E.getKind() == PointerToGOT;
+ }
+
+ Symbol &createGOTEntry(Symbol &Target) {
+ auto &GOTEntryBlock = G.createContentBlock(
+ getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0);
+ GOTEntryBlock.addEdge(Pointer64, 0, Target, 0);
+ return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
+ }
+
+ void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
+ if (E.getKind() == GOTPage21 || E.getKind() == GOTPageOffset12) {
+ // Update the target, but leave the edge addend as-is.
+ E.setTarget(GOTEntry);
+ } else if (E.getKind() == PointerToGOT) {
+ E.setTarget(GOTEntry);
+ E.setKind(Delta32);
+ } else
+ llvm_unreachable("Not a GOT edge?");
+ }
+
+ bool isExternalBranchEdge(Edge &E) {
+ return E.getKind() == Branch26 && !E.getTarget().isDefined();
+ }
+
+ Symbol &createStub(Symbol &Target) {
+ auto &StubContentBlock =
+ G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
+ // Re-use GOT entries for stub targets.
+ auto &GOTEntrySymbol = getGOTEntrySymbol(Target);
+ StubContentBlock.addEdge(LDRLiteral19, 0, GOTEntrySymbol, 0);
+ return G.addAnonymousSymbol(StubContentBlock, 0, 8, true, false);
+ }
+
+ void fixExternalBranchEdge(Edge &E, Symbol &Stub) {
+ assert(E.getKind() == Branch26 && "Not a Branch32 edge?");
+ assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?");
+ E.setTarget(Stub);
+ }
+
+private:
+ Section &getGOTSection() {
+ if (!GOTSection)
+ GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ);
+ return *GOTSection;
+ }
+
+ Section &getStubsSection() {
+ if (!StubsSection) {
+ auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
+ sys::Memory::MF_READ | sys::Memory::MF_EXEC);
+ StubsSection = &G.createSection("$__STUBS", StubsProt);
+ }
+ return *StubsSection;
+ }
+
+ StringRef getGOTEntryBlockContent() {
+ return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent),
+ sizeof(NullGOTEntryContent));
+ }
+
+ StringRef getStubBlockContent() {
+ return StringRef(reinterpret_cast<const char *>(StubContent),
+ sizeof(StubContent));
+ }
+
+ static const uint8_t NullGOTEntryContent[8];
+ static const uint8_t StubContent[8];
+ Section *GOTSection = nullptr;
+ Section *StubsSection = nullptr;
+};
+
+const uint8_t MachO_arm64_GOTAndStubsBuilder::NullGOTEntryContent[8] = {
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
+const uint8_t MachO_arm64_GOTAndStubsBuilder::StubContent[8] = {
+ 0x10, 0x00, 0x00, 0x58, // LDR x16, <literal>
+ 0x00, 0x02, 0x1f, 0xd6 // BR x16
+};
+
+} // namespace
+
+namespace llvm {
+namespace jitlink {
+
+class MachOJITLinker_arm64 : public JITLinker<MachOJITLinker_arm64> {
+ friend class JITLinker<MachOJITLinker_arm64>;
+
+public:
+ MachOJITLinker_arm64(std::unique_ptr<JITLinkContext> Ctx,
+ PassConfiguration PassConfig)
+ : JITLinker(std::move(Ctx), std::move(PassConfig)) {}
+
+private:
+ StringRef getEdgeKindName(Edge::Kind R) const override {
+ return getMachOARM64RelocationKindName(R);
+ }
+
+ Expected<std::unique_ptr<LinkGraph>>
+ buildGraph(MemoryBufferRef ObjBuffer) override {
+ auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer);
+ if (!MachOObj)
+ return MachOObj.takeError();
+ return MachOLinkGraphBuilder_arm64(**MachOObj).buildGraph();
+ }
+
+ static Error targetOutOfRangeError(const Block &B, const Edge &E) {
+ std::string ErrMsg;
+ {
+ raw_string_ostream ErrStream(ErrMsg);
+ ErrStream << "Relocation target out of range: ";
+ printEdge(ErrStream, B, E, getMachOARM64RelocationKindName(E.getKind()));
+ ErrStream << "\n";
+ }
+ return make_error<JITLinkError>(std::move(ErrMsg));
+ }
+
+ static unsigned getPageOffset12Shift(uint32_t Instr) {
+ constexpr uint32_t LDRLiteralMask = 0x3ffffc00;
+
+ // Check for a GPR LDR immediate with a zero embedded literal.
+ // If found, the top two bits contain the shift.
+ if ((Instr & LDRLiteralMask) == 0x39400000)
+ return Instr >> 30;
+
+ // Check for a Neon LDR immediate of size 64-bit or less with a zero
+ // embedded literal. If found, the top two bits contain the shift.
+ if ((Instr & LDRLiteralMask) == 0x3d400000)
+ return Instr >> 30;
+
+ // Check for a Neon LDR immediate of size 128-bit with a zero embedded
+ // literal.
+ constexpr uint32_t SizeBitsMask = 0xc0000000;
+ if ((Instr & (LDRLiteralMask | SizeBitsMask)) == 0x3dc00000)
+ return 4;
+
+ return 0;
+ }
+
+ Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const {
+ using namespace support;
+
+ char *FixupPtr = BlockWorkingMem + E.getOffset();
+ JITTargetAddress FixupAddress = B.getAddress() + E.getOffset();
+
+ switch (E.getKind()) {
+ case Branch26: {
+ assert((FixupAddress & 0x3) == 0 && "Branch-inst is not 32-bit aligned");
+
+ int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
+
+ if (static_cast<uint64_t>(Value) & 0x3)
+ return make_error<JITLinkError>("Branch26 target is not 32-bit "
+ "aligned");
+
+ if (Value < -(1 << 27) || Value > ((1 << 27) - 1))
+ return targetOutOfRangeError(B, E);
+
+ uint32_t RawInstr = *(little32_t *)FixupPtr;
+ assert((RawInstr & 0x7fffffff) == 0x14000000 &&
+ "RawInstr isn't a B or BR immediate instruction");
+ uint32_t Imm = (static_cast<uint32_t>(Value) & ((1 << 28) - 1)) >> 2;
+ uint32_t FixedInstr = RawInstr | Imm;
+ *(little32_t *)FixupPtr = FixedInstr;
+ break;
+ }
+ case Pointer32: {
+ uint64_t Value = E.getTarget().getAddress() + E.getAddend();
+ if (Value > std::numeric_limits<uint32_t>::max())
+ return targetOutOfRangeError(B, E);
+ *(ulittle32_t *)FixupPtr = Value;
+ break;
+ }
+ case Pointer64: {
+ uint64_t Value = E.getTarget().getAddress() + E.getAddend();
+ *(ulittle64_t *)FixupPtr = Value;
+ break;
+ }
+ case Page21:
+ case GOTPage21: {
+ assert(E.getAddend() == 0 && "PAGE21/GOTPAGE21 with non-zero addend");
+ uint64_t TargetPage =
+ E.getTarget().getAddress() & ~static_cast<uint64_t>(4096 - 1);
+ uint64_t PCPage = B.getAddress() & ~static_cast<uint64_t>(4096 - 1);
+
+ int64_t PageDelta = TargetPage - PCPage;
+ if (PageDelta < -(1 << 30) || PageDelta > ((1 << 30) - 1))
+ return targetOutOfRangeError(B, E);
+
+ uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+ assert((RawInstr & 0xffffffe0) == 0x90000000 &&
+ "RawInstr isn't an ADRP instruction");
+ uint32_t ImmLo = (static_cast<uint64_t>(PageDelta) >> 12) & 0x3;
+ uint32_t ImmHi = (static_cast<uint64_t>(PageDelta) >> 14) & 0x7ffff;
+ uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5);
+ *(ulittle32_t *)FixupPtr = FixedInstr;
+ break;
+ }
+ case PageOffset12: {
+ assert(E.getAddend() == 0 && "PAGEOFF12 with non-zero addend");
+ uint64_t TargetOffset = E.getTarget().getAddress() & 0xfff;
+
+ uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+ unsigned ImmShift = getPageOffset12Shift(RawInstr);
+
+ if (TargetOffset & ((1 << ImmShift) - 1))
+ return make_error<JITLinkError>("PAGEOFF12 target is not aligned");
+
+ uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;
+ uint32_t FixedInstr = RawInstr | EncodedImm;
+ *(ulittle32_t *)FixupPtr = FixedInstr;
+ break;
+ }
+ case GOTPageOffset12: {
+ assert(E.getAddend() == 0 && "GOTPAGEOF12 with non-zero addend");
+
+ uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+ assert((RawInstr & 0xfffffc00) == 0xf9400000 &&
+ "RawInstr isn't a 64-bit LDR immediate");
+
+ uint32_t TargetOffset = E.getTarget().getAddress() & 0xfff;
+ assert((TargetOffset & 0x7) == 0 && "GOT entry is not 8-byte aligned");
+ uint32_t EncodedImm = (TargetOffset >> 3) << 10;
+ uint32_t FixedInstr = RawInstr | EncodedImm;
+ *(ulittle32_t *)FixupPtr = FixedInstr;
+ break;
+ }
+ case LDRLiteral19: {
+ assert((FixupAddress & 0x3) == 0 && "LDR is not 32-bit aligned");
+ assert(E.getAddend() == 0 && "LDRLiteral19 with non-zero addend");
+ uint32_t RawInstr = *(ulittle32_t *)FixupPtr;
+ assert(RawInstr == 0x58000010 && "RawInstr isn't a 64-bit LDR literal");
+ int64_t Delta = E.getTarget().getAddress() - FixupAddress;
+ if (Delta & 0x3)
+ return make_error<JITLinkError>("LDR literal target is not 32-bit "
+ "aligned");
+ if (Delta < -(1 << 20) || Delta > ((1 << 20) - 1))
+ return targetOutOfRangeError(B, E);
+
+ uint32_t EncodedImm = (static_cast<uint32_t>(Delta) >> 2) << 5;
+ uint32_t FixedInstr = RawInstr | EncodedImm;
+ *(ulittle32_t *)FixupPtr = FixedInstr;
+ break;
+ }
+ case Delta32:
+ case Delta64:
+ case NegDelta32:
+ case NegDelta64: {
+ int64_t Value;
+ if (E.getKind() == Delta32 || E.getKind() == Delta64)
+ Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();
+ else
+ Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();
+
+ if (E.getKind() == Delta32 || E.getKind() == NegDelta32) {
+ if (Value < std::numeric_limits<int32_t>::min() ||
+ Value > std::numeric_limits<int32_t>::max())
+ return targetOutOfRangeError(B, E);
+ *(little32_t *)FixupPtr = Value;
+ } else
+ *(little64_t *)FixupPtr = Value;
+ break;
+ }
+ default:
+ llvm_unreachable("Unrecognized edge kind");
+ }
+
+ return Error::success();
+ }
+
+ uint64_t NullValue = 0;
+};
+
+void jitLink_MachO_arm64(std::unique_ptr<JITLinkContext> Ctx) {
+ PassConfiguration Config;
+ Triple TT("arm64-apple-ios");
+
+ if (Ctx->shouldAddDefaultTargetPasses(TT)) {
+ // Add a mark-live pass.
+ if (auto MarkLive = Ctx->getMarkLivePass(TT))
+ Config.PrePrunePasses.push_back(std::move(MarkLive));
+ else
+ Config.PrePrunePasses.push_back(markAllSymbolsLive);
+
+ // Add an in-place GOT/Stubs pass.
+ Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error {
+ MachO_arm64_GOTAndStubsBuilder(G).run();
+ return Error::success();
+ });
+ }
+
+ if (auto Err = Ctx->modifyPassConfig(TT, Config))
+ return Ctx->notifyFailed(std::move(Err));
+
+ // Construct a JITLinker and run the link function.
+ MachOJITLinker_arm64::link(std::move(Ctx), std::move(Config));
+}
+
+StringRef getMachOARM64RelocationKindName(Edge::Kind R) {
+ switch (R) {
+ case Branch26:
+ return "Branch26";
+ case Pointer64:
+ return "Pointer64";
+ case Pointer64Anon:
+ return "Pointer64Anon";
+ case Page21:
+ return "Page21";
+ case PageOffset12:
+ return "PageOffset12";
+ case GOTPage21:
+ return "GOTPage21";
+ case GOTPageOffset12:
+ return "GOTPageOffset12";
+ case PointerToGOT:
+ return "PointerToGOT";
+ case PairedAddend:
+ return "PairedAddend";
+ case LDRLiteral19:
+ return "LDRLiteral19";
+ case Delta32:
+ return "Delta32";
+ case Delta64:
+ return "Delta64";
+ case NegDelta32:
+ return "NegDelta32";
+ case NegDelta64:
+ return "NegDelta64";
+ default:
+ return getGenericEdgeKindName(static_cast<Edge::Kind>(R));
+ }
+}
+
+} // end namespace jitlink
+} // end namespace llvm
diff --git a/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
index 4010678c6d33..d83787ffd598 100644
--- a/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
+++ b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp
@@ -13,7 +13,7 @@
#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h"
#include "BasicGOTAndStubsBuilder.h"
-#include "MachOAtomGraphBuilder.h"
+#include "MachOLinkGraphBuilder.h"
#define DEBUG_TYPE "jitlink"
@@ -23,16 +23,21 @@ using namespace llvm::jitlink::MachO_x86_64_Edges;
namespace {
-class MachOAtomGraphBuilder_x86_64 : public MachOAtomGraphBuilder {
+class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder {
public:
- MachOAtomGraphBuilder_x86_64(const object::MachOObjectFile &Obj)
- : MachOAtomGraphBuilder(Obj),
- NumSymbols(Obj.getSymtabLoadCommand().nsyms) {
- addCustomAtomizer("__eh_frame", [this](MachOSection &EHFrameSection) {
- return addEHFrame(getGraph(), EHFrameSection.getGenericSection(),
- EHFrameSection.getContent(),
- EHFrameSection.getAddress(), NegDelta32, Delta64);
- });
+ MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj)
+ : MachOLinkGraphBuilder(Obj) {
+ addCustomSectionParser(
+ "__eh_frame", [this](NormalizedSection &EHFrameSection) {
+ if (!EHFrameSection.Data)
+ return make_error<JITLinkError>(
+ "__eh_frame section is marked zero-fill");
+ return MachOEHFrameBinaryParser(
+ *this, EHFrameSection.Address,
+ StringRef(EHFrameSection.Data, EHFrameSection.Size),
+ *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64)
+ .addToGraph();
+ });
}
private:
@@ -40,8 +45,12 @@ private:
getRelocationKind(const MachO::relocation_info &RI) {
switch (RI.r_type) {
case MachO::X86_64_RELOC_UNSIGNED:
- if (!RI.r_pcrel && RI.r_length == 3)
- return RI.r_extern ? Pointer64 : Pointer64Anon;
+ if (!RI.r_pcrel) {
+ if (RI.r_length == 3)
+ return RI.r_extern ? Pointer64 : Pointer64Anon;
+ else if (RI.r_extern && RI.r_length == 2)
+ return Pointer32;
+ }
break;
case MachO::X86_64_RELOC_SIGNED:
if (RI.r_pcrel && RI.r_length == 2)
@@ -94,21 +103,10 @@ private:
", symbolnum=" + formatv("{0:x6}", RI.r_symbolnum) +
", kind=" + formatv("{0:x1}", RI.r_type) +
", pc_rel=" + (RI.r_pcrel ? "true" : "false") +
- ", extern= " + (RI.r_extern ? "true" : "false") +
+ ", extern=" + (RI.r_extern ? "true" : "false") +
", length=" + formatv("{0:d}", RI.r_length));
}
- Expected<Atom &> findAtomBySymbolIndex(const MachO::relocation_info &RI) {
- auto &Obj = getObject();
- if (RI.r_symbolnum >= NumSymbols)
- return make_error<JITLinkError>("Symbol index out of range");
- auto SymI = Obj.getSymbolByIndex(RI.r_symbolnum);
- auto Name = SymI->getName();
- if (!Name)
- return Name.takeError();
- return getGraph().getAtomByName(*Name);
- }
-
MachO::relocation_info
getRelocationInfo(const object::relocation_iterator RelItr) {
MachO::any_relocation_info ARI =
@@ -118,12 +116,12 @@ private:
return RI;
}
- using PairRelocInfo = std::tuple<MachOX86RelocationKind, Atom *, uint64_t>;
+ using PairRelocInfo = std::tuple<MachOX86RelocationKind, Symbol *, uint64_t>;
// Parses paired SUBTRACTOR/UNSIGNED relocations and, on success,
// returns the edge kind and addend to be used.
Expected<PairRelocInfo>
- parsePairRelocation(DefinedAtom &AtomToFix, Edge::Kind SubtractorKind,
+ parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind,
const MachO::relocation_info &SubRI,
JITTargetAddress FixupAddress, const char *FixupContent,
object::relocation_iterator &UnsignedRelItr,
@@ -150,9 +148,11 @@ private:
return make_error<JITLinkError>("length of x86_64 SUBTRACTOR and paired "
"UNSIGNED reloc must match");
- auto FromAtom = findAtomBySymbolIndex(SubRI);
- if (!FromAtom)
- return FromAtom.takeError();
+ Symbol *FromSymbol;
+ if (auto FromSymbolOrErr = findSymbolByIndex(SubRI.r_symbolnum))
+ FromSymbol = FromSymbolOrErr->GraphSymbol;
+ else
+ return FromSymbolOrErr.takeError();
// Read the current fixup value.
uint64_t FixupValue = 0;
@@ -161,54 +161,60 @@ private:
else
FixupValue = *(const little32_t *)FixupContent;
- // Find 'ToAtom' using symbol number or address, depending on whether the
+ // Find 'ToSymbol' using symbol number or address, depending on whether the
// paired UNSIGNED relocation is extern.
- Atom *ToAtom = nullptr;
+ Symbol *ToSymbol = nullptr;
if (UnsignedRI.r_extern) {
- // Find target atom by symbol index.
- if (auto ToAtomOrErr = findAtomBySymbolIndex(UnsignedRI))
- ToAtom = &*ToAtomOrErr;
+ // Find target symbol by symbol index.
+ if (auto ToSymbolOrErr = findSymbolByIndex(UnsignedRI.r_symbolnum))
+ ToSymbol = ToSymbolOrErr->GraphSymbol;
else
- return ToAtomOrErr.takeError();
+ return ToSymbolOrErr.takeError();
} else {
- if (auto ToAtomOrErr = getGraph().findAtomByAddress(FixupValue))
- ToAtom = &*ToAtomOrErr;
+ if (auto ToSymbolOrErr = findSymbolByAddress(FixupValue))
+ ToSymbol = &*ToSymbolOrErr;
else
- return ToAtomOrErr.takeError();
- FixupValue -= ToAtom->getAddress();
+ return ToSymbolOrErr.takeError();
+ FixupValue -= ToSymbol->getAddress();
}
MachOX86RelocationKind DeltaKind;
- Atom *TargetAtom;
+ Symbol *TargetSymbol;
uint64_t Addend;
- if (areLayoutLocked(AtomToFix, *FromAtom)) {
- TargetAtom = ToAtom;
+ if (&BlockToFix == &FromSymbol->getAddressable()) {
+ TargetSymbol = ToSymbol;
DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32;
- Addend = FixupValue + (FixupAddress - FromAtom->getAddress());
+ Addend = FixupValue + (FixupAddress - FromSymbol->getAddress());
// FIXME: handle extern 'from'.
- } else if (areLayoutLocked(AtomToFix, *ToAtom)) {
- TargetAtom = &*FromAtom;
+ } else if (&BlockToFix == &ToSymbol->getAddressable()) {
+ TargetSymbol = FromSymbol;
DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32;
- Addend = FixupValue - (FixupAddress - ToAtom->getAddress());
+ Addend = FixupValue - (FixupAddress - ToSymbol->getAddress());
} else {
- // AtomToFix was neither FromAtom nor ToAtom.
+ // BlockToFix was neither FromSymbol nor ToSymbol.
return make_error<JITLinkError>("SUBTRACTOR relocation must fix up "
- "either 'A' or 'B' (or an atom in one "
- "of their alt-entry groups)");
+ "either 'A' or 'B' (or a symbol in one "
+ "of their alt-entry chains)");
}
- return PairRelocInfo(DeltaKind, TargetAtom, Addend);
+ return PairRelocInfo(DeltaKind, TargetSymbol, Addend);
}
Error addRelocations() override {
using namespace support;
- auto &G = getGraph();
auto &Obj = getObject();
for (auto &S : Obj.sections()) {
JITTargetAddress SectionAddress = S.getAddress();
+ if (S.isVirtual()) {
+ if (S.relocation_begin() != S.relocation_end())
+ return make_error<JITLinkError>("Virtual section contains "
+ "relocations");
+ continue;
+ }
+
for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end();
RelItr != RelEnd; ++RelItr) {
@@ -227,26 +233,26 @@ private:
<< format("0x%016" PRIx64, FixupAddress) << "\n";
});
- // Find the atom that the fixup points to.
- DefinedAtom *AtomToFix = nullptr;
+ // Find the block that the fixup points to.
+ Block *BlockToFix = nullptr;
{
- auto AtomToFixOrErr = G.findAtomByAddress(FixupAddress);
- if (!AtomToFixOrErr)
- return AtomToFixOrErr.takeError();
- AtomToFix = &*AtomToFixOrErr;
+ auto SymbolToFixOrErr = findSymbolByAddress(FixupAddress);
+ if (!SymbolToFixOrErr)
+ return SymbolToFixOrErr.takeError();
+ BlockToFix = &SymbolToFixOrErr->getBlock();
}
if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) >
- AtomToFix->getAddress() + AtomToFix->getContent().size())
+ BlockToFix->getAddress() + BlockToFix->getContent().size())
return make_error<JITLinkError>(
- "Relocation content extends past end of fixup atom");
+ "Relocation extends past end of fixup block");
// Get a pointer to the fixup content.
- const char *FixupContent = AtomToFix->getContent().data() +
- (FixupAddress - AtomToFix->getAddress());
+ const char *FixupContent = BlockToFix->getContent().data() +
+ (FixupAddress - BlockToFix->getAddress());
- // The target atom and addend will be populated by the switch below.
- Atom *TargetAtom = nullptr;
+ // The target symbol and addend will be populated by the switch below.
+ Symbol *TargetSymbol = nullptr;
uint64_t Addend = 0;
switch (*Kind) {
@@ -254,46 +260,53 @@ private:
case PCRel32:
case PCRel32GOTLoad:
case PCRel32GOT:
- if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI))
- TargetAtom = &*TargetAtomOrErr;
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
+ else
+ return TargetSymbolOrErr.takeError();
+ Addend = *(const ulittle32_t *)FixupContent;
+ break;
+ case Pointer32:
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
else
- return TargetAtomOrErr.takeError();
+ return TargetSymbolOrErr.takeError();
Addend = *(const ulittle32_t *)FixupContent;
break;
case Pointer64:
- if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI))
- TargetAtom = &*TargetAtomOrErr;
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
else
- return TargetAtomOrErr.takeError();
+ return TargetSymbolOrErr.takeError();
Addend = *(const ulittle64_t *)FixupContent;
break;
case Pointer64Anon: {
JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent;
- if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress))
- TargetAtom = &*TargetAtomOrErr;
+ if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress))
+ TargetSymbol = &*TargetSymbolOrErr;
else
- return TargetAtomOrErr.takeError();
- Addend = TargetAddress - TargetAtom->getAddress();
+ return TargetSymbolOrErr.takeError();
+ Addend = TargetAddress - TargetSymbol->getAddress();
break;
}
case PCRel32Minus1:
case PCRel32Minus2:
case PCRel32Minus4:
- if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI))
- TargetAtom = &*TargetAtomOrErr;
+ if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum))
+ TargetSymbol = TargetSymbolOrErr->GraphSymbol;
else
- return TargetAtomOrErr.takeError();
+ return TargetSymbolOrErr.takeError();
Addend = *(const ulittle32_t *)FixupContent +
(1 << (*Kind - PCRel32Minus1));
break;
case PCRel32Anon: {
JITTargetAddress TargetAddress =
FixupAddress + 4 + *(const ulittle32_t *)FixupContent;
- if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress))
- TargetAtom = &*TargetAtomOrErr;
+ if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress))
+ TargetSymbol = &*TargetSymbolOrErr;
else
- return TargetAtomOrErr.takeError();
- Addend = TargetAddress - TargetAtom->getAddress();
+ return TargetSymbolOrErr.takeError();
+ Addend = TargetAddress - TargetSymbol->getAddress();
break;
}
case PCRel32Minus1Anon:
@@ -303,11 +316,11 @@ private:
static_cast<JITTargetAddress>(1ULL << (*Kind - PCRel32Minus1Anon));
JITTargetAddress TargetAddress =
FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent;
- if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress))
- TargetAtom = &*TargetAtomOrErr;
+ if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress))
+ TargetSymbol = &*TargetSymbolOrErr;
else
- return TargetAtomOrErr.takeError();
- Addend = TargetAddress - TargetAtom->getAddress();
+ return TargetSymbolOrErr.takeError();
+ Addend = TargetAddress - TargetSymbol->getAddress();
break;
}
case Delta32:
@@ -318,12 +331,12 @@ private:
// NegDelta32/NegDelta64, depending on the direction of the
// subtraction) along with the addend.
auto PairInfo =
- parsePairRelocation(*AtomToFix, *Kind, RI, FixupAddress,
+ parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress,
FixupContent, ++RelItr, RelEnd);
if (!PairInfo)
return PairInfo.takeError();
- std::tie(*Kind, TargetAtom, Addend) = *PairInfo;
- assert(TargetAtom && "No target atom from parsePairRelocation?");
+ std::tie(*Kind, TargetSymbol, Addend) = *PairInfo;
+ assert(TargetSymbol && "No target symbol from parsePairRelocation?");
break;
}
default:
@@ -332,41 +345,38 @@ private:
}
LLVM_DEBUG({
- Edge GE(*Kind, FixupAddress - AtomToFix->getAddress(), *TargetAtom,
+ Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol,
Addend);
- printEdge(dbgs(), *AtomToFix, GE,
+ printEdge(dbgs(), *BlockToFix, GE,
getMachOX86RelocationKindName(*Kind));
dbgs() << "\n";
});
- AtomToFix->addEdge(*Kind, FixupAddress - AtomToFix->getAddress(),
- *TargetAtom, Addend);
+ BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(),
+ *TargetSymbol, Addend);
}
}
return Error::success();
}
-
- unsigned NumSymbols = 0;
};
class MachO_x86_64_GOTAndStubsBuilder
: public BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder> {
public:
- MachO_x86_64_GOTAndStubsBuilder(AtomGraph &G)
+ MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G)
: BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder>(G) {}
bool isGOTEdge(Edge &E) const {
return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad;
}
- DefinedAtom &createGOTEntry(Atom &Target) {
- auto &GOTEntryAtom = G.addAnonymousAtom(getGOTSection(), 0x0, 8);
- GOTEntryAtom.setContent(
- StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), 8));
- GOTEntryAtom.addEdge(Pointer64, 0, Target, 0);
- return GOTEntryAtom;
+ Symbol &createGOTEntry(Symbol &Target) {
+ auto &GOTEntryBlock = G.createContentBlock(
+ getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0);
+ GOTEntryBlock.addEdge(Pointer64, 0, Target, 0);
+ return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false);
}
- void fixGOTEdge(Edge &E, Atom &GOTEntry) {
+ void fixGOTEdge(Edge &E, Symbol &GOTEntry) {
assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) &&
"Not a GOT edge?");
E.setKind(PCRel32);
@@ -378,19 +388,16 @@ public:
return E.getKind() == Branch32 && !E.getTarget().isDefined();
}
- DefinedAtom &createStub(Atom &Target) {
- auto &StubAtom = G.addAnonymousAtom(getStubsSection(), 0x0, 2);
- StubAtom.setContent(
- StringRef(reinterpret_cast<const char *>(StubContent), 6));
-
+ Symbol &createStub(Symbol &Target) {
+ auto &StubContentBlock =
+ G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0);
// Re-use GOT entries for stub targets.
- auto &GOTEntryAtom = getGOTEntryAtom(Target);
- StubAtom.addEdge(PCRel32, 2, GOTEntryAtom, 0);
-
- return StubAtom;
+ auto &GOTEntrySymbol = getGOTEntrySymbol(Target);
+ StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, 0);
+ return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false);
}
- void fixExternalBranchEdge(Edge &E, Atom &Stub) {
+ void fixExternalBranchEdge(Edge &E, Symbol &Stub) {
assert(E.getKind() == Branch32 && "Not a Branch32 edge?");
assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?");
E.setTarget(Stub);
@@ -399,7 +406,7 @@ public:
private:
Section &getGOTSection() {
if (!GOTSection)
- GOTSection = &G.createSection("$__GOT", 8, sys::Memory::MF_READ, false);
+ GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ);
return *GOTSection;
}
@@ -407,11 +414,21 @@ private:
if (!StubsSection) {
auto StubsProt = static_cast<sys::Memory::ProtectionFlags>(
sys::Memory::MF_READ | sys::Memory::MF_EXEC);
- StubsSection = &G.createSection("$__STUBS", 8, StubsProt, false);
+ StubsSection = &G.createSection("$__STUBS", StubsProt);
}
return *StubsSection;
}
+ StringRef getGOTEntryBlockContent() {
+ return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent),
+ sizeof(NullGOTEntryContent));
+ }
+
+ StringRef getStubBlockContent() {
+ return StringRef(reinterpret_cast<const char *>(StubContent),
+ sizeof(StubContent));
+ }
+
static const uint8_t NullGOTEntryContent[8];
static const uint8_t StubContent[6];
Section *GOTSection = nullptr;
@@ -440,30 +457,31 @@ private:
return getMachOX86RelocationKindName(R);
}
- Expected<std::unique_ptr<AtomGraph>>
+ Expected<std::unique_ptr<LinkGraph>>
buildGraph(MemoryBufferRef ObjBuffer) override {
auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer);
if (!MachOObj)
return MachOObj.takeError();
- return MachOAtomGraphBuilder_x86_64(**MachOObj).buildGraph();
+ return MachOLinkGraphBuilder_x86_64(**MachOObj).buildGraph();
}
- static Error targetOutOfRangeError(const Atom &A, const Edge &E) {
+ static Error targetOutOfRangeError(const Block &B, const Edge &E) {
std::string ErrMsg;
{
raw_string_ostream ErrStream(ErrMsg);
ErrStream << "Relocation target out of range: ";
- printEdge(ErrStream, A, E, getMachOX86RelocationKindName(E.getKind()));
+ printEdge(ErrStream, B, E, getMachOX86RelocationKindName(E.getKind()));
ErrStream << "\n";
}
return make_error<JITLinkError>(std::move(ErrMsg));
}
- Error applyFixup(DefinedAtom &A, const Edge &E, char *AtomWorkingMem) const {
+ Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const {
+
using namespace support;
- char *FixupPtr = AtomWorkingMem + E.getOffset();
- JITTargetAddress FixupAddress = A.getAddress() + E.getOffset();
+ char *FixupPtr = BlockWorkingMem + E.getOffset();
+ JITTargetAddress FixupAddress = B.getAddress() + E.getOffset();
switch (E.getKind()) {
case Branch32:
@@ -473,7 +491,7 @@ private:
E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend();
if (Value < std::numeric_limits<int32_t>::min() ||
Value > std::numeric_limits<int32_t>::max())
- return targetOutOfRangeError(A, E);
+ return targetOutOfRangeError(B, E);
*(little32_t *)FixupPtr = Value;
break;
}
@@ -491,7 +509,7 @@ private:
E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend();
if (Value < std::numeric_limits<int32_t>::min() ||
Value > std::numeric_limits<int32_t>::max())
- return targetOutOfRangeError(A, E);
+ return targetOutOfRangeError(B, E);
*(little32_t *)FixupPtr = Value;
break;
}
@@ -503,7 +521,7 @@ private:
E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend();
if (Value < std::numeric_limits<int32_t>::min() ||
Value > std::numeric_limits<int32_t>::max())
- return targetOutOfRangeError(A, E);
+ return targetOutOfRangeError(B, E);
*(little32_t *)FixupPtr = Value;
break;
}
@@ -520,12 +538,19 @@ private:
if (E.getKind() == Delta32 || E.getKind() == NegDelta32) {
if (Value < std::numeric_limits<int32_t>::min() ||
Value > std::numeric_limits<int32_t>::max())
- return targetOutOfRangeError(A, E);
+ return targetOutOfRangeError(B, E);
*(little32_t *)FixupPtr = Value;
} else
*(little64_t *)FixupPtr = Value;
break;
}
+ case Pointer32: {
+ uint64_t Value = E.getTarget().getAddress() + E.getAddend();
+ if (Value > std::numeric_limits<uint32_t>::max())
+ return targetOutOfRangeError(B, E);
+ *(ulittle32_t *)FixupPtr = Value;
+ break;
+ }
default:
llvm_unreachable("Unrecognized edge kind");
}
@@ -545,10 +570,10 @@ void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) {
if (auto MarkLive = Ctx->getMarkLivePass(TT))
Config.PrePrunePasses.push_back(std::move(MarkLive));
else
- Config.PrePrunePasses.push_back(markAllAtomsLive);
+ Config.PrePrunePasses.push_back(markAllSymbolsLive);
// Add an in-place GOT/Stubs pass.
- Config.PostPrunePasses.push_back([](AtomGraph &G) -> Error {
+ Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error {
MachO_x86_64_GOTAndStubsBuilder(G).run();
return Error::success();
});
@@ -565,6 +590,8 @@ StringRef getMachOX86RelocationKindName(Edge::Kind R) {
switch (R) {
case Branch32:
return "Branch32";
+ case Pointer32:
+ return "Pointer32";
case Pointer64:
return "Pointer64";
case Pointer64Anon:
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-02 01:39:28 +0000