diff options
| author | Dimitry Andric <dim@FreeBSD.org> | 2023-12-09 13:28:42 +0000 |
|---|---|---|
| committer | Dimitry Andric <dim@FreeBSD.org> | 2023-12-09 13:28:42 +0000 |
| commit | b1c73532ee8997fe5dfbeb7d223027bdf99758a0 (patch) | |
| tree | 7d6e51c294ab6719475d660217aa0c0ad0526292 /llvm/lib/ExecutionEngine | |
| parent | 7fa27ce4a07f19b07799a767fc29416f3b625afb (diff) | |
Diffstat (limited to 'llvm/lib/ExecutionEngine')
65 files changed, 3036 insertions, 1117 deletions
diff --git a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp index 768d84501337..2559ed6a31a6 100644 --- a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp @@ -340,7 +340,7 @@ void *ArgvArray::reset(LLVMContext &C, ExecutionEngine *EE, Array = std::make_unique<char[]>((InputArgv.size()+1)*PtrSize); LLVM_DEBUG(dbgs() << "JIT: ARGV = " << (void *)Array.get() << "\n"); - Type *SBytePtr = Type::getInt8PtrTy(C); + Type *SBytePtr = PointerType::getUnqual(C); for (unsigned i = 0; i != InputArgv.size(); ++i) { unsigned Size = InputArgv[i].size()+1; @@ -430,7 +430,7 @@ int ExecutionEngine::runFunctionAsMain(Function *Fn, // Check main() type unsigned NumArgs = Fn->getFunctionType()->getNumParams(); FunctionType *FTy = Fn->getFunctionType(); - Type* PPInt8Ty = Type::getInt8PtrTy(Fn->getContext())->getPointerTo(); + Type *PPInt8Ty = PointerType::get(Fn->getContext(), 0); // Check the argument types. if (NumArgs > 3) @@ -471,7 +471,7 @@ EngineBuilder::EngineBuilder() : EngineBuilder(nullptr) {} EngineBuilder::EngineBuilder(std::unique_ptr<Module> M) : M(std::move(M)), WhichEngine(EngineKind::Either), ErrorStr(nullptr), - OptLevel(CodeGenOpt::Default), MemMgr(nullptr), Resolver(nullptr) { + OptLevel(CodeGenOptLevel::Default), MemMgr(nullptr), Resolver(nullptr) { // IR module verification is enabled by default in debug builds, and disabled // by default in release builds. #ifndef NDEBUG @@ -618,7 +618,18 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { case Type::ScalableVectorTyID: report_fatal_error( "Scalable vector support not yet implemented in ExecutionEngine"); - case Type::FixedVectorTyID: + case Type::ArrayTyID: { + auto *ArrTy = cast<ArrayType>(C->getType()); + Type *ElemTy = ArrTy->getElementType(); + unsigned int elemNum = ArrTy->getNumElements(); + Result.AggregateVal.resize(elemNum); + if (ElemTy->isIntegerTy()) + for (unsigned int i = 0; i < elemNum; ++i) + Result.AggregateVal[i].IntVal = + APInt(ElemTy->getPrimitiveSizeInBits(), 0); + break; + } + case Type::FixedVectorTyID: { // if the whole vector is 'undef' just reserve memory for the value. auto *VTy = cast<FixedVectorType>(C->getType()); Type *ElemTy = VTy->getElementType(); @@ -629,6 +640,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { Result.AggregateVal[i].IntVal = APInt(ElemTy->getPrimitiveSizeInBits(), 0); break; + } } return Result; } diff --git a/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp index dc9a07e3f212..772a3fa93c51 100644 --- a/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -138,8 +138,8 @@ LLVMBool LLVMCreateJITCompilerForModule(LLVMExecutionEngineRef *OutJIT, std::string Error; EngineBuilder builder(std::unique_ptr<Module>(unwrap(M))); builder.setEngineKind(EngineKind::JIT) - .setErrorStr(&Error) - .setOptLevel((CodeGenOpt::Level)OptLevel); + .setErrorStr(&Error) + .setOptLevel((CodeGenOptLevel)OptLevel); if (ExecutionEngine *JIT = builder.create()) { *OutJIT = wrap(JIT); return 0; @@ -196,9 +196,9 @@ LLVMBool LLVMCreateMCJITCompilerForModule( std::string Error; EngineBuilder builder(std::move(Mod)); builder.setEngineKind(EngineKind::JIT) - .setErrorStr(&Error) - .setOptLevel((CodeGenOpt::Level)options.OptLevel) - .setTargetOptions(targetOptions); + .setErrorStr(&Error) + .setOptLevel((CodeGenOptLevel)options.OptLevel) + .setTargetOptions(targetOptions); bool JIT; if (std::optional<CodeModel::Model> CM = unwrap(options.CodeModel, JIT)) builder.setCodeModel(*CM); diff --git a/llvm/lib/ExecutionEngine/JITLink/COFF.cpp b/llvm/lib/ExecutionEngine/JITLink/COFF.cpp index fddc9b813fb2..f4701bc830d6 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFF.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/COFF.cpp @@ -15,7 +15,6 @@ #include "llvm/BinaryFormat/COFF.h" #include "llvm/ExecutionEngine/JITLink/COFF_x86_64.h" #include "llvm/Object/COFF.h" -#include "llvm/Support/Endian.h" #include "llvm/Support/Format.h" #include "llvm/Support/MemoryBuffer.h" #include <cstring> diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp b/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp index 30c1579a1ba0..f23f3ed9406b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp @@ -36,20 +36,10 @@ static constexpr const ArrayRef<StringLiteral> PrefixTable(PrefixTable_init, std::size(PrefixTable_init) - 1); // Create table mapping all options defined in COFFOptions.td +using namespace llvm::opt; static constexpr opt::OptTable::Info infoTable[] = { -#define OPTION(X1, X2, ID, KIND, GROUP, ALIAS, X7, X8, X9, X10, X11, X12) \ - {X1, \ - X2, \ - X10, \ - X11, \ - COFF_OPT_##ID, \ - opt::Option::KIND##Class, \ - X9, \ - X8, \ - COFF_OPT_##GROUP, \ - COFF_OPT_##ALIAS, \ - X7, \ - X12}, +#define OPTION(...) \ + LLVM_CONSTRUCT_OPT_INFO_WITH_ID_PREFIX(COFF_OPT_, __VA_ARGS__), #include "COFFOptions.inc" #undef OPTION }; diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h b/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h index 5c953da7581f..21808f0afcb5 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h +++ b/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h @@ -26,7 +26,7 @@ namespace jitlink { enum { COFF_OPT_INVALID = 0, -#define OPTION(_1, _2, ID, _4, _5, _6, _7, _8, _9, _10, _11, _12) COFF_OPT_##ID, +#define OPTION(...) LLVM_MAKE_OPT_ID_WITH_ID_PREFIX(COFF_OPT_, __VA_ARGS__), #include "COFFOptions.inc" #undef OPTION }; diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp index 6668854e1a6a..1fd2a33d3f11 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// Generic COFF LinkGraph buliding code. +// Generic COFF LinkGraph building code. // //===----------------------------------------------------------------------===// #include "COFFLinkGraphBuilder.h" @@ -43,9 +43,10 @@ COFFLinkGraphBuilder::getPointerSize(const object::COFFObjectFile &Obj) { return Obj.getBytesInAddress(); } -support::endianness +llvm::endianness COFFLinkGraphBuilder::getEndianness(const object::COFFObjectFile &Obj) { - return Obj.isLittleEndian() ? support::little : support::big; + return Obj.isLittleEndian() ? llvm::endianness::little + : llvm::endianness::big; } uint64_t COFFLinkGraphBuilder::getSectionSize(const object::COFFObjectFile &Obj, @@ -161,7 +162,7 @@ Error COFFLinkGraphBuilder::graphifySections() { if (!GraphSec) { GraphSec = &G->createSection(SectionName, Prot); if ((*Sec)->Characteristics & COFF::IMAGE_SCN_LNK_REMOVE) - GraphSec->setMemLifetimePolicy(orc::MemLifetimePolicy::NoAlloc); + GraphSec->setMemLifetime(orc::MemLifetime::NoAlloc); } if (GraphSec->getMemProt() != Prot) return make_error<JITLinkError>("MemProt should match"); @@ -606,7 +607,7 @@ COFFLinkGraphBuilder::exportCOMDATSymbol(COFFSymbolIndex SymIndex, object::COFFSymbolRef Symbol) { Block *B = getGraphBlock(Symbol.getSectionNumber()); auto &PendingComdatExport = PendingComdatExports[Symbol.getSectionNumber()]; - // NOTE: ComdatDef->Legnth is the size of "section" not size of symbol. + // NOTE: ComdatDef->Length is the size of "section" not size of symbol. // We use zero symbol size to not reach out of bound of block when symbol // offset is non-zero. auto GSym = &G->addDefinedSymbol( diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h index e64823759540..e5f3ce8c53f5 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h @@ -14,7 +14,6 @@ #define LIB_EXECUTIONENGINE_JITLINK_COFFLINKGRAPHBUILDER_H #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/StringMap.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/Object/COFF.h" @@ -162,7 +161,7 @@ private: const object::coff_section *Section); static bool isComdatSection(const object::coff_section *Section); static unsigned getPointerSize(const object::COFFObjectFile &Obj); - static support::endianness getEndianness(const object::COFFObjectFile &Obj); + static llvm::endianness getEndianness(const object::COFFObjectFile &Obj); static StringRef getDLLImportStubPrefix() { return "__imp_"; } static StringRef getDirectiveSectionName() { return ".drectve"; } StringRef getCOFFSectionName(COFFSectionIndex SectionIndex, diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 86249591a9be..c11577b03fd7 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -126,83 +126,71 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { } // Find the offsets of any existing edges from this block. - BlockEdgeMap BlockEdges; + BlockEdgesInfo BlockEdges; for (auto &E : B.edges()) if (E.isRelocation()) { - if (BlockEdges.count(E.getOffset())) - return make_error<JITLinkError>( - "Multiple relocations at offset " + - formatv("{0:x16}", E.getOffset()) + " in " + EHFrameSectionName + - " block at address " + formatv("{0:x16}", B.getAddress())); + // Check if we already saw more than one relocation at this offset. + if (BlockEdges.Multiple.contains(E.getOffset())) + continue; - BlockEdges[E.getOffset()] = EdgeTarget(E); + // Otherwise check if we previously had exactly one relocation at this + // offset. If so, we now have a second one and move it from the TargetMap + // into the Multiple set. + auto It = BlockEdges.TargetMap.find(E.getOffset()); + if (It != BlockEdges.TargetMap.end()) { + BlockEdges.TargetMap.erase(It); + BlockEdges.Multiple.insert(E.getOffset()); + } else { + BlockEdges.TargetMap[E.getOffset()] = EdgeTarget(E); + } } - CIEInfosMap CIEInfos; BinaryStreamReader BlockReader( StringRef(B.getContent().data(), B.getContent().size()), PC.G.getEndianness()); - while (!BlockReader.empty()) { - size_t RecordStartOffset = BlockReader.getOffset(); - LLVM_DEBUG({ - dbgs() << " Processing CFI record at " - << (B.getAddress() + RecordStartOffset) << "\n"; - }); + // Get the record length. + Expected<size_t> RecordRemaining = readCFIRecordLength(B, BlockReader); + if (!RecordRemaining) + return RecordRemaining.takeError(); - // Get the record length. - Expected<size_t> RecordRemaining = readCFIRecordLength(B, BlockReader); - if (!RecordRemaining) - return RecordRemaining.takeError(); + // We expect DWARFRecordSectionSplitter to split each CFI record into its own + // block. + if (BlockReader.bytesRemaining() != *RecordRemaining) + return make_error<JITLinkError>("Incomplete CFI record at " + + formatv("{0:x16}", B.getAddress())); - if (BlockReader.bytesRemaining() < *RecordRemaining) - return make_error<JITLinkError>( - "Incomplete CFI record at " + - formatv("{0:x16}", B.getAddress() + RecordStartOffset)); + // Read the CIE delta for this record. + uint64_t CIEDeltaFieldOffset = BlockReader.getOffset(); + uint32_t CIEDelta; + if (auto Err = BlockReader.readInteger(CIEDelta)) + return Err; - // Read the CIE delta for this record. - uint64_t CIEDeltaFieldOffset = BlockReader.getOffset() - RecordStartOffset; - uint32_t CIEDelta; - if (auto Err = BlockReader.readInteger(CIEDelta)) + if (CIEDelta == 0) { + if (auto Err = processCIE(PC, B, CIEDeltaFieldOffset, BlockEdges)) + return Err; + } else { + if (auto Err = processFDE(PC, B, CIEDeltaFieldOffset, CIEDelta, BlockEdges)) return Err; - - if (CIEDelta == 0) { - if (auto Err = processCIE(PC, B, RecordStartOffset, - CIEDeltaFieldOffset + *RecordRemaining, - CIEDeltaFieldOffset, BlockEdges)) - return Err; - } else { - if (auto Err = processFDE(PC, B, RecordStartOffset, - CIEDeltaFieldOffset + *RecordRemaining, - CIEDeltaFieldOffset, CIEDelta, BlockEdges)) - return Err; - } - - // Move to the next record. - BlockReader.setOffset(RecordStartOffset + CIEDeltaFieldOffset + - *RecordRemaining); } return Error::success(); } Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, - size_t RecordOffset, size_t RecordLength, size_t CIEDeltaFieldOffset, - const BlockEdgeMap &BlockEdges) { + const BlockEdgesInfo &BlockEdges) { - LLVM_DEBUG(dbgs() << " Record is CIE\n"); + LLVM_DEBUG(dbgs() << " Record is CIE\n"); - auto RecordContent = B.getContent().slice(RecordOffset, RecordLength); BinaryStreamReader RecordReader( - StringRef(RecordContent.data(), RecordContent.size()), + StringRef(B.getContent().data(), B.getContent().size()), PC.G.getEndianness()); // Skip past the CIE delta field: we've already processed this far. RecordReader.setOffset(CIEDeltaFieldOffset + 4); - auto &CIESymbol = - PC.G.addAnonymousSymbol(B, RecordOffset, RecordLength, false, false); + auto &CIESymbol = PC.G.addAnonymousSymbol(B, 0, B.getSize(), false, false); CIEInformation CIEInfo(CIESymbol); uint8_t Version = 0; @@ -268,7 +256,7 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, if (auto Err = getOrCreateEncodedPointerEdge( PC, BlockEdges, *PersonalityPointerEncoding, RecordReader, - B, RecordOffset + RecordReader.getOffset(), "personality") + B, RecordReader.getOffset(), "personality") .takeError()) return Err; break; @@ -279,7 +267,7 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, if (CIEInfo.AddressEncoding == dwarf::DW_EH_PE_omit) return make_error<JITLinkError>( "Invalid address encoding DW_EH_PE_omit in CIE at " + - formatv("{0:x}", (B.getAddress() + RecordOffset).getValue())); + formatv("{0:x}", B.getAddress().getValue())); } else return PE.takeError(); break; @@ -302,35 +290,37 @@ Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, } Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, - size_t RecordOffset, size_t RecordLength, size_t CIEDeltaFieldOffset, uint32_t CIEDelta, - const BlockEdgeMap &BlockEdges) { - LLVM_DEBUG(dbgs() << " Record is FDE\n"); + const BlockEdgesInfo &BlockEdges) { + LLVM_DEBUG(dbgs() << " Record is FDE\n"); - orc::ExecutorAddr RecordAddress = B.getAddress() + RecordOffset; + orc::ExecutorAddr RecordAddress = B.getAddress(); - auto RecordContent = B.getContent().slice(RecordOffset, RecordLength); BinaryStreamReader RecordReader( - StringRef(RecordContent.data(), RecordContent.size()), + StringRef(B.getContent().data(), B.getContent().size()), PC.G.getEndianness()); // Skip past the CIE delta field: we've already read this far. RecordReader.setOffset(CIEDeltaFieldOffset + 4); - auto &FDESymbol = - PC.G.addAnonymousSymbol(B, RecordOffset, RecordLength, false, false); + auto &FDESymbol = PC.G.addAnonymousSymbol(B, 0, B.getSize(), false, false); CIEInformation *CIEInfo = nullptr; { // Process the CIE pointer field. - auto CIEEdgeItr = BlockEdges.find(RecordOffset + CIEDeltaFieldOffset); + if (BlockEdges.Multiple.contains(CIEDeltaFieldOffset)) + return make_error<JITLinkError>( + "CIE pointer field already has multiple edges at " + + formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset)); + + auto CIEEdgeItr = BlockEdges.TargetMap.find(CIEDeltaFieldOffset); + orc::ExecutorAddr CIEAddress = RecordAddress + orc::ExecutorAddrDiff(CIEDeltaFieldOffset) - orc::ExecutorAddrDiff(CIEDelta); - if (CIEEdgeItr == BlockEdges.end()) { - + if (CIEEdgeItr == BlockEdges.TargetMap.end()) { LLVM_DEBUG({ dbgs() << " Adding edge at " << (RecordAddress + CIEDeltaFieldOffset) @@ -341,8 +331,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, else return CIEInfoOrErr.takeError(); assert(CIEInfo->CIESymbol && "CIEInfo has no CIE symbol set"); - B.addEdge(NegDelta32, RecordOffset + CIEDeltaFieldOffset, - *CIEInfo->CIESymbol, 0); + B.addEdge(NegDelta32, CIEDeltaFieldOffset, *CIEInfo->CIESymbol, 0); } else { LLVM_DEBUG({ dbgs() << " Already has edge at " @@ -364,7 +353,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, // Process the PC-Begin field. LLVM_DEBUG({ - dbgs() << " Processing PC-begin at " + dbgs() << " Processing PC-begin at " << (RecordAddress + RecordReader.getOffset()) << "\n"; }); if (auto PCBegin = getOrCreateEncodedPointerEdge( @@ -375,14 +364,14 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, // Add a keep-alive edge from the FDE target to the FDE to ensure that the // FDE is kept alive if its target is. LLVM_DEBUG({ - dbgs() << " Adding keep-alive edge from target at " + dbgs() << " Adding keep-alive edge from target at " << (*PCBegin)->getBlock().getAddress() << " to FDE at " << RecordAddress << "\n"; }); (*PCBegin)->getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); } else { LLVM_DEBUG({ - dbgs() << " WARNING: Not adding keep-alive edge to FDE at " + dbgs() << " WARNING: Not adding keep-alive edge to FDE at " << RecordAddress << ", which points to " << ((*PCBegin)->isExternal() ? "external" : "absolute") << " symbol \"" << (*PCBegin)->getName() @@ -409,7 +398,7 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, .takeError()) return Err; } else { - LLVM_DEBUG(dbgs() << " Record does not have LSDA field.\n"); + LLVM_DEBUG(dbgs() << " Record does not have LSDA field.\n"); } return Error::success(); @@ -520,7 +509,7 @@ Error EHFrameEdgeFixer::skipEncodedPointer(uint8_t PointerEncoding, } Expected<Symbol *> EHFrameEdgeFixer::getOrCreateEncodedPointerEdge( - ParseContext &PC, const BlockEdgeMap &BlockEdges, uint8_t PointerEncoding, + ParseContext &PC, const BlockEdgesInfo &BlockEdges, uint8_t PointerEncoding, BinaryStreamReader &RecordReader, Block &BlockToFix, size_t PointerFieldOffset, const char *FieldName) { using namespace dwarf; @@ -531,10 +520,10 @@ Expected<Symbol *> EHFrameEdgeFixer::getOrCreateEncodedPointerEdge( // If there's already an edge here then just skip the encoded pointer and // return the edge's target. { - auto EdgeI = BlockEdges.find(PointerFieldOffset); - if (EdgeI != BlockEdges.end()) { + auto EdgeI = BlockEdges.TargetMap.find(PointerFieldOffset); + if (EdgeI != BlockEdges.TargetMap.end()) { LLVM_DEBUG({ - dbgs() << " Existing edge at " + dbgs() << " Existing edge at " << (BlockToFix.getAddress() + PointerFieldOffset) << " to " << FieldName << " at " << EdgeI->second.Target->getAddress(); if (EdgeI->second.Target->hasName()) @@ -545,6 +534,10 @@ Expected<Symbol *> EHFrameEdgeFixer::getOrCreateEncodedPointerEdge( return std::move(Err); return EdgeI->second.Target; } + + if (BlockEdges.Multiple.contains(PointerFieldOffset)) + return make_error<JITLinkError>("Multiple relocations at offset " + + formatv("{0:x16}", PointerFieldOffset)); } // Switch absptr to corresponding udata encoding. @@ -596,7 +589,7 @@ Expected<Symbol *> EHFrameEdgeFixer::getOrCreateEncodedPointerEdge( BlockToFix.addEdge(PtrEdgeKind, PointerFieldOffset, *TargetSym, 0); LLVM_DEBUG({ - dbgs() << " Adding edge at " + dbgs() << " Adding edge at " << (BlockToFix.getAddress() + PointerFieldOffset) << " to " << FieldName << " at " << TargetSym->getAddress(); if (TargetSym->hasName()) diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index 55cf7fc63ee7..49fbf650e7a7 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -60,7 +60,11 @@ private: Edge::AddendT Addend = 0; }; - using BlockEdgeMap = DenseMap<Edge::OffsetT, EdgeTarget>; + struct BlockEdgesInfo { + DenseMap<Edge::OffsetT, EdgeTarget> TargetMap; + DenseSet<Edge::OffsetT> Multiple; + }; + using CIEInfosMap = DenseMap<orc::ExecutorAddr, CIEInformation>; struct ParseContext { @@ -81,12 +85,10 @@ private: }; Error processBlock(ParseContext &PC, Block &B); - Error processCIE(ParseContext &PC, Block &B, size_t RecordOffset, - size_t RecordLength, size_t CIEDeltaFieldOffset, - const BlockEdgeMap &BlockEdges); - Error processFDE(ParseContext &PC, Block &B, size_t RecordOffset, - size_t RecordLength, size_t CIEDeltaFieldOffset, - uint32_t CIEDelta, const BlockEdgeMap &BlockEdges); + Error processCIE(ParseContext &PC, Block &B, size_t CIEDeltaFieldOffset, + const BlockEdgesInfo &BlockEdges); + Error processFDE(ParseContext &PC, Block &B, size_t CIEDeltaFieldOffset, + uint32_t CIEDelta, const BlockEdgesInfo &BlockEdges); Expected<AugmentationInfo> parseAugmentationString(BinaryStreamReader &RecordReader); @@ -96,9 +98,9 @@ private: Error skipEncodedPointer(uint8_t PointerEncoding, BinaryStreamReader &RecordReader); Expected<Symbol *> getOrCreateEncodedPointerEdge( - ParseContext &PC, const BlockEdgeMap &BlockEdges, uint8_t PointerEncoding, - BinaryStreamReader &RecordReader, Block &BlockToFix, - size_t PointerFieldOffset, const char *FieldName); + ParseContext &PC, const BlockEdgesInfo &BlockEdges, + uint8_t PointerEncoding, BinaryStreamReader &RecordReader, + Block &BlockToFix, size_t PointerFieldOffset, const char *FieldName); Expected<Symbol &> getOrCreateSymbol(ParseContext &PC, orc::ExecutorAddr Addr); diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp index dd08a23306ff..fdcce20cd2d1 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp @@ -21,7 +21,6 @@ #include "llvm/ExecutionEngine/JITLink/ELF_riscv.h" #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" #include "llvm/Object/ELF.h" -#include "llvm/Support/Endian.h" #include "llvm/Support/Format.h" #include "llvm/Support/MemoryBuffer.h" #include <cstring> @@ -52,6 +51,22 @@ Expected<uint16_t> readTargetMachineArch(StringRef Buffer) { } } + if (Data[ELF::EI_DATA] == ELF::ELFDATA2MSB) { + if (Data[ELF::EI_CLASS] == ELF::ELFCLASS64) { + if (auto File = llvm::object::ELF64BEFile::create(Buffer)) { + return File->getHeader().e_machine; + } else { + return File.takeError(); + } + } else if (Data[ELF::EI_CLASS] == ELF::ELFCLASS32) { + if (auto File = llvm::object::ELF32BEFile::create(Buffer)) { + return File->getHeader().e_machine; + } else { + return File.takeError(); + } + } + } + return ELF::EM_NONE; } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.cpp index 5a983c219627..e081f47ca42f 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// Generic ELF LinkGraph buliding code. +// Generic ELF LinkGraph building code. // //===----------------------------------------------------------------------===// diff --git a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h index e72645798349..56d1efa4bdef 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h @@ -51,7 +51,7 @@ private: Section *CommonSection = nullptr; }; -/// Ling-graph building code that's specific to the given ELFT, but common +/// LinkGraph building code that's specific to the given ELFT, but common /// across all architectures. template <typename ELFT> class ELFLinkGraphBuilder : public ELFLinkGraphBuilderBase { @@ -193,7 +193,7 @@ ELFLinkGraphBuilder<ELFT>::ELFLinkGraphBuilder( StringRef FileName, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName) : ELFLinkGraphBuilderBase(std::make_unique<LinkGraph>( FileName.str(), Triple(std::move(TT)), std::move(Features), - ELFT::Is64Bits ? 8 : 4, support::endianness(ELFT::TargetEndianness), + ELFT::Is64Bits ? 8 : 4, llvm::endianness(ELFT::TargetEndianness), std::move(GetEdgeKindName))), Obj(Obj) { LLVM_DEBUG( @@ -366,7 +366,7 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySections() { GraphSec = &G->createSection(*Name, Prot); // Non-SHF_ALLOC sections get NoAlloc memory lifetimes. if (!(Sec.sh_flags & ELF::SHF_ALLOC)) { - GraphSec->setMemLifetimePolicy(orc::MemLifetimePolicy::NoAlloc); + GraphSec->setMemLifetime(orc::MemLifetime::NoAlloc); LLVM_DEBUG({ dbgs() << " " << SecIndex << ": \"" << *Name << "\" is not a SHF_ALLOC section. Using NoAlloc lifetime.\n"; @@ -374,7 +374,14 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySections() { } } - assert(GraphSec->getMemProt() == Prot && "MemProt should match"); + if (GraphSec->getMemProt() != Prot) { + std::string ErrMsg; + raw_string_ostream(ErrMsg) + << "In " << G->getName() << ", section " << *Name + << " is present more than once with different permissions: " + << GraphSec->getMemProt() << " vs " << Prot; + return make_error<JITLinkError>(std::move(ErrMsg)); + } Block *B = nullptr; if (Sec.sh_type != ELF::SHT_NOBITS) { @@ -499,6 +506,22 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { TargetFlagsType Flags = makeTargetFlags(Sym); orc::ExecutorAddrDiff Offset = getRawOffset(Sym, Flags); + if (Offset + Sym.st_size > B->getSize()) { + std::string ErrMsg; + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "In " << G->getName() << ", symbol "; + if (!Name->empty()) + ErrStream << *Name; + else + ErrStream << "<anon>"; + ErrStream << " (" << (B->getAddress() + Offset) << " -- " + << (B->getAddress() + Offset + Sym.st_size) << ") extends " + << formatv("{0:x}", Offset + Sym.st_size - B->getSize()) + << " bytes past the end of its containing block (" + << B->getRange() << ")"; + return make_error<JITLinkError>(std::move(ErrMsg)); + } + // In RISCV, temporary symbols (Used to generate dwarf, eh_frame // sections...) will appear in object code's symbol table, and LLVM does // not use names on these temporary symbols (RISCV gnu toolchain uses diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp index a1bc4c853323..132989fcbce0 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp @@ -17,7 +17,6 @@ #include "llvm/ExecutionEngine/JITLink/aarch32.h" #include "llvm/Object/ELF.h" #include "llvm/Object/ELFObjectFile.h" -#include "llvm/Support/Endian.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TargetParser/ARMTargetParser.h" @@ -40,6 +39,12 @@ Expected<aarch32::EdgeKind_aarch32> getJITLinkEdgeKind(uint32_t ELFType) { return aarch32::Data_Delta32; case ELF::R_ARM_CALL: return aarch32::Arm_Call; + case ELF::R_ARM_JUMP24: + return aarch32::Arm_Jump24; + case ELF::R_ARM_MOVW_ABS_NC: + return aarch32::Arm_MovwAbsNC; + case ELF::R_ARM_MOVT_ABS: + return aarch32::Arm_MovtAbs; case ELF::R_ARM_THM_CALL: return aarch32::Thumb_Call; case ELF::R_ARM_THM_JUMP24: @@ -48,6 +53,10 @@ Expected<aarch32::EdgeKind_aarch32> getJITLinkEdgeKind(uint32_t ELFType) { return aarch32::Thumb_MovwAbsNC; case ELF::R_ARM_THM_MOVT_ABS: return aarch32::Thumb_MovtAbs; + case ELF::R_ARM_THM_MOVW_PREL_NC: + return aarch32::Thumb_MovwPrelNC; + case ELF::R_ARM_THM_MOVT_PREL: + return aarch32::Thumb_MovtPrel; } return make_error<JITLinkError>( @@ -64,6 +73,12 @@ Expected<uint32_t> getELFRelocationType(Edge::Kind Kind) { return ELF::R_ARM_ABS32; case aarch32::Arm_Call: return ELF::R_ARM_CALL; + case aarch32::Arm_Jump24: + return ELF::R_ARM_JUMP24; + case aarch32::Arm_MovwAbsNC: + return ELF::R_ARM_MOVW_ABS_NC; + case aarch32::Arm_MovtAbs: + return ELF::R_ARM_MOVT_ABS; case aarch32::Thumb_Call: return ELF::R_ARM_THM_CALL; case aarch32::Thumb_Jump24: @@ -72,6 +87,10 @@ Expected<uint32_t> getELFRelocationType(Edge::Kind Kind) { return ELF::R_ARM_THM_MOVW_ABS_NC; case aarch32::Thumb_MovtAbs: return ELF::R_ARM_THM_MOVT_ABS; + case aarch32::Thumb_MovwPrelNC: + return ELF::R_ARM_THM_MOVW_PREL_NC; + case aarch32::Thumb_MovtPrel: + return ELF::R_ARM_THM_MOVT_PREL; } return make_error<JITLinkError>(formatv("Invalid aarch32 edge {0:d}: ", @@ -102,7 +121,7 @@ private: } }; -template <support::endianness DataEndianness> +template <llvm::endianness DataEndianness> class ELFLinkGraphBuilder_aarch32 : public ELFLinkGraphBuilder<ELFType<DataEndianness, false>> { private: @@ -154,14 +173,13 @@ private: auto FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); - Edge E(*Kind, Offset, *GraphSymbol, 0); Expected<int64_t> Addend = - aarch32::readAddend(*Base::G, BlockToFix, E, ArmCfg); + aarch32::readAddend(*Base::G, BlockToFix, Offset, *Kind, ArmCfg); if (!Addend) return Addend.takeError(); - E.setAddend(*Addend); + Edge E(*Kind, Offset, *GraphSymbol, *Addend); LLVM_DEBUG({ dbgs() << " "; printEdge(dbgs(), BlockToFix, E, getELFAArch32EdgeKindName(*Kind)); @@ -253,7 +271,7 @@ createLinkGraphFromELFObject_aarch32(MemoryBufferRef ObjectBuffer) { case Triple::arm: case Triple::thumb: { auto &ELFFile = cast<ELFObjectFile<ELF32LE>>(**ELFObj).getELFFile(); - return ELFLinkGraphBuilder_aarch32<support::little>( + return ELFLinkGraphBuilder_aarch32<llvm::endianness::little>( (*ELFObj)->getFileName(), ELFFile, TT, std::move(*Features), ArmCfg) .buildGraph(); @@ -261,7 +279,7 @@ createLinkGraphFromELFObject_aarch32(MemoryBufferRef ObjectBuffer) { case Triple::armeb: case Triple::thumbeb: { auto &ELFFile = cast<ELFObjectFile<ELF32BE>>(**ELFObj).getELFFile(); - return ELFLinkGraphBuilder_aarch32<support::big>( + return ELFLinkGraphBuilder_aarch32<llvm::endianness::big>( (*ELFObj)->getFileName(), ELFFile, TT, std::move(*Features), ArmCfg) .buildGraph(); diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 652eb931190e..f17b2c626ac2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -598,7 +598,7 @@ void link_ELF_aarch64(std::unique_ptr<LinkGraph> G, PassConfiguration Config; const Triple &TT = G->getTargetTriple(); if (Ctx->shouldAddDefaultTargetPasses(TT)) { - // Add eh-frame passses. + // Add eh-frame passes. Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); Config.PrePrunePasses.push_back(EHFrameEdgeFixer( ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp index 7f76b45aecbb..aa9385fcb183 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp @@ -186,7 +186,7 @@ void link_ELF_loongarch(std::unique_ptr<LinkGraph> G, PassConfiguration Config; const Triple &TT = G->getTargetTriple(); if (Ctx->shouldAddDefaultTargetPasses(TT)) { - // Add eh-frame passses. + // Add eh-frame passes. Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); Config.PrePrunePasses.push_back( EHFrameEdgeFixer(".eh_frame", G->getPointerSize(), Pointer32, Pointer64, diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp index a30b9ce51c84..3b86250b60a4 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp @@ -15,7 +15,6 @@ #include "llvm/ExecutionEngine/JITLink/TableManager.h" #include "llvm/ExecutionEngine/JITLink/ppc64.h" #include "llvm/Object/ELFObjectFile.h" -#include "llvm/Support/Endian.h" #include "EHFrameSupportImpl.h" #include "ELFLinkGraphBuilder.h" @@ -31,8 +30,77 @@ using namespace llvm::jitlink; constexpr StringRef ELFTOCSymbolName = ".TOC."; constexpr StringRef TOCSymbolAliasIdent = "__TOC__"; constexpr uint64_t ELFTOCBaseOffset = 0x8000; +constexpr StringRef ELFTLSInfoSectionName = "$__TLSINFO"; -template <support::endianness Endianness> +template <llvm::endianness Endianness> +class TLSInfoTableManager_ELF_ppc64 + : public TableManager<TLSInfoTableManager_ELF_ppc64<Endianness>> { +public: + static const uint8_t TLSInfoEntryContent[16]; + + static StringRef getSectionName() { return ELFTLSInfoSectionName; } + + bool visitEdge(LinkGraph &G, Block *B, Edge &E) { + Edge::Kind K = E.getKind(); + switch (K) { + case ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16HA: + E.setKind(ppc64::TOCDelta16HA); + E.setTarget(this->getEntryForTarget(G, E.getTarget())); + return true; + case ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16LO: + E.setKind(ppc64::TOCDelta16LO); + E.setTarget(this->getEntryForTarget(G, E.getTarget())); + return true; + case ppc64::RequestTLSDescInGOTAndTransformToDelta34: + E.setKind(ppc64::Delta34); + E.setTarget(this->getEntryForTarget(G, E.getTarget())); + return true; + default: + return false; + } + } + + Symbol &createEntry(LinkGraph &G, Symbol &Target) { + // The TLS Info entry's key value will be written by + // `fixTLVSectionsAndEdges`, so create mutable content. + auto &TLSInfoEntry = G.createMutableContentBlock( + getTLSInfoSection(G), G.allocateContent(getTLSInfoEntryContent()), + orc::ExecutorAddr(), 8, 0); + TLSInfoEntry.addEdge(ppc64::Pointer64, 8, Target, 0); + return G.addAnonymousSymbol(TLSInfoEntry, 0, 16, false, false); + } + +private: + Section &getTLSInfoSection(LinkGraph &G) { + if (!TLSInfoTable) + TLSInfoTable = + &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read); + return *TLSInfoTable; + } + + ArrayRef<char> getTLSInfoEntryContent() const { + return {reinterpret_cast<const char *>(TLSInfoEntryContent), + sizeof(TLSInfoEntryContent)}; + } + + Section *TLSInfoTable = nullptr; +}; + +template <> +const uint8_t TLSInfoTableManager_ELF_ppc64< + llvm::endianness::little>::TLSInfoEntryContent[16] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ +}; + +template <> +const uint8_t TLSInfoTableManager_ELF_ppc64< + llvm::endianness::big>::TLSInfoEntryContent[16] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /*pthread key */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /*data address*/ +}; + +template <llvm::endianness Endianness> Symbol &createELFGOTHeader(LinkGraph &G, ppc64::TOCTableManager<Endianness> &TOC) { Symbol *TOCSymbol = nullptr; @@ -58,7 +126,7 @@ Symbol &createELFGOTHeader(LinkGraph &G, } // Register preexisting GOT entries with TOC table manager. -template <support::endianness Endianness> +template <llvm::endianness Endianness> inline void registerExistingGOTEntries(LinkGraph &G, ppc64::TOCTableManager<Endianness> &TOC) { @@ -76,7 +144,7 @@ registerExistingGOTEntries(LinkGraph &G, } } -template <support::endianness Endianness> +template <llvm::endianness Endianness> Error buildTables_ELF_ppc64(LinkGraph &G) { LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); ppc64::TOCTableManager<Endianness> TOC; @@ -91,8 +159,8 @@ Error buildTables_ELF_ppc64(LinkGraph &G) { registerExistingGOTEntries(G, TOC); ppc64::PLTTableManager<Endianness> PLT(TOC); - visitExistingEdges(G, TOC, PLT); - // TODO: Add TLS support. + TLSInfoTableManager_ELF_ppc64<Endianness> TLSInfo; + visitExistingEdges(G, TOC, PLT, TLSInfo); // After visiting edges in LinkGraph, we have GOT entries built in the // synthesized section. @@ -125,7 +193,7 @@ Error buildTables_ELF_ppc64(LinkGraph &G) { namespace llvm::jitlink { -template <support::endianness Endianness> +template <llvm::endianness Endianness> class ELFLinkGraphBuilder_ppc64 : public ELFLinkGraphBuilder<object::ELFType<Endianness, true>> { private: @@ -164,6 +232,21 @@ private: if (LLVM_UNLIKELY(ELFReloc == ELF::R_PPC64_NONE)) return Error::success(); + // TLS model markers. We only support global-dynamic model now. + if (ELFReloc == ELF::R_PPC64_TLSGD) + return Error::success(); + if (ELFReloc == ELF::R_PPC64_TLSLD) + return make_error<StringError>("Local-dynamic TLS model is not supported", + inconvertibleErrorCode()); + + if (ELFReloc == ELF::R_PPC64_PCREL_OPT) + // TODO: Support PCREL optimization, now ignore it. + return Error::success(); + + if (ELFReloc == ELF::R_PPC64_TPREL34) + return make_error<StringError>("Local-exec TLS model is not supported", + inconvertibleErrorCode()); + auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); if (!ObjSymbol) return ObjSymbol.takeError(); @@ -192,9 +275,60 @@ private: case ELF::R_PPC64_ADDR64: Kind = ppc64::Pointer64; break; + case ELF::R_PPC64_ADDR32: + Kind = ppc64::Pointer32; + break; + case ELF::R_PPC64_ADDR16: + Kind = ppc64::Pointer16; + break; + case ELF::R_PPC64_ADDR16_DS: + Kind = ppc64::Pointer16DS; + break; + case ELF::R_PPC64_ADDR16_HA: + Kind = ppc64::Pointer16HA; + break; + case ELF::R_PPC64_ADDR16_HI: + Kind = ppc64::Pointer16HI; + break; + case ELF::R_PPC64_ADDR16_HIGH: + Kind = ppc64::Pointer16HIGH; + break; + case ELF::R_PPC64_ADDR16_HIGHA: + Kind = ppc64::Pointer16HIGHA; + break; + case ELF::R_PPC64_ADDR16_HIGHER: + Kind = ppc64::Pointer16HIGHER; + break; + case ELF::R_PPC64_ADDR16_HIGHERA: + Kind = ppc64::Pointer16HIGHERA; + break; + case ELF::R_PPC64_ADDR16_HIGHEST: + Kind = ppc64::Pointer16HIGHEST; + break; + case ELF::R_PPC64_ADDR16_HIGHESTA: + Kind = ppc64::Pointer16HIGHESTA; + break; + case ELF::R_PPC64_ADDR16_LO: + Kind = ppc64::Pointer16LO; + break; + case ELF::R_PPC64_ADDR16_LO_DS: + Kind = ppc64::Pointer16LODS; + break; + case ELF::R_PPC64_ADDR14: + Kind = ppc64::Pointer14; + break; + case ELF::R_PPC64_TOC: + Kind = ppc64::TOC; + break; + case ELF::R_PPC64_TOC16: + Kind = ppc64::TOCDelta16; + break; case ELF::R_PPC64_TOC16_HA: Kind = ppc64::TOCDelta16HA; break; + case ELF::R_PPC64_TOC16_HI: + Kind = ppc64::TOCDelta16HI; + break; case ELF::R_PPC64_TOC16_DS: Kind = ppc64::TOCDelta16DS; break; @@ -210,6 +344,9 @@ private: case ELF::R_PPC64_REL16_HA: Kind = ppc64::Delta16HA; break; + case ELF::R_PPC64_REL16_HI: + Kind = ppc64::Delta16HI; + break; case ELF::R_PPC64_REL16_LO: Kind = ppc64::Delta16LO; break; @@ -217,26 +354,36 @@ private: Kind = ppc64::Delta32; break; case ELF::R_PPC64_REL24_NOTOC: - case ELF::R_PPC64_REL24: { - bool isLocal = !GraphSymbol->isExternal(); - if (isLocal) { - // TODO: There are cases a local function call need a call stub. - // 1. Caller uses TOC, the callee doesn't, need a r2 save stub. - // 2. Caller doesn't use TOC, the callee does, need a r12 setup stub. - // FIXME: For a local call, we might need a thunk if branch target is - // out of range. - Kind = ppc64::CallBranchDelta; - // Branch to local entry. - Addend += ELF::decodePPC64LocalEntryOffset((*ObjSymbol)->st_other); - } else { - Kind = ELFReloc == ELF::R_PPC64_REL24 ? ppc64::RequestPLTCallStubSaveTOC - : ppc64::RequestPLTCallStubNoTOC; - } + Kind = ppc64::RequestCallNoTOC; + break; + case ELF::R_PPC64_REL24: + Kind = ppc64::RequestCall; + // Determining a target is external or not is deferred in PostPrunePass. + // We assume branching to local entry by default, since in PostPrunePass, + // we don't have any context to determine LocalEntryOffset. If it finally + // turns out to be an external call, we'll have a stub for the external + // target, the target of this edge will be the stub and its addend will be + // set 0. + Addend += ELF::decodePPC64LocalEntryOffset((*ObjSymbol)->st_other); break; - } case ELF::R_PPC64_REL64: Kind = ppc64::Delta64; break; + case ELF::R_PPC64_PCREL34: + Kind = ppc64::Delta34; + break; + case ELF::R_PPC64_GOT_PCREL34: + Kind = ppc64::RequestGOTAndTransformToDelta34; + break; + case ELF::R_PPC64_GOT_TLSGD16_HA: + Kind = ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16HA; + break; + case ELF::R_PPC64_GOT_TLSGD16_LO: + Kind = ppc64::RequestTLSDescInGOTAndTransformToTOCDelta16LO; + break; + case ELF::R_PPC64_GOT_TLSGD_PCREL34: + Kind = ppc64::RequestTLSDescInGOTAndTransformToDelta34; + break; } Edge GE(Kind, Offset, *GraphSymbol, Addend); @@ -252,7 +399,7 @@ public: FileName, ppc64::getEdgeKindName) {} }; -template <support::endianness Endianness> +template <llvm::endianness Endianness> class ELFJITLinker_ppc64 : public JITLinker<ELFJITLinker_ppc64<Endianness>> { using JITLinkerBase = JITLinker<ELFJITLinker_ppc64<Endianness>>; friend JITLinkerBase; @@ -314,7 +461,7 @@ private: } }; -template <support::endianness Endianness> +template <llvm::endianness Endianness> Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) { LLVM_DEBUG({ @@ -338,7 +485,7 @@ createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) { .buildGraph(); } -template <support::endianness Endianness> +template <llvm::endianness Endianness> void link_ELF_ppc64(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) { PassConfiguration Config; @@ -346,7 +493,7 @@ void link_ELF_ppc64(std::unique_ptr<LinkGraph> G, if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { // Construct a JITLinker and run the link function. - // Add eh-frame passses. + // Add eh-frame passes. Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); Config.PrePrunePasses.push_back(EHFrameEdgeFixer( ".eh_frame", G->getPointerSize(), ppc64::Pointer32, ppc64::Pointer64, @@ -371,26 +518,26 @@ void link_ELF_ppc64(std::unique_ptr<LinkGraph> G, Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) { - return createLinkGraphFromELFObject_ppc64<support::big>( + return createLinkGraphFromELFObject_ppc64<llvm::endianness::big>( std::move(ObjectBuffer)); } Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject_ppc64le(MemoryBufferRef ObjectBuffer) { - return createLinkGraphFromELFObject_ppc64<support::little>( + return createLinkGraphFromELFObject_ppc64<llvm::endianness::little>( std::move(ObjectBuffer)); } /// jit-link the given object buffer, which must be a ELF ppc64 object file. void link_ELF_ppc64(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) { - return link_ELF_ppc64<support::big>(std::move(G), std::move(Ctx)); + return link_ELF_ppc64<llvm::endianness::big>(std::move(G), std::move(Ctx)); } /// jit-link the given object buffer, which must be a ELF ppc64le object file. void link_ELF_ppc64le(std::unique_ptr<LinkGraph> G, std::unique_ptr<JITLinkContext> Ctx) { - return link_ELF_ppc64<support::little>(std::move(G), std::move(Ctx)); + return link_ELF_ppc64<llvm::endianness::little>(std::move(G), std::move(Ctx)); } } // end namespace llvm::jitlink diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp index 410dd7fedad1..d0701ba08bd9 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp @@ -11,10 +11,12 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/JITLink/ELF_riscv.h" +#include "EHFrameSupportImpl.h" #include "ELFLinkGraphBuilder.h" #include "JITLinkGeneric.h" #include "PerGraphGOTAndPLTStubsBuilder.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/ExecutionEngine/JITLink/riscv.h" #include "llvm/Object/ELF.h" @@ -456,6 +458,13 @@ private: case AlignRelaxable: // Ignore when the relaxation pass did not run break; + case NegDelta32: { + int64_t Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + if (LLVM_UNLIKELY(!isInRangeForImm(Value, 32))) + return makeTargetOutOfRangeError(G, B, E); + *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); + break; + } } return Error::success(); } @@ -516,8 +525,7 @@ static RelaxAux initRelaxAux(LinkGraph &G) { RelaxAux Aux; Aux.Config.IsRV32 = G.getTargetTriple().isRISCV32(); const auto &Features = G.getFeatures().getFeatures(); - Aux.Config.HasRVC = - std::find(Features.begin(), Features.end(), "+c") != Features.end(); + Aux.Config.HasRVC = llvm::is_contained(Features, "+c"); for (auto &S : G.sections()) { if (!shouldRelax(S)) @@ -959,6 +967,13 @@ void link_ELF_riscv(std::unique_ptr<LinkGraph> G, PassConfiguration Config; const Triple &TT = G->getTargetTriple(); if (Ctx->shouldAddDefaultTargetPasses(TT)) { + + Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + ".eh_frame", G->getPointerSize(), Edge::Invalid, Edge::Invalid, + Edge::Invalid, Edge::Invalid, NegDelta32)); + Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); + if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); else diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 1bdddd4c722b..46f8064bb168 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -16,7 +16,6 @@ #include "llvm/ExecutionEngine/JITLink/TableManager.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/Object/ELFObjectFile.h" -#include "llvm/Support/Endian.h" #include "DefineExternalSectionStartAndEndSymbols.h" #include "EHFrameSupportImpl.h" @@ -242,8 +241,10 @@ public: std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { - getPassConfig().PostAllocationPasses.push_back( - [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); + + if (shouldAddDefaultTargetPasses(getGraph().getTargetTriple())) + getPassConfig().PostAllocationPasses.push_back( + [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); } private: diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index 4a2755d3696b..d86ceb99ded0 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -13,6 +13,10 @@ #include "llvm/ExecutionEngine/JITLink/COFF.h" #include "llvm/ExecutionEngine/JITLink/ELF.h" #include "llvm/ExecutionEngine/JITLink/MachO.h" +#include "llvm/ExecutionEngine/JITLink/aarch64.h" +#include "llvm/ExecutionEngine/JITLink/i386.h" +#include "llvm/ExecutionEngine/JITLink/loongarch.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/Support/Format.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/raw_ostream.h" @@ -417,6 +421,38 @@ Error makeAlignmentError(llvm::orc::ExecutorAddr Loc, uint64_t Value, int N, " is not aligned to " + Twine(N) + " bytes"); } +AnonymousPointerCreator getAnonymousPointerCreator(const Triple &TT) { + switch (TT.getArch()) { + case Triple::aarch64: + return aarch64::createAnonymousPointer; + case Triple::x86_64: + return x86_64::createAnonymousPointer; + case Triple::x86: + return i386::createAnonymousPointer; + case Triple::loongarch32: + case Triple::loongarch64: + return loongarch::createAnonymousPointer; + default: + return nullptr; + } +} + +PointerJumpStubCreator getPointerJumpStubCreator(const Triple &TT) { + switch (TT.getArch()) { + case Triple::aarch64: + return aarch64::createAnonymousPointerJumpStub; + case Triple::x86_64: + return x86_64::createAnonymousPointerJumpStub; + case Triple::x86: + return i386::createAnonymousPointerJumpStub; + case Triple::loongarch32: + case Triple::loongarch64: + return loongarch::createAnonymousPointerJumpStub; + default: + return nullptr; + } +} + Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromObject(MemoryBufferRef ObjectBuffer) { auto Magic = identify_magic(ObjectBuffer.getBuffer()); diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index feaa0fb6a58c..5361272ae79e 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -65,7 +65,7 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, if (AR) Alloc = std::move(*AR); else - return abandonAllocAndBailOut(std::move(Self), AR.takeError()); + return Ctx->notifyFailed(AR.takeError()); LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index e69eddd6e119..e5d05e6b1b7b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -13,7 +13,6 @@ #ifndef LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H #define LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H -#include "llvm/ADT/DenseSet.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #define DEBUG_TYPE "jitlink" @@ -43,6 +42,16 @@ protected: using AllocResult = Expected<std::unique_ptr<InFlightAlloc>>; using FinalizeResult = Expected<JITLinkMemoryManager::FinalizedAlloc>; + // Returns a reference to the graph being linked. + LinkGraph &getGraph() { return *G; } + + // Returns true if the context says that the linker should add default + // passes. This can be used by JITLinkerBase implementations when deciding + // whether they should add default passes. + bool shouldAddDefaultTargetPasses(const Triple &TT) { + return Ctx->shouldAddDefaultTargetPasses(TT); + } + // Returns the PassConfiguration for this instance. This can be used by // JITLinkerBase implementations to add late passes that reference their // own data structures (e.g. for ELF implementations to locate / construct @@ -124,8 +133,7 @@ private: LLVM_DEBUG(dbgs() << "Fixing up blocks:\n"); for (auto &Sec : G.sections()) { - bool NoAllocSection = - Sec.getMemLifetimePolicy() == orc::MemLifetimePolicy::NoAlloc; + bool NoAllocSection = Sec.getMemLifetime() == orc::MemLifetime::NoAlloc; for (auto *B : Sec.blocks()) { LLVM_DEBUG(dbgs() << " " << *B << ":\n"); @@ -153,12 +161,11 @@ private: // If B is a block in a Standard or Finalize section then make sure // that no edges point to symbols in NoAlloc sections. - assert( - (NoAllocSection || !E.getTarget().isDefined() || - E.getTarget().getBlock().getSection().getMemLifetimePolicy() != - orc::MemLifetimePolicy::NoAlloc) && - "Block in allocated section has edge pointing to no-alloc " - "section"); + assert((NoAllocSection || !E.getTarget().isDefined() || + E.getTarget().getBlock().getSection().getMemLifetime() != + orc::MemLifetime::NoAlloc) && + "Block in allocated section has edge pointing to no-alloc " + "section"); // Dispatch to LinkerImpl for fixup. if (auto Err = impl().applyFixup(G, *B, E)) diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index f481504135a5..474a0b5160bc 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -26,10 +26,10 @@ BasicLayout::BasicLayout(LinkGraph &G) : G(G) { for (auto &Sec : G.sections()) { // Skip empty sections, and sections with NoAlloc lifetime policies. if (Sec.blocks().empty() || - Sec.getMemLifetimePolicy() == orc::MemLifetimePolicy::NoAlloc) + Sec.getMemLifetime() == orc::MemLifetime::NoAlloc) continue; - auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemLifetimePolicy()}]; + auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemLifetime()}]; for (auto *B : Sec.blocks()) if (LLVM_LIKELY(!B->isZeroFill())) Seg.ContentBlocks.push_back(B); @@ -90,7 +90,7 @@ BasicLayout::getContiguousPageBasedLayoutSizes(uint64_t PageSize) { inconvertibleErrorCode()); uint64_t SegSize = alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize); - if (AG.getMemLifetimePolicy() == orc::MemLifetimePolicy::Standard) + if (AG.getMemLifetime() == orc::MemLifetime::Standard) SegsSizes.StandardSegs += SegSize; else SegsSizes.FinalizeSegs += SegSize; @@ -155,8 +155,8 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, "__---.finalize", "__R--.finalize", "__-W-.finalize", "__RW-.finalize", "__--X.finalize", "__R-X.finalize", "__-WX.finalize", "__RWX.finalize"}; - auto G = - std::make_unique<LinkGraph>("", Triple(), 0, support::native, nullptr); + auto G = std::make_unique<LinkGraph>("", Triple(), 0, + llvm::endianness::native, nullptr); orc::AllocGroupSmallMap<Block *> ContentBlocks; orc::ExecutorAddr NextAddr(0x100000); @@ -164,15 +164,15 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, auto &AG = KV.first; auto &Seg = KV.second; - assert(AG.getMemLifetimePolicy() != orc::MemLifetimePolicy::NoAlloc && + assert(AG.getMemLifetime() != orc::MemLifetime::NoAlloc && "NoAlloc segments are not supported by SimpleSegmentAlloc"); auto AGSectionName = AGSectionNames[static_cast<unsigned>(AG.getMemProt()) | - static_cast<bool>(AG.getMemLifetimePolicy()) << 3]; + static_cast<bool>(AG.getMemLifetime()) << 3]; auto &Sec = G->createSection(AGSectionName, AG.getMemProt()); - Sec.setMemLifetimePolicy(AG.getMemLifetimePolicy()); + Sec.setMemLifetime(AG.getMemLifetime()); if (Seg.ContentSize != 0) { NextAddr = @@ -419,10 +419,9 @@ void InProcessMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, auto &AG = KV.first; auto &Seg = KV.second; - auto &SegAddr = - (AG.getMemLifetimePolicy() == orc::MemLifetimePolicy::Standard) - ? NextStandardSegAddr - : NextFinalizeSegAddr; + auto &SegAddr = (AG.getMemLifetime() == orc::MemLifetime::Standard) + ? NextStandardSegAddr + : NextFinalizeSegAddr; Seg.WorkingMem = SegAddr.toPtr<char *>(); Seg.Addr = SegAddr; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp index c40e0f9ffc8d..bcbc429cae12 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -6,7 +6,7 @@ // //===----------------------------------------------------------------------===// // -// Generic MachO LinkGraph buliding code. +// Generic MachO LinkGraph building code. // //===----------------------------------------------------------------------===// @@ -106,9 +106,10 @@ MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { return Obj.is64Bit() ? 8 : 4; } -support::endianness +llvm::endianness MachOLinkGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { - return Obj.isLittleEndian() ? support::little : support::big; + return Obj.isLittleEndian() ? llvm::endianness::little + : llvm::endianness::big; } Section &MachOLinkGraphBuilder::getCommonSection() { @@ -192,7 +193,7 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { // TODO: Are there any other criteria for NoAlloc lifetime? if (NSec.Flags & MachO::S_ATTR_DEBUG) - NSec.GraphSection->setMemLifetimePolicy(orc::MemLifetimePolicy::NoAlloc); + NSec.GraphSection->setMemLifetime(orc::MemLifetime::NoAlloc); IndexToSection.insert(std::make_pair(SecIndex, std::move(NSec))); } diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index 2805c2960b9b..a4ae0ac1ecfc 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -181,7 +181,7 @@ protected: private: static unsigned getPointerSize(const object::MachOObjectFile &Obj); - static support::endianness getEndianness(const object::MachOObjectFile &Obj); + static llvm::endianness getEndianness(const object::MachOObjectFile &Obj); void setCanonicalSymbol(NormalizedSection &NSec, Symbol &Sym) { auto *&CanonicalSymEntry = NSec.CanonicalSymbols[Sym.getAddress()]; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index dd0b5d37d1b7..409bec7a874b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -567,7 +567,7 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G, Config.PrePrunePasses.push_back( CompactUnwindSplitter("__LD,__compact_unwind")); - // Add eh-frame passses. + // Add eh-frame passes. // FIXME: Prune eh-frames for which compact-unwind is available once // we support compact-unwind registration with libunwind. Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_arm64()); diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 4dba27bc61cb..49f619357f08 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -482,7 +482,7 @@ void link_MachO_x86_64(std::unique_ptr<LinkGraph> G, PassConfiguration Config; if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { - // Add eh-frame passses. + // Add eh-frame passes. Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_x86_64()); Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_x86_64()); diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp index ffc3950cdec8..671ee1a81252 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp @@ -17,6 +17,7 @@ #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/Endian.h" +#include "llvm/Support/ManagedStatic.h" #include "llvm/Support/MathExtras.h" #define DEBUG_TYPE "jitlink" @@ -25,6 +26,11 @@ namespace llvm { namespace jitlink { namespace aarch32 { +/// Check whether the given target flags are set for this Symbol. +bool hasTargetFlags(Symbol &Sym, TargetFlagsType Flags) { + return static_cast<TargetFlagsType>(Sym.getTargetFlags()) & Flags; +} + /// Encode 22-bit immediate value for branch instructions without J1J2 range /// extension (formats B T4, BL T1 and BLX T2). /// @@ -78,6 +84,24 @@ int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) { return SignExtend64<25>(S << 14 | I1 | I2 | Imm10 << 12 | Imm11 << 1); } +/// Encode 26-bit immediate value for branch instructions +/// (formats B A1, BL A1 and BLX A2). +/// +/// Imm24:00 -> 00000000:Imm24 +/// +uint32_t encodeImmBA1BlA1BlxA2(int64_t Value) { + return (Value >> 2) & 0x00ffffff; +} + +/// Decode 26-bit immediate value for branch instructions +/// (formats B A1, BL A1 and BLX A2). +/// +/// 00000000:Imm24 -> Imm24:00 +/// +int64_t decodeImmBA1BlA1BlxA2(int64_t Value) { + return SignExtend64<26>((Value & 0x00ffffff) << 2); +} + /// Encode 16-bit immediate value for move instruction formats MOVT T1 and /// MOVW T3. /// @@ -124,6 +148,50 @@ int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) { return Rd4; } +/// Encode 16-bit immediate value for move instruction formats MOVT A1 and +/// MOVW A2. +/// +/// Imm4:Imm12 -> 000000000000:Imm4:0000:Imm12 +/// +uint32_t encodeImmMovtA1MovwA2(uint16_t Value) { + uint32_t Imm4 = (Value >> 12) & 0x0f; + uint32_t Imm12 = Value & 0x0fff; + return (Imm4 << 16) | Imm12; +} + +/// Decode 16-bit immediate value for move instruction formats MOVT A1 and +/// MOVW A2. +/// +/// 000000000000:Imm4:0000:Imm12 -> Imm4:Imm12 +/// +uint16_t decodeImmMovtA1MovwA2(uint64_t Value) { + uint32_t Imm4 = (Value >> 16) & 0x0f; + uint32_t Imm12 = Value & 0x0fff; + return (Imm4 << 12) | Imm12; +} + +/// Encode register ID for instruction formats MOVT A1 and +/// MOVW A2. +/// +/// Rd4 -> 0000000000000000:Rd4:000000000000 +/// +uint32_t encodeRegMovtA1MovwA2(int64_t Value) { + uint32_t Rd4 = (Value & 0x00000f) << 12; + return Rd4; +} + +/// Decode register ID for instruction formats MOVT A1 and +/// MOVW A2. +/// +/// 0000000000000000:Rd4:000000000000 -> Rd4 +/// +int64_t decodeRegMovtA1MovwA2(uint64_t Value) { + uint32_t Rd4 = (Value >> 12) & 0x00000f; + return Rd4; +} + +namespace { + /// 32-bit Thumb instructions are stored as two little-endian halfwords. /// An instruction at address A encodes bytes A+1, A in the first halfword (Hi), /// followed by bytes A+3, A+2 in the second halfword (Lo). @@ -151,18 +219,126 @@ struct ThumbRelocation { const support::ulittle16_t &Lo; // Second halfword }; +struct WritableArmRelocation { + WritableArmRelocation(char *FixupPtr) + : Wd{*reinterpret_cast<support::ulittle32_t *>(FixupPtr)} {} + + support::ulittle32_t &Wd; +}; + +struct ArmRelocation { + ArmRelocation(const char *FixupPtr) + : Wd{*reinterpret_cast<const support::ulittle32_t *>(FixupPtr)} {} + + ArmRelocation(WritableArmRelocation &Writable) : Wd{Writable.Wd} {} + + const support::ulittle32_t &Wd; +}; + Error makeUnexpectedOpcodeError(const LinkGraph &G, const ThumbRelocation &R, Edge::Kind Kind) { return make_error<JITLinkError>( - formatv("Invalid opcode [ 0x{0:x4}, 0x{1:x4} ] for relocation: {2}", + formatv("Invalid opcode [ {0:x4}, {1:x4} ] for relocation: {2}", static_cast<uint16_t>(R.Hi), static_cast<uint16_t>(R.Lo), G.getEdgeKindName(Kind))); } -template <EdgeKind_aarch32 Kind> bool checkOpcode(const ThumbRelocation &R) { - uint16_t Hi = R.Hi & FixupInfo<Kind>::OpcodeMask.Hi; - uint16_t Lo = R.Lo & FixupInfo<Kind>::OpcodeMask.Lo; - return Hi == FixupInfo<Kind>::Opcode.Hi && Lo == FixupInfo<Kind>::Opcode.Lo; +Error makeUnexpectedOpcodeError(const LinkGraph &G, const ArmRelocation &R, + Edge::Kind Kind) { + return make_error<JITLinkError>( + formatv("Invalid opcode {0:x8} for relocation: {1}", + static_cast<uint32_t>(R.Wd), G.getEdgeKindName(Kind))); +} + +template <EdgeKind_aarch32 K> constexpr bool isArm() { + return FirstArmRelocation <= K && K <= LastArmRelocation; +} +template <EdgeKind_aarch32 K> constexpr bool isThumb() { + return FirstThumbRelocation <= K && K <= LastThumbRelocation; +} + +template <EdgeKind_aarch32 K> static bool checkOpcodeArm(uint32_t Wd) { + return (Wd & FixupInfo<K>::OpcodeMask) == FixupInfo<K>::Opcode; +} + +template <EdgeKind_aarch32 K> +static bool checkOpcodeThumb(uint16_t Hi, uint16_t Lo) { + return (Hi & FixupInfo<K>::OpcodeMask.Hi) == FixupInfo<K>::Opcode.Hi && + (Lo & FixupInfo<K>::OpcodeMask.Lo) == FixupInfo<K>::Opcode.Lo; +} + +class FixupInfoTable { + static constexpr size_t Items = LastRelocation + 1; + +public: + FixupInfoTable() { + populateEntries<FirstArmRelocation, LastArmRelocation>(); + populateEntries<FirstThumbRelocation, LastThumbRelocation>(); + } + + const FixupInfoBase *getEntry(Edge::Kind K) { + assert(K < Data.size() && "Index out of bounds"); + return Data.at(K).get(); + } + +private: + template <EdgeKind_aarch32 K, EdgeKind_aarch32 LastK> void populateEntries() { + assert(K < Data.size() && "Index out of range"); + assert(Data.at(K) == nullptr && "Initialized entries are immutable"); + Data[K] = initEntry<K>(); + if constexpr (K < LastK) { + constexpr auto Next = static_cast<EdgeKind_aarch32>(K + 1); + populateEntries<Next, LastK>(); + } + } + + template <EdgeKind_aarch32 K> + static std::unique_ptr<FixupInfoBase> initEntry() { + auto Entry = std::make_unique<FixupInfo<K>>(); + static_assert(isArm<K>() != isThumb<K>(), "Classes are mutually exclusive"); + if constexpr (isArm<K>()) + Entry->checkOpcode = checkOpcodeArm<K>; + if constexpr (isThumb<K>()) + Entry->checkOpcode = checkOpcodeThumb<K>; + return Entry; + } + +private: + std::array<std::unique_ptr<FixupInfoBase>, Items> Data; +}; + +ManagedStatic<FixupInfoTable> DynFixupInfos; + +} // namespace + +static Error checkOpcode(LinkGraph &G, const ArmRelocation &R, + Edge::Kind Kind) { + assert(Kind >= FirstArmRelocation && Kind <= LastArmRelocation && + "Edge kind must be Arm relocation"); + const FixupInfoBase *Entry = DynFixupInfos->getEntry(Kind); + const FixupInfoArm &Info = *static_cast<const FixupInfoArm *>(Entry); + assert(Info.checkOpcode && "Opcode check is mandatory for Arm edges"); + if (!Info.checkOpcode(R.Wd)) + return makeUnexpectedOpcodeError(G, R, Kind); + + return Error::success(); +} + +static Error checkOpcode(LinkGraph &G, const ThumbRelocation &R, + Edge::Kind Kind) { + assert(Kind >= FirstThumbRelocation && Kind <= LastThumbRelocation && + "Edge kind must be Thumb relocation"); + const FixupInfoBase *Entry = DynFixupInfos->getEntry(Kind); + const FixupInfoThumb &Info = *static_cast<const FixupInfoThumb *>(Entry); + assert(Info.checkOpcode && "Opcode check is mandatory for Thumb edges"); + if (!Info.checkOpcode(R.Hi, R.Lo)) + return makeUnexpectedOpcodeError(G, R, Kind); + + return Error::success(); +} + +const FixupInfoBase *FixupInfoBase::getDynFixupInfo(Edge::Kind K) { + return DynFixupInfos->getEntry(K); } template <EdgeKind_aarch32 Kind> @@ -173,30 +349,48 @@ bool checkRegister(const ThumbRelocation &R, HalfWords Reg) { } template <EdgeKind_aarch32 Kind> -bool writeRegister(WritableThumbRelocation &R, HalfWords Reg) { +bool checkRegister(const ArmRelocation &R, uint32_t Reg) { + uint32_t Wd = R.Wd & FixupInfo<Kind>::RegMask; + return Wd == Reg; +} + +template <EdgeKind_aarch32 Kind> +void writeRegister(WritableThumbRelocation &R, HalfWords Reg) { static constexpr HalfWords Mask = FixupInfo<Kind>::RegMask; - assert((Mask.Hi & Reg.Hi) == Reg.Hi && (Mask.Hi & Reg.Hi) == Reg.Hi && + assert((Mask.Hi & Reg.Hi) == Reg.Hi && (Mask.Lo & Reg.Lo) == Reg.Lo && "Value bits exceed bit range of given mask"); R.Hi = (R.Hi & ~Mask.Hi) | Reg.Hi; R.Lo = (R.Lo & ~Mask.Lo) | Reg.Lo; } template <EdgeKind_aarch32 Kind> +void writeRegister(WritableArmRelocation &R, uint32_t Reg) { + static constexpr uint32_t Mask = FixupInfo<Kind>::RegMask; + assert((Mask & Reg) == Reg && "Value bits exceed bit range of given mask"); + R.Wd = (R.Wd & ~Mask) | Reg; +} + +template <EdgeKind_aarch32 Kind> void writeImmediate(WritableThumbRelocation &R, HalfWords Imm) { static constexpr HalfWords Mask = FixupInfo<Kind>::ImmMask; - assert((Mask.Hi & Imm.Hi) == Imm.Hi && (Mask.Hi & Imm.Hi) == Imm.Hi && + assert((Mask.Hi & Imm.Hi) == Imm.Hi && (Mask.Lo & Imm.Lo) == Imm.Lo && "Value bits exceed bit range of given mask"); R.Hi = (R.Hi & ~Mask.Hi) | Imm.Hi; R.Lo = (R.Lo & ~Mask.Lo) | Imm.Lo; } -Expected<int64_t> readAddendData(LinkGraph &G, Block &B, const Edge &E) { - support::endianness Endian = G.getEndianness(); - assert(Endian != support::native && "Declare as little or big explicitly"); +template <EdgeKind_aarch32 Kind> +void writeImmediate(WritableArmRelocation &R, uint32_t Imm) { + static constexpr uint32_t Mask = FixupInfo<Kind>::ImmMask; + assert((Mask & Imm) == Imm && "Value bits exceed bit range of given mask"); + R.Wd = (R.Wd & ~Mask) | Imm; +} - Edge::Kind Kind = E.getKind(); +Expected<int64_t> readAddendData(LinkGraph &G, Block &B, Edge::OffsetT Offset, + Edge::Kind Kind) { + endianness Endian = G.getEndianness(); const char *BlockWorkingMem = B.getContent().data(); - const char *FixupPtr = BlockWorkingMem + E.getOffset(); + const char *FixupPtr = BlockWorkingMem + Offset; switch (Kind) { case Data_Delta32: @@ -206,59 +400,53 @@ Expected<int64_t> readAddendData(LinkGraph &G, Block &B, const Edge &E) { return make_error<JITLinkError>( "In graph " + G.getName() + ", section " + B.getSection().getName() + " can not read implicit addend for aarch32 edge kind " + - G.getEdgeKindName(E.getKind())); + G.getEdgeKindName(Kind)); } } -Expected<int64_t> readAddendArm(LinkGraph &G, Block &B, const Edge &E) { - Edge::Kind Kind = E.getKind(); +Expected<int64_t> readAddendArm(LinkGraph &G, Block &B, Edge::OffsetT Offset, + Edge::Kind Kind) { + ArmRelocation R(B.getContent().data() + Offset); + if (Error Err = checkOpcode(G, R, Kind)) + return std::move(Err); switch (Kind) { case Arm_Call: - return make_error<JITLinkError>( - "Addend extraction for relocation type not yet implemented: " + - StringRef(G.getEdgeKindName(Kind))); + case Arm_Jump24: + return decodeImmBA1BlA1BlxA2(R.Wd); + + case Arm_MovtAbs: + case Arm_MovwAbsNC: + return decodeImmMovtA1MovwA2(R.Wd); + default: return make_error<JITLinkError>( "In graph " + G.getName() + ", section " + B.getSection().getName() + " can not read implicit addend for aarch32 edge kind " + - G.getEdgeKindName(E.getKind())); + G.getEdgeKindName(Kind)); } } -Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, const Edge &E, - const ArmConfig &ArmCfg) { - ThumbRelocation R(B.getContent().data() + E.getOffset()); - Edge::Kind Kind = E.getKind(); +Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, Edge::OffsetT Offset, + Edge::Kind Kind, const ArmConfig &ArmCfg) { + ThumbRelocation R(B.getContent().data() + Offset); + if (Error Err = checkOpcode(G, R, Kind)) + return std::move(Err); switch (Kind) { case Thumb_Call: - if (!checkOpcode<Thumb_Call>(R)) - return makeUnexpectedOpcodeError(G, R, Kind); + case Thumb_Jump24: return LLVM_LIKELY(ArmCfg.J1J2BranchEncoding) ? decodeImmBT4BlT1BlxT2_J1J2(R.Hi, R.Lo) : decodeImmBT4BlT1BlxT2(R.Hi, R.Lo); - case Thumb_Jump24: - if (!checkOpcode<Thumb_Jump24>(R)) - return makeUnexpectedOpcodeError(G, R, Kind); - if (R.Lo & FixupInfo<Thumb_Jump24>::LoBitConditional) - return make_error<JITLinkError>("Relocation expects an unconditional " - "B.W branch instruction: " + - StringRef(G.getEdgeKindName(Kind))); - return LLVM_LIKELY(ArmCfg.J1J2BranchEncoding) - ? decodeImmBT4BlT1BlxT2_J1J2(R.Hi, R.Lo) - : decodeImmBT4BlT1BlxT2(R.Hi, R.Lo); - case Thumb_MovwAbsNC: - if (!checkOpcode<Thumb_MovwAbsNC>(R)) - return makeUnexpectedOpcodeError(G, R, Kind); + case Thumb_MovwPrelNC: // Initial addend is interpreted as a signed value return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo)); case Thumb_MovtAbs: - if (!checkOpcode<Thumb_MovtAbs>(R)) - return makeUnexpectedOpcodeError(G, R, Kind); + case Thumb_MovtPrel: // Initial addend is interpreted as a signed value return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo)); @@ -266,7 +454,7 @@ Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, const Edge &E, return make_error<JITLinkError>( "In graph " + G.getName() + ", section " + B.getSection().getName() + " can not read implicit addend for aarch32 edge kind " + - G.getEdgeKindName(E.getKind())); + G.getEdgeKindName(Kind)); } } @@ -277,13 +465,12 @@ Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) { char *FixupPtr = BlockWorkingMem + E.getOffset(); auto Write32 = [FixupPtr, Endian = G.getEndianness()](int64_t Value) { - assert(Endian != native && "Must be explicit: little or big"); assert(isInt<32>(Value) && "Must be in signed 32-bit range"); uint32_t Imm = static_cast<int32_t>(Value); - if (LLVM_LIKELY(Endian == little)) - endian::write32<little>(FixupPtr, Imm); + if (LLVM_LIKELY(Endian == endianness::little)) + endian::write32<endianness::little>(FixupPtr, Imm); else - endian::write32<big>(FixupPtr, Imm); + endian::write32<endianness::big>(FixupPtr, Imm); }; Edge::Kind Kind = E.getKind(); @@ -291,7 +478,6 @@ Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) { int64_t Addend = E.getAddend(); Symbol &TargetSymbol = E.getTarget(); uint64_t TargetAddress = TargetSymbol.getAddress().getValue(); - assert(!TargetSymbol.hasTargetFlags(ThumbSymbol)); // Regular data relocations have size 4, alignment 1 and write the full 32-bit // result to the place; no need for overflow checking. There are three @@ -320,13 +506,71 @@ Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) { } Error applyFixupArm(LinkGraph &G, Block &B, const Edge &E) { + WritableArmRelocation R(B.getAlreadyMutableContent().data() + E.getOffset()); Edge::Kind Kind = E.getKind(); + if (Error Err = checkOpcode(G, R, Kind)) + return Err; + + uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue(); + int64_t Addend = E.getAddend(); + Symbol &TargetSymbol = E.getTarget(); + uint64_t TargetAddress = TargetSymbol.getAddress().getValue(); switch (Kind) { - case Arm_Call: - return make_error<JITLinkError>( - "Fix-up for relocation type not yet implemented: " + - StringRef(G.getEdgeKindName(Kind))); + case Arm_Jump24: { + if (hasTargetFlags(TargetSymbol, ThumbSymbol)) + return make_error<JITLinkError>("Branch relocation needs interworking " + "stub when bridging to Thumb: " + + StringRef(G.getEdgeKindName(Kind))); + + int64_t Value = TargetAddress - FixupAddress + Addend; + + if (!isInt<26>(Value)) + return makeTargetOutOfRangeError(G, B, E); + writeImmediate<Arm_Jump24>(R, encodeImmBA1BlA1BlxA2(Value)); + + return Error::success(); + } + case Arm_Call: { + if ((R.Wd & FixupInfo<Arm_Call>::CondMask) != + FixupInfo<Arm_Call>::Unconditional) + return make_error<JITLinkError>("Relocation expects an unconditional " + "BL/BLX branch instruction: " + + StringRef(G.getEdgeKindName(Kind))); + + int64_t Value = TargetAddress - FixupAddress + Addend; + + // The call instruction itself is Arm. The call destination can either be + // Thumb or Arm. We use BL to stay in Arm and BLX to change to Thumb. + bool TargetIsThumb = hasTargetFlags(TargetSymbol, ThumbSymbol); + bool InstrIsBlx = (~R.Wd & FixupInfo<Arm_Call>::BitBlx) == 0; + if (TargetIsThumb != InstrIsBlx) { + if (LLVM_LIKELY(TargetIsThumb)) { + // Change opcode BL -> BLX + R.Wd = R.Wd | FixupInfo<Arm_Call>::BitBlx; + R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitH; + } else { + // Change opcode BLX -> BL + R.Wd = R.Wd & ~FixupInfo<Arm_Call>::BitBlx; + } + } + + if (!isInt<26>(Value)) + return makeTargetOutOfRangeError(G, B, E); + writeImmediate<Arm_Call>(R, encodeImmBA1BlA1BlxA2(Value)); + + return Error::success(); + } + case Arm_MovwAbsNC: { + uint16_t Value = (TargetAddress + Addend) & 0xffff; + writeImmediate<Arm_MovwAbsNC>(R, encodeImmMovtA1MovwA2(Value)); + return Error::success(); + } + case Arm_MovtAbs: { + uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff; + writeImmediate<Arm_MovtAbs>(R, encodeImmMovtA1MovwA2(Value)); + return Error::success(); + } default: return make_error<JITLinkError>( "In graph " + G.getName() + ", section " + B.getSection().getName() + @@ -339,24 +583,18 @@ Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E, const ArmConfig &ArmCfg) { WritableThumbRelocation R(B.getAlreadyMutableContent().data() + E.getOffset()); - Edge::Kind Kind = E.getKind(); + if (Error Err = checkOpcode(G, R, Kind)) + return Err; + uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue(); int64_t Addend = E.getAddend(); Symbol &TargetSymbol = E.getTarget(); uint64_t TargetAddress = TargetSymbol.getAddress().getValue(); - if (TargetSymbol.hasTargetFlags(ThumbSymbol)) - TargetAddress |= 0x01; switch (Kind) { case Thumb_Jump24: { - if (!checkOpcode<Thumb_Jump24>(R)) - return makeUnexpectedOpcodeError(G, R, Kind); - if (R.Lo & FixupInfo<Thumb_Jump24>::LoBitConditional) - return make_error<JITLinkError>("Relocation expects an unconditional " - "B.W branch instruction: " + - StringRef(G.getEdgeKindName(Kind))); - if (!(TargetSymbol.hasTargetFlags(ThumbSymbol))) + if (!hasTargetFlags(TargetSymbol, ThumbSymbol)) return make_error<JITLinkError>("Branch relocation needs interworking " "stub when bridging to ARM: " + StringRef(G.getEdgeKindName(Kind))); @@ -376,27 +614,22 @@ Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E, } case Thumb_Call: { - if (!checkOpcode<Thumb_Call>(R)) - return makeUnexpectedOpcodeError(G, R, Kind); - int64_t Value = TargetAddress - FixupAddress + Addend; // The call instruction itself is Thumb. The call destination can either be // Thumb or Arm. We use BL to stay in Thumb and BLX to change to Arm. - bool TargetIsArm = !TargetSymbol.hasTargetFlags(ThumbSymbol); + bool TargetIsArm = !hasTargetFlags(TargetSymbol, ThumbSymbol); bool InstrIsBlx = (R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) == 0; if (TargetIsArm != InstrIsBlx) { if (LLVM_LIKELY(TargetIsArm)) { - // Change opcode BL -> BLX and fix range value (account for 4-byte + // Change opcode BL -> BLX and fix range value: account for 4-byte // aligned destination while instruction may only be 2-byte aligned - // and clear Thumb bit). R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx; R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitH; Value = alignTo(Value, 4); } else { - // Change opcode BLX -> BL and set Thumb bit + // Change opcode BLX -> BL R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx; - Value |= 0x01; } } @@ -417,20 +650,25 @@ Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E, } case Thumb_MovwAbsNC: { - if (!checkOpcode<Thumb_MovwAbsNC>(R)) - return makeUnexpectedOpcodeError(G, R, Kind); uint16_t Value = (TargetAddress + Addend) & 0xffff; writeImmediate<Thumb_MovwAbsNC>(R, encodeImmMovtT1MovwT3(Value)); return Error::success(); } - case Thumb_MovtAbs: { - if (!checkOpcode<Thumb_MovtAbs>(R)) - return makeUnexpectedOpcodeError(G, R, Kind); uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff; writeImmediate<Thumb_MovtAbs>(R, encodeImmMovtT1MovwT3(Value)); return Error::success(); } + case Thumb_MovwPrelNC: { + uint16_t Value = ((TargetAddress + Addend - FixupAddress) & 0xffff); + writeImmediate<Thumb_MovwPrelNC>(R, encodeImmMovtT1MovwT3(Value)); + return Error::success(); + } + case Thumb_MovtPrel: { + uint16_t Value = (((TargetAddress + Addend - FixupAddress) >> 16) & 0xffff); + writeImmediate<Thumb_MovtPrel>(R, encodeImmMovtT1MovwT3(Value)); + return Error::success(); + } default: return make_error<JITLinkError>( @@ -471,11 +709,17 @@ const char *getEdgeKindName(Edge::Kind K) { switch (K) { KIND_NAME_CASE(Data_Delta32) + KIND_NAME_CASE(Data_Pointer32) KIND_NAME_CASE(Arm_Call) + KIND_NAME_CASE(Arm_Jump24) + KIND_NAME_CASE(Arm_MovwAbsNC) + KIND_NAME_CASE(Arm_MovtAbs) KIND_NAME_CASE(Thumb_Call) KIND_NAME_CASE(Thumb_Jump24) KIND_NAME_CASE(Thumb_MovwAbsNC) KIND_NAME_CASE(Thumb_MovtAbs) + KIND_NAME_CASE(Thumb_MovwPrelNC) + KIND_NAME_CASE(Thumb_MovtPrel) default: return getGenericEdgeKindName(K); } diff --git a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp index 4e21eace21d0..27484aaf2059 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp @@ -64,8 +64,36 @@ const char *getEdgeKindName(Edge::Kind K) { return "Pointer64"; case Pointer32: return "Pointer32"; + case Pointer16: + return "Pointer16"; + case Pointer16DS: + return "Pointer16DS"; + case Pointer16HA: + return "Pointer16HA"; + case Pointer16HI: + return "Pointer16HI"; + case Pointer16HIGH: + return "Pointer16HIGH"; + case Pointer16HIGHA: + return "Pointer16HIGHA"; + case Pointer16HIGHER: + return "Pointer16HIGHER"; + case Pointer16HIGHERA: + return "Pointer16HIGHERA"; + case Pointer16HIGHEST: + return "Pointer16HIGHEST"; + case Pointer16HIGHESTA: + return "Pointer16HIGHESTA"; + case Pointer16LO: + return "Pointer16LO"; + case Pointer16LODS: + return "Pointer16LODS"; + case Pointer14: + return "Pointer14"; case Delta64: return "Delta64"; + case Delta34: + return "Delta34"; case Delta32: return "Delta32"; case NegDelta32: @@ -74,26 +102,40 @@ const char *getEdgeKindName(Edge::Kind K) { return "Delta16"; case Delta16HA: return "Delta16HA"; + case Delta16HI: + return "Delta16HI"; case Delta16LO: return "Delta16LO"; + case TOC: + return "TOC"; + case TOCDelta16: + return "TOCDelta16"; + case TOCDelta16DS: + return "TOCDelta16DS"; case TOCDelta16HA: return "TOCDelta16HA"; + case TOCDelta16HI: + return "TOCDelta16HI"; case TOCDelta16LO: return "TOCDelta16LO"; - case TOCDelta16DS: - return "TOCDelta16DS"; case TOCDelta16LODS: return "TOCDelta16LODS"; + case RequestGOTAndTransformToDelta34: + return "RequestGOTAndTransformToDelta34"; case CallBranchDelta: return "CallBranchDelta"; case CallBranchDeltaRestoreTOC: return "CallBranchDeltaRestoreTOC"; - case RequestPLTCallStub: - return "RequestPLTCallStub"; - case RequestPLTCallStubSaveTOC: - return "RequestPLTCallStubSaveTOC"; - case RequestPLTCallStubNoTOC: - return "RequestPLTCallStubNoTOC"; + case RequestCall: + return "RequestCall"; + case RequestCallNoTOC: + return "RequestCallNoTOC"; + case RequestTLSDescInGOTAndTransformToTOCDelta16HA: + return "RequestTLSDescInGOTAndTransformToTOCDelta16HA"; + case RequestTLSDescInGOTAndTransformToTOCDelta16LO: + return "RequestTLSDescInGOTAndTransformToTOCDelta16LO"; + case RequestTLSDescInGOTAndTransformToDelta34: + return "RequestTLSDescInGOTAndTransformToDelta34"; default: return getGenericEdgeKindName(static_cast<Edge::Kind>(K)); } diff --git a/llvm/lib/ExecutionEngine/JITLink/riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/riscv.cpp index a78843b16147..a4e4daef97fb 100644 --- a/llvm/lib/ExecutionEngine/JITLink/riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/riscv.cpp @@ -82,6 +82,8 @@ const char *getEdgeKindName(Edge::Kind K) { return "CallRelaxable"; case AlignRelaxable: return "AlignRelaxable"; + case NegDelta32: + return "NegDelta32"; } return getGenericEdgeKindName(K); } diff --git a/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp index 7c869bead0b0..c8f5a99099ea 100644 --- a/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp @@ -54,13 +54,13 @@ public: void materialize(std::unique_ptr<MaterializationResponsibility> R) override { unsigned PointerSize; - support::endianness Endianness; + llvm::endianness Endianness; const auto &TT = CP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::x86_64: PointerSize = 8; - Endianness = support::endianness::little; + Endianness = llvm::endianness::little; break; default: llvm_unreachable("Unrecognized architecture"); diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index 0c23f2b25219..56838e9bc86d 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -31,6 +31,7 @@ char SymbolsCouldNotBeRemoved::ID = 0; char MissingSymbolDefinitions::ID = 0; char UnexpectedSymbolDefinitions::ID = 0; char MaterializationTask::ID = 0; +char LookupTask::ID = 0; RegisterDependenciesFunction NoDependenciesToRegister = RegisterDependenciesFunction(); @@ -348,7 +349,7 @@ void ReExportsMaterializationUnit::materialize( } } - // The OnResolveInfo struct will hold the aliases and responsibilty for each + // The OnResolveInfo struct will hold the aliases and responsibility for each // query in the list. struct OnResolveInfo { OnResolveInfo(std::unique_ptr<MaterializationResponsibility> R, @@ -529,11 +530,16 @@ public: SymbolLookupSet LookupSet; SymbolState RequiredState; - std::unique_lock<std::mutex> GeneratorLock; size_t CurSearchOrderIndex = 0; bool NewJITDylib = true; SymbolLookupSet DefGeneratorCandidates; SymbolLookupSet DefGeneratorNonCandidates; + + enum { + NotInGenerator, // Not currently using a generator. + ResumedForGenerator, // Resumed after being auto-suspended before generator. + InGenerator // Currently using generator. + } GenState = NotInGenerator; std::vector<std::weak_ptr<DefinitionGenerator>> CurDefGeneratorStack; }; @@ -547,15 +553,11 @@ public: OnComplete(std::move(OnComplete)) {} void complete(std::unique_ptr<InProgressLookupState> IPLS) override { - GeneratorLock = {}; // Unlock and release. auto &ES = SearchOrder.front().first->getExecutionSession(); ES.OL_completeLookupFlags(std::move(IPLS), std::move(OnComplete)); } - void fail(Error Err) override { - GeneratorLock = {}; // Unlock and release. - OnComplete(std::move(Err)); - } + void fail(Error Err) override { OnComplete(std::move(Err)); } private: unique_function<void(Expected<SymbolFlagsMap>)> OnComplete; @@ -574,14 +576,12 @@ public: } void complete(std::unique_ptr<InProgressLookupState> IPLS) override { - GeneratorLock = {}; // Unlock and release. auto &ES = SearchOrder.front().first->getExecutionSession(); ES.OL_completeLookup(std::move(IPLS), std::move(Q), std::move(RegisterDependencies)); } void fail(Error Err) override { - GeneratorLock = {}; Q->detach(); Q->handleFailed(std::move(Err)); } @@ -638,7 +638,19 @@ void LookupState::continueLookup(Error Err) { ES.OL_applyQueryPhase1(std::move(IPLS), std::move(Err)); } -DefinitionGenerator::~DefinitionGenerator() = default; +DefinitionGenerator::~DefinitionGenerator() { + std::deque<LookupState> LookupsToFail; + { + std::lock_guard<std::mutex> Lock(M); + std::swap(PendingLookups, LookupsToFail); + InUse = false; + } + + for (auto &LS : LookupsToFail) + LS.continueLookup(make_error<StringError>( + "Query waiting on DefinitionGenerator that was destroyed", + inconvertibleErrorCode())); +} JITDylib::~JITDylib() { LLVM_DEBUG(dbgs() << "Destroying JITDylib " << getName() << "\n"); @@ -677,6 +689,10 @@ ResourceTrackerSP JITDylib::createResourceTracker() { } void JITDylib::removeGenerator(DefinitionGenerator &G) { + // DefGenerator moved into TmpDG to ensure that it's destroyed outside the + // session lock (since it may have to send errors to pending queries). + std::shared_ptr<DefinitionGenerator> TmpDG; + ES.runSessionLocked([&] { assert(State == Open && "JD is defunct"); auto I = llvm::find_if(DefGenerators, @@ -684,6 +700,7 @@ void JITDylib::removeGenerator(DefinitionGenerator &G) { return H.get() == &G; }); assert(I != DefGenerators.end() && "Generator not found"); + TmpDG = std::move(*I); DefGenerators.erase(I); }); } @@ -1336,7 +1353,7 @@ void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) { ES.runSessionLocked([&]() { for (auto &KV : NewLinks) { // Skip elements of NewLinks that are already in the link order. - if (llvm::find(LinkOrder, KV) != LinkOrder.end()) + if (llvm::is_contained(LinkOrder, KV)) continue; LinkOrder.push_back(std::move(KV)); @@ -1903,6 +1920,10 @@ void MaterializationTask::printDescription(raw_ostream &OS) { void MaterializationTask::run() { MU->materialize(std::move(MR)); } +void LookupTask::printDescription(raw_ostream &OS) { OS << "Lookup task"; } + +void LookupTask::run() { LS.continueLookup(Error::success()); } + ExecutionSession::ExecutionSession(std::unique_ptr<ExecutorProcessControl> EPC) : EPC(std::move(EPC)) { // Associated EPC and this. @@ -1918,16 +1939,14 @@ ExecutionSession::~ExecutionSession() { Error ExecutionSession::endSession() { LLVM_DEBUG(dbgs() << "Ending ExecutionSession " << this << "\n"); - std::vector<JITDylibSP> JITDylibsToClose = runSessionLocked([&] { + auto JDsToRemove = runSessionLocked([&] { SessionOpen = false; - return std::move(JDs); + return JDs; }); - // TODO: notifiy platform? run static deinits? + std::reverse(JDsToRemove.begin(), JDsToRemove.end()); - Error Err = Error::success(); - for (auto &JD : reverse(JITDylibsToClose)) - Err = joinErrors(std::move(Err), JD->clear()); + auto Err = removeJITDylibs(std::move(JDsToRemove)); Err = joinErrors(std::move(Err), EPC->disconnect()); @@ -1977,42 +1996,44 @@ Expected<JITDylib &> ExecutionSession::createJITDylib(std::string Name) { return JD; } -Error ExecutionSession::removeJITDylib(JITDylib &JD) { - // Keep JD alive throughout this routine, even if all other references - // have been dropped. - JITDylibSP JDKeepAlive = &JD; +Error ExecutionSession::removeJITDylibs(std::vector<JITDylibSP> JDsToRemove) { // Set JD to 'Closing' state and remove JD from the ExecutionSession. runSessionLocked([&] { - assert(JD.State == JITDylib::Open && "JD already closed"); - JD.State = JITDylib::Closing; - auto I = llvm::find(JDs, &JD); - assert(I != JDs.end() && "JD does not appear in session JDs"); - JDs.erase(I); + for (auto &JD : JDsToRemove) { + assert(JD->State == JITDylib::Open && "JD already closed"); + JD->State = JITDylib::Closing; + auto I = llvm::find(JDs, JD); + assert(I != JDs.end() && "JD does not appear in session JDs"); + JDs.erase(I); + } }); - // Clear the JITDylib. Hold on to any error while we clean up the - // JITDylib members below. - auto Err = JD.clear(); - - // Notify the platform of the teardown. - if (P) - Err = joinErrors(std::move(Err), P->teardownJITDylib(JD)); + // Clear JITDylibs and notify the platform. + Error Err = Error::success(); + for (auto JD : JDsToRemove) { + Err = joinErrors(std::move(Err), JD->clear()); + if (P) + Err = joinErrors(std::move(Err), P->teardownJITDylib(*JD)); + } // Set JD to closed state. Clear remaining data structures. runSessionLocked([&] { - assert(JD.State == JITDylib::Closing && "JD should be closing"); - JD.State = JITDylib::Closed; - assert(JD.Symbols.empty() && "JD.Symbols is not empty after clear"); - assert(JD.UnmaterializedInfos.empty() && - "JD.UnmaterializedInfos is not empty after clear"); - assert(JD.MaterializingInfos.empty() && - "JD.MaterializingInfos is not empty after clear"); - assert(JD.TrackerSymbols.empty() && - "TrackerSymbols is not empty after clear"); - JD.DefGenerators.clear(); - JD.LinkOrder.clear(); + for (auto &JD : JDsToRemove) { + assert(JD->State == JITDylib::Closing && "JD should be closing"); + JD->State = JITDylib::Closed; + assert(JD->Symbols.empty() && "JD.Symbols is not empty after clear"); + assert(JD->UnmaterializedInfos.empty() && + "JD.UnmaterializedInfos is not empty after clear"); + assert(JD->MaterializingInfos.empty() && + "JD.MaterializingInfos is not empty after clear"); + assert(JD->TrackerSymbols.empty() && + "TrackerSymbols is not empty after clear"); + JD->DefGenerators.clear(); + JD->LinkOrder.clear(); + } }); + return Err; } @@ -2406,6 +2427,37 @@ Error ExecutionSession::IL_updateCandidatesFor( }); } +void ExecutionSession::OL_resumeLookupAfterGeneration( + InProgressLookupState &IPLS) { + + assert(IPLS.GenState != InProgressLookupState::NotInGenerator && + "Should not be called for not-in-generator lookups"); + IPLS.GenState = InProgressLookupState::NotInGenerator; + + LookupState LS; + + if (auto DG = IPLS.CurDefGeneratorStack.back().lock()) { + IPLS.CurDefGeneratorStack.pop_back(); + std::lock_guard<std::mutex> Lock(DG->M); + + // If there are no pending lookups then mark the generator as free and + // return. + if (DG->PendingLookups.empty()) { + DG->InUse = false; + return; + } + + // Otherwise resume the next lookup. + LS = std::move(DG->PendingLookups.front()); + DG->PendingLookups.pop_front(); + } + + if (LS.IPLS) { + LS.IPLS->GenState = InProgressLookupState::ResumedForGenerator; + dispatchTask(std::make_unique<LookupTask>(std::move(LS))); + } +} + void ExecutionSession::OL_applyQueryPhase1( std::unique_ptr<InProgressLookupState> IPLS, Error Err) { @@ -2422,6 +2474,12 @@ void ExecutionSession::OL_applyQueryPhase1( << IPLS->DefGeneratorNonCandidates << "\n"; }); + if (IPLS->GenState == InProgressLookupState::InGenerator) + OL_resumeLookupAfterGeneration(*IPLS); + + assert(IPLS->GenState != InProgressLookupState::InGenerator && + "Lookup should not be in InGenerator state here"); + // FIXME: We should attach the query as we go: This provides a result in a // single pass in the common case where all symbols have already reached the // required state. The query could be detached again in the 'fail' method on @@ -2447,10 +2505,6 @@ void ExecutionSession::OL_applyQueryPhase1( // If we've just reached a new JITDylib then perform some setup. if (IPLS->NewJITDylib) { - - // Acquire the generator lock for this JITDylib. - IPLS->GeneratorLock = std::unique_lock<std::mutex>(JD.GeneratorsMutex); - // Add any non-candidates from the last JITDylib (if any) back on to the // list of definition candidates for this JITDylib, reset definition // non-candidates to the empty set. @@ -2488,6 +2542,13 @@ void ExecutionSession::OL_applyQueryPhase1( dbgs() << " Remaining candidates = " << IPLS->DefGeneratorCandidates << "\n"; }); + + // If this lookup was resumed after auto-suspension but all candidates + // have already been generated (by some previous call to the generator) + // treat the lookup as if it had completed generation. + if (IPLS->GenState == InProgressLookupState::ResumedForGenerator && + IPLS->DefGeneratorCandidates.empty()) + OL_resumeLookupAfterGeneration(*IPLS); }); // If we encountered an error while filtering generation candidates then @@ -2509,13 +2570,32 @@ void ExecutionSession::OL_applyQueryPhase1( while (!IPLS->CurDefGeneratorStack.empty() && !IPLS->DefGeneratorCandidates.empty()) { auto DG = IPLS->CurDefGeneratorStack.back().lock(); - IPLS->CurDefGeneratorStack.pop_back(); if (!DG) return IPLS->fail(make_error<StringError>( "DefinitionGenerator removed while lookup in progress", inconvertibleErrorCode())); + // At this point the lookup is in either the NotInGenerator state, or in + // the ResumedForGenerator state. + // If this lookup is in the NotInGenerator state then check whether the + // generator is in use. If the generator is not in use then move the + // lookup to the InGenerator state and continue. If the generator is + // already in use then just add this lookup to the pending lookups list + // and bail out. + // If this lookup is in the ResumedForGenerator state then just move it + // to InGenerator and continue. + if (IPLS->GenState == InProgressLookupState::NotInGenerator) { + std::lock_guard<std::mutex> Lock(DG->M); + if (DG->InUse) { + DG->PendingLookups.push_back(std::move(IPLS)); + return; + } + DG->InUse = true; + } + + IPLS->GenState = InProgressLookupState::InGenerator; + auto K = IPLS->K; auto &LookupSet = IPLS->DefGeneratorCandidates; @@ -2528,6 +2608,11 @@ void ExecutionSession::OL_applyQueryPhase1( IPLS = std::move(LS.IPLS); } + // If the lookup returned then pop the generator stack and unblock the + // next lookup on this generator (if any). + if (IPLS) + OL_resumeLookupAfterGeneration(*IPLS); + // If there was an error then fail the query. if (Err) { LLVM_DEBUG({ @@ -2677,7 +2762,7 @@ void ExecutionSession::OL_completeLookup( // Otherwise this is a match. - // If this symbol is already in the requried state then notify the + // If this symbol is already in the required state then notify the // query, remove the symbol and continue. if (SymI->second.getState() >= Q->getRequiredState()) { LLVM_DEBUG(dbgs() diff --git a/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp deleted file mode 100644 index 830582bb3649..000000000000 --- a/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp +++ /dev/null @@ -1,472 +0,0 @@ -//===------- DebuggerSupportPlugin.cpp - Utils for debugger support -------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h" - -#include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringSet.h" -#include "llvm/BinaryFormat/MachO.h" - -#define DEBUG_TYPE "orc" - -using namespace llvm; -using namespace llvm::jitlink; -using namespace llvm::orc; - -static const char *SynthDebugSectionName = "__jitlink_synth_debug_object"; - -namespace { - -struct MachO64LE { - using UIntPtr = uint64_t; - - using Header = MachO::mach_header_64; - using SegmentLC = MachO::segment_command_64; - using Section = MachO::section_64; - using NList = MachO::nlist_64; - - static constexpr support::endianness Endianness = support::little; - static constexpr const uint32_t Magic = MachO::MH_MAGIC_64; - static constexpr const uint32_t SegmentCmd = MachO::LC_SEGMENT_64; -}; - -class MachODebugObjectSynthesizerBase - : public GDBJITDebugInfoRegistrationPlugin::DebugSectionSynthesizer { -public: - static bool isDebugSection(Section &Sec) { - return Sec.getName().startswith("__DWARF,"); - } - - MachODebugObjectSynthesizerBase(LinkGraph &G, ExecutorAddr RegisterActionAddr) - : G(G), RegisterActionAddr(RegisterActionAddr) {} - virtual ~MachODebugObjectSynthesizerBase() = default; - - Error preserveDebugSections() { - if (G.findSectionByName(SynthDebugSectionName)) { - LLVM_DEBUG({ - dbgs() << "MachODebugObjectSynthesizer skipping graph " << G.getName() - << " which contains an unexpected existing " - << SynthDebugSectionName << " section.\n"; - }); - return Error::success(); - } - - LLVM_DEBUG({ - dbgs() << "MachODebugObjectSynthesizer visiting graph " << G.getName() - << "\n"; - }); - for (auto &Sec : G.sections()) { - if (!isDebugSection(Sec)) - continue; - // Preserve blocks in this debug section by marking one existing symbol - // live for each block, and introducing a new live, anonymous symbol for - // each currently unreferenced block. - LLVM_DEBUG({ - dbgs() << " Preserving debug section " << Sec.getName() << "\n"; - }); - SmallSet<Block *, 8> PreservedBlocks; - for (auto *Sym : Sec.symbols()) { - bool NewPreservedBlock = - PreservedBlocks.insert(&Sym->getBlock()).second; - if (NewPreservedBlock) - Sym->setLive(true); - } - for (auto *B : Sec.blocks()) - if (!PreservedBlocks.count(B)) - G.addAnonymousSymbol(*B, 0, 0, false, true); - } - return Error::success(); - } - -protected: - LinkGraph &G; - ExecutorAddr RegisterActionAddr; -}; - -template <typename MachOTraits> -class MachODebugObjectSynthesizer : public MachODebugObjectSynthesizerBase { -private: - class MachOStructWriter { - public: - MachOStructWriter(MutableArrayRef<char> Buffer) : Buffer(Buffer) {} - - size_t getOffset() const { return Offset; } - - template <typename MachOStruct> void write(MachOStruct S) { - assert(Offset + sizeof(S) <= Buffer.size() && - "Container block overflow while constructing debug MachO"); - if (MachOTraits::Endianness != support::endian::system_endianness()) - MachO::swapStruct(S); - memcpy(Buffer.data() + Offset, &S, sizeof(S)); - Offset += sizeof(S); - } - - private: - MutableArrayRef<char> Buffer; - size_t Offset = 0; - }; - -public: - using MachODebugObjectSynthesizerBase::MachODebugObjectSynthesizerBase; - - Error startSynthesis() override { - LLVM_DEBUG({ - dbgs() << "Creating " << SynthDebugSectionName << " for " << G.getName() - << "\n"; - }); - auto &SDOSec = G.createSection(SynthDebugSectionName, MemProt::Read); - - struct DebugSectionInfo { - Section *Sec = nullptr; - StringRef SegName; - StringRef SecName; - uint64_t Alignment = 0; - orc::ExecutorAddr StartAddr; - uint64_t Size = 0; - }; - - SmallVector<DebugSectionInfo, 12> DebugSecInfos; - size_t NumSections = 0; - for (auto &Sec : G.sections()) { - if (Sec.blocks().empty()) - continue; - - ++NumSections; - if (isDebugSection(Sec)) { - size_t SepPos = Sec.getName().find(','); - if (SepPos > 16 || (Sec.getName().size() - (SepPos + 1) > 16)) { - LLVM_DEBUG({ - dbgs() << "Skipping debug object synthesis for graph " - << G.getName() - << ": encountered non-standard DWARF section name \"" - << Sec.getName() << "\"\n"; - }); - return Error::success(); - } - DebugSecInfos.push_back({&Sec, Sec.getName().substr(0, SepPos), - Sec.getName().substr(SepPos + 1), 0, - orc::ExecutorAddr(), 0}); - } else { - NonDebugSections.push_back(&Sec); - - // If the first block in the section has a non-zero alignment offset - // then we need to add a padding block, since the section command in - // the header doesn't allow for aligment offsets. - SectionRange R(Sec); - if (!R.empty()) { - auto &FB = *R.getFirstBlock(); - if (FB.getAlignmentOffset() != 0) { - auto Padding = G.allocateBuffer(FB.getAlignmentOffset()); - memset(Padding.data(), 0, Padding.size()); - G.createContentBlock(Sec, Padding, - FB.getAddress() - FB.getAlignmentOffset(), - FB.getAlignment(), 0); - } - } - } - } - - // Create container block. - size_t SectionsCmdSize = - sizeof(typename MachOTraits::Section) * NumSections; - size_t SegmentLCSize = - sizeof(typename MachOTraits::SegmentLC) + SectionsCmdSize; - size_t ContainerBlockSize = - sizeof(typename MachOTraits::Header) + SegmentLCSize; - auto ContainerBlockContent = G.allocateBuffer(ContainerBlockSize); - MachOContainerBlock = &G.createMutableContentBlock( - SDOSec, ContainerBlockContent, orc::ExecutorAddr(), 8, 0); - - // Copy debug section blocks and symbols. - orc::ExecutorAddr NextBlockAddr(MachOContainerBlock->getSize()); - for (auto &SI : DebugSecInfos) { - assert(!SI.Sec->blocks().empty() && "Empty debug info section?"); - - // Update addresses in debug section. - LLVM_DEBUG({ - dbgs() << " Appending " << SI.Sec->getName() << " (" - << SI.Sec->blocks_size() << " block(s)) at " - << formatv("{0:x8}", NextBlockAddr) << "\n"; - }); - for (auto *B : SI.Sec->blocks()) { - NextBlockAddr = alignToBlock(NextBlockAddr, *B); - B->setAddress(NextBlockAddr); - NextBlockAddr += B->getSize(); - } - - auto &FirstBlock = **SI.Sec->blocks().begin(); - if (FirstBlock.getAlignmentOffset() != 0) - return make_error<StringError>( - "First block in " + SI.Sec->getName() + - " section has non-zero alignment offset", - inconvertibleErrorCode()); - if (FirstBlock.getAlignment() > std::numeric_limits<uint32_t>::max()) - return make_error<StringError>("First block in " + SI.Sec->getName() + - " has alignment >4Gb", - inconvertibleErrorCode()); - - SI.Alignment = FirstBlock.getAlignment(); - SI.StartAddr = FirstBlock.getAddress(); - SI.Size = NextBlockAddr - SI.StartAddr; - G.mergeSections(SDOSec, *SI.Sec); - SI.Sec = nullptr; - } - size_t DebugSectionsSize = - NextBlockAddr - orc::ExecutorAddr(MachOContainerBlock->getSize()); - - // Write MachO header and debug section load commands. - MachOStructWriter Writer(MachOContainerBlock->getAlreadyMutableContent()); - typename MachOTraits::Header Hdr; - memset(&Hdr, 0, sizeof(Hdr)); - Hdr.magic = MachOTraits::Magic; - switch (G.getTargetTriple().getArch()) { - case Triple::x86_64: - Hdr.cputype = MachO::CPU_TYPE_X86_64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; - break; - case Triple::aarch64: - Hdr.cputype = MachO::CPU_TYPE_ARM64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; - break; - default: - llvm_unreachable("Unsupported architecture"); - } - Hdr.filetype = MachO::MH_OBJECT; - Hdr.ncmds = 1; - Hdr.sizeofcmds = SegmentLCSize; - Hdr.flags = 0; - Writer.write(Hdr); - - typename MachOTraits::SegmentLC SegLC; - memset(&SegLC, 0, sizeof(SegLC)); - SegLC.cmd = MachOTraits::SegmentCmd; - SegLC.cmdsize = SegmentLCSize; - SegLC.vmaddr = ContainerBlockSize; - SegLC.vmsize = DebugSectionsSize; - SegLC.fileoff = ContainerBlockSize; - SegLC.filesize = DebugSectionsSize; - SegLC.maxprot = - MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE; - SegLC.initprot = - MachO::VM_PROT_READ | MachO::VM_PROT_WRITE | MachO::VM_PROT_EXECUTE; - SegLC.nsects = NumSections; - SegLC.flags = 0; - Writer.write(SegLC); - - StringSet<> ExistingLongNames; - for (auto &SI : DebugSecInfos) { - typename MachOTraits::Section Sec; - memset(&Sec, 0, sizeof(Sec)); - memcpy(Sec.sectname, SI.SecName.data(), SI.SecName.size()); - memcpy(Sec.segname, SI.SegName.data(), SI.SegName.size()); - Sec.addr = SI.StartAddr.getValue(); - Sec.size = SI.Size; - Sec.offset = SI.StartAddr.getValue(); - Sec.align = SI.Alignment; - Sec.reloff = 0; - Sec.nreloc = 0; - Sec.flags = MachO::S_ATTR_DEBUG; - Writer.write(Sec); - } - - // Set MachOContainerBlock to indicate success to - // completeSynthesisAndRegister. - NonDebugSectionsStart = Writer.getOffset(); - return Error::success(); - } - - Error completeSynthesisAndRegister() override { - if (!MachOContainerBlock) { - LLVM_DEBUG({ - dbgs() << "Not writing MachO debug object header for " << G.getName() - << " since createDebugSection failed\n"; - }); - return Error::success(); - } - - LLVM_DEBUG({ - dbgs() << "Writing MachO debug object header for " << G.getName() << "\n"; - }); - - MachOStructWriter Writer( - MachOContainerBlock->getAlreadyMutableContent().drop_front( - NonDebugSectionsStart)); - - unsigned LongSectionNameIdx = 0; - for (auto *Sec : NonDebugSections) { - size_t SepPos = Sec->getName().find(','); - StringRef SegName, SecName; - std::string CustomSecName; - - if ((SepPos == StringRef::npos && Sec->getName().size() <= 16)) { - // No embedded segment name, short section name. - SegName = "__JITLINK_CUSTOM"; - SecName = Sec->getName(); - } else if (SepPos < 16 && (Sec->getName().size() - (SepPos + 1) <= 16)) { - // Canonical embedded segment and section name. - SegName = Sec->getName().substr(0, SepPos); - SecName = Sec->getName().substr(SepPos + 1); - } else { - // Long section name that needs to be truncated. - assert(Sec->getName().size() > 16 && - "Short section name should have been handled above"); - SegName = "__JITLINK_CUSTOM"; - auto IdxStr = std::to_string(++LongSectionNameIdx); - CustomSecName = Sec->getName().substr(0, 15 - IdxStr.size()).str(); - CustomSecName += "."; - CustomSecName += IdxStr; - SecName = StringRef(CustomSecName.data(), 16); - } - - SectionRange R(*Sec); - if (R.getFirstBlock()->getAlignmentOffset() != 0) - return make_error<StringError>( - "While building MachO debug object for " + G.getName() + - " first block has non-zero alignment offset", - inconvertibleErrorCode()); - - typename MachOTraits::Section SecCmd; - memset(&SecCmd, 0, sizeof(SecCmd)); - memcpy(SecCmd.sectname, SecName.data(), SecName.size()); - memcpy(SecCmd.segname, SegName.data(), SegName.size()); - SecCmd.addr = R.getStart().getValue(); - SecCmd.size = R.getSize(); - SecCmd.offset = 0; - SecCmd.align = R.getFirstBlock()->getAlignment(); - SecCmd.reloff = 0; - SecCmd.nreloc = 0; - SecCmd.flags = 0; - Writer.write(SecCmd); - } - - static constexpr bool AutoRegisterCode = true; - SectionRange R(MachOContainerBlock->getSection()); - G.allocActions().push_back( - {cantFail(shared::WrapperFunctionCall::Create< - shared::SPSArgList<shared::SPSExecutorAddrRange, bool>>( - RegisterActionAddr, R.getRange(), AutoRegisterCode)), - {}}); - return Error::success(); - } - -private: - Block *MachOContainerBlock = nullptr; - SmallVector<Section *, 16> NonDebugSections; - size_t NonDebugSectionsStart = 0; -}; - -} // end anonymous namespace - -namespace llvm { -namespace orc { - -Expected<std::unique_ptr<GDBJITDebugInfoRegistrationPlugin>> -GDBJITDebugInfoRegistrationPlugin::Create(ExecutionSession &ES, - JITDylib &ProcessJD, - const Triple &TT) { - auto RegisterActionAddr = - TT.isOSBinFormatMachO() - ? ES.intern("_llvm_orc_registerJITLoaderGDBAllocAction") - : ES.intern("llvm_orc_registerJITLoaderGDBAllocAction"); - - if (auto RegisterSym = ES.lookup({&ProcessJD}, RegisterActionAddr)) - return std::make_unique<GDBJITDebugInfoRegistrationPlugin>( - RegisterSym->getAddress()); - else - return RegisterSym.takeError(); -} - -Error GDBJITDebugInfoRegistrationPlugin::notifyFailed( - MaterializationResponsibility &MR) { - return Error::success(); -} - -Error GDBJITDebugInfoRegistrationPlugin::notifyRemovingResources( - JITDylib &JD, ResourceKey K) { - return Error::success(); -} - -void GDBJITDebugInfoRegistrationPlugin::notifyTransferringResources( - JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) {} - -void GDBJITDebugInfoRegistrationPlugin::modifyPassConfig( - MaterializationResponsibility &MR, LinkGraph &LG, - PassConfiguration &PassConfig) { - - if (LG.getTargetTriple().getObjectFormat() == Triple::MachO) - modifyPassConfigForMachO(MR, LG, PassConfig); - else { - LLVM_DEBUG({ - dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unspported graph " - << LG.getName() << "(triple = " << LG.getTargetTriple().str() - << "\n"; - }); - } -} - -void GDBJITDebugInfoRegistrationPlugin::modifyPassConfigForMachO( - MaterializationResponsibility &MR, jitlink::LinkGraph &LG, - jitlink::PassConfiguration &PassConfig) { - - switch (LG.getTargetTriple().getArch()) { - case Triple::x86_64: - case Triple::aarch64: - // Supported, continue. - assert(LG.getPointerSize() == 8 && "Graph has incorrect pointer size"); - assert(LG.getEndianness() == support::little && - "Graph has incorrect endianness"); - break; - default: - // Unsupported. - LLVM_DEBUG({ - dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unsupported " - << "MachO graph " << LG.getName() - << "(triple = " << LG.getTargetTriple().str() - << ", pointer size = " << LG.getPointerSize() << ", endianness = " - << (LG.getEndianness() == support::big ? "big" : "little") - << ")\n"; - }); - return; - } - - // Scan for debug sections. If we find one then install passes. - bool HasDebugSections = false; - for (auto &Sec : LG.sections()) - if (MachODebugObjectSynthesizerBase::isDebugSection(Sec)) { - HasDebugSections = true; - break; - } - - if (HasDebugSections) { - LLVM_DEBUG({ - dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() - << " contains debug info. Installing debugger support passes.\n"; - }); - - auto MDOS = std::make_shared<MachODebugObjectSynthesizer<MachO64LE>>( - LG, RegisterActionAddr); - PassConfig.PrePrunePasses.push_back( - [=](LinkGraph &G) { return MDOS->preserveDebugSections(); }); - PassConfig.PostPrunePasses.push_back( - [=](LinkGraph &G) { return MDOS->startSynthesis(); }); - PassConfig.PreFixupPasses.push_back( - [=](LinkGraph &G) { return MDOS->completeSynthesisAndRegister(); }); - } else { - LLVM_DEBUG({ - dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() - << " contains no debug info. Skipping.\n"; - }); - } -} - -} // namespace orc -} // namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp new file mode 100644 index 000000000000..b541db3672f4 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebugInfoSupport.cpp @@ -0,0 +1,121 @@ +//===--- DebugInfoSupport.cpp -- Utils for debug info support ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Utilities to preserve and parse debug info from LinkGraphs. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" + +#include "llvm/Support/SmallVectorMemoryBuffer.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::jitlink; + +namespace { +static DenseSet<StringRef> DWARFSectionNames = { +#define HANDLE_DWARF_SECTION(ENUM_NAME, ELF_NAME, CMDLINE_NAME, OPTION) \ + StringRef(ELF_NAME), +#include "llvm/BinaryFormat/Dwarf.def" +#undef HANDLE_DWARF_SECTION +}; + +// We might be able to drop relocations to symbols that do end up +// being pruned by the linker, but for now we just preserve all +static void preserveDWARFSection(LinkGraph &G, Section &Sec) { + DenseMap<Block *, Symbol *> Preserved; + for (auto Sym : Sec.symbols()) { + if (Sym->isLive()) + Preserved[&Sym->getBlock()] = Sym; + else if (!Preserved.count(&Sym->getBlock())) + Preserved[&Sym->getBlock()] = Sym; + } + for (auto Block : Sec.blocks()) { + auto &PSym = Preserved[Block]; + if (!PSym) + PSym = &G.addAnonymousSymbol(*Block, 0, 0, false, true); + else if (!PSym->isLive()) + PSym->setLive(true); + } +} + +static SmallVector<char, 0> getSectionData(Section &Sec) { + SmallVector<char, 0> SecData; + SmallVector<Block *, 8> SecBlocks(Sec.blocks().begin(), Sec.blocks().end()); + std::sort(SecBlocks.begin(), SecBlocks.end(), [](Block *LHS, Block *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + // Convert back to what object file would have, one blob of section content + // Assumes all zerofill + // TODO handle alignment? + // TODO handle alignment offset? + for (auto *Block : SecBlocks) { + if (Block->isZeroFill()) + SecData.resize(SecData.size() + Block->getSize(), 0); + else + SecData.append(Block->getContent().begin(), Block->getContent().end()); + } + return SecData; +} + +static void dumpDWARFContext(DWARFContext &DC) { + auto options = llvm::DIDumpOptions(); + options.DumpType &= ~DIDT_UUID; + options.DumpType &= ~(1 << DIDT_ID_DebugFrame); + LLVM_DEBUG(DC.dump(dbgs(), options)); +} + +} // namespace + +Error llvm::orc::preserveDebugSections(LinkGraph &G) { + if (!G.getTargetTriple().isOSBinFormatELF()) { + return make_error<StringError>( + "preserveDebugSections only supports ELF LinkGraphs!", + inconvertibleErrorCode()); + } + for (auto &Sec : G.sections()) { + if (DWARFSectionNames.count(Sec.getName())) { + LLVM_DEBUG(dbgs() << "Preserving DWARF section " << Sec.getName() + << "\n"); + preserveDWARFSection(G, Sec); + } + } + return Error::success(); +} + +Expected<std::pair<std::unique_ptr<DWARFContext>, + StringMap<std::unique_ptr<MemoryBuffer>>>> +llvm::orc::createDWARFContext(LinkGraph &G) { + if (!G.getTargetTriple().isOSBinFormatELF()) { + return make_error<StringError>( + "createDWARFContext only supports ELF LinkGraphs!", + inconvertibleErrorCode()); + } + StringMap<std::unique_ptr<MemoryBuffer>> DWARFSectionData; + for (auto &Sec : G.sections()) { + if (DWARFSectionNames.count(Sec.getName())) { + auto SecData = getSectionData(Sec); + auto Name = Sec.getName(); + // DWARFContext expects the section name to not start with a dot + if (Name.startswith(".")) + Name = Name.drop_front(); + LLVM_DEBUG(dbgs() << "Creating DWARFContext section " << Name + << " with size " << SecData.size() << "\n"); + DWARFSectionData[Name] = + std::make_unique<SmallVectorMemoryBuffer>(std::move(SecData)); + } + } + auto Ctx = + DWARFContext::create(DWARFSectionData, G.getPointerSize(), + G.getEndianness() == llvm::endianness::little); + dumpDWARFContext(*Ctx); + return std::make_pair(std::move(Ctx), std::move(DWARFSectionData)); +} diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp new file mode 100644 index 000000000000..9ba6dd90f50d --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp @@ -0,0 +1,61 @@ +//===------ DebuggerSupport.cpp - Utils for enabling debugger support -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" +#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; + +namespace llvm::orc { + +Error enableDebuggerSupport(LLJIT &J) { + auto *ObjLinkingLayer = dyn_cast<ObjectLinkingLayer>(&J.getObjLinkingLayer()); + if (!ObjLinkingLayer) + return make_error<StringError>("Cannot enable LLJIT debugger support: " + "Debugger support requires JITLink", + inconvertibleErrorCode()); + auto ProcessSymsJD = J.getProcessSymbolsJITDylib(); + if (!ProcessSymsJD) + return make_error<StringError>("Cannot enable LLJIT debugger support: " + "Process symbols are not available", + inconvertibleErrorCode()); + + auto &ES = J.getExecutionSession(); + const auto &TT = J.getTargetTriple(); + + switch (TT.getObjectFormat()) { + case Triple::ELF: { + auto Registrar = createJITLoaderGDBRegistrar(ES); + if (!Registrar) + return Registrar.takeError(); + ObjLinkingLayer->addPlugin(std::make_unique<DebugObjectManagerPlugin>( + ES, std::move(*Registrar), true, true)); + return Error::success(); + } + case Triple::MachO: { + auto DS = GDBJITDebugInfoRegistrationPlugin::Create(ES, *ProcessSymsJD, TT); + if (!DS) + return DS.takeError(); + ObjLinkingLayer->addPlugin(std::move(*DS)); + return Error::success(); + } + default: + return make_error<StringError>( + "Cannot enable LLJIT debugger support: " + + Triple::getObjectFormatTypeName(TT.getObjectFormat()) + + " is not supported", + inconvertibleErrorCode()); + } +} + +} // namespace llvm::orc diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp new file mode 100644 index 000000000000..c6ffd9f7c2e3 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.cpp @@ -0,0 +1,423 @@ +//===------- DebuggerSupportPlugin.cpp - Utils for debugger support -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupportPlugin.h" +#include "llvm/ExecutionEngine/Orc/MachOBuilder.h" + +#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/DebugInfo/DWARF/DWARFContext.h" +#include "llvm/DebugInfo/DWARF/DWARFDebugLine.h" + +#include <chrono> + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +static const char *SynthDebugSectionName = "__jitlink_synth_debug_object"; + +namespace { + +class MachODebugObjectSynthesizerBase + : public GDBJITDebugInfoRegistrationPlugin::DebugSectionSynthesizer { +public: + static bool isDebugSection(Section &Sec) { + return Sec.getName().startswith("__DWARF,"); + } + + MachODebugObjectSynthesizerBase(LinkGraph &G, ExecutorAddr RegisterActionAddr) + : G(G), RegisterActionAddr(RegisterActionAddr) {} + virtual ~MachODebugObjectSynthesizerBase() = default; + + Error preserveDebugSections() { + if (G.findSectionByName(SynthDebugSectionName)) { + LLVM_DEBUG({ + dbgs() << "MachODebugObjectSynthesizer skipping graph " << G.getName() + << " which contains an unexpected existing " + << SynthDebugSectionName << " section.\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "MachODebugObjectSynthesizer visiting graph " << G.getName() + << "\n"; + }); + for (auto &Sec : G.sections()) { + if (!isDebugSection(Sec)) + continue; + // Preserve blocks in this debug section by marking one existing symbol + // live for each block, and introducing a new live, anonymous symbol for + // each currently unreferenced block. + LLVM_DEBUG({ + dbgs() << " Preserving debug section " << Sec.getName() << "\n"; + }); + SmallSet<Block *, 8> PreservedBlocks; + for (auto *Sym : Sec.symbols()) { + bool NewPreservedBlock = + PreservedBlocks.insert(&Sym->getBlock()).second; + if (NewPreservedBlock) + Sym->setLive(true); + } + for (auto *B : Sec.blocks()) + if (!PreservedBlocks.count(B)) + G.addAnonymousSymbol(*B, 0, 0, false, true); + } + + return Error::success(); + } + +protected: + LinkGraph &G; + ExecutorAddr RegisterActionAddr; +}; + +template <typename MachOTraits> +class MachODebugObjectSynthesizer : public MachODebugObjectSynthesizerBase { +public: + MachODebugObjectSynthesizer(ExecutionSession &ES, LinkGraph &G, + ExecutorAddr RegisterActionAddr) + : MachODebugObjectSynthesizerBase(G, RegisterActionAddr), + Builder(ES.getPageSize()) {} + + using MachODebugObjectSynthesizerBase::MachODebugObjectSynthesizerBase; + + Error startSynthesis() override { + LLVM_DEBUG({ + dbgs() << "Creating " << SynthDebugSectionName << " for " << G.getName() + << "\n"; + }); + + for (auto &Sec : G.sections()) { + if (Sec.blocks().empty()) + continue; + + // Skip sections whose name's don't fit the MachO standard. + if (Sec.getName().empty() || Sec.getName().size() > 33 || + Sec.getName().find(',') > 16) + continue; + + if (isDebugSection(Sec)) + DebugSections.push_back({&Sec, nullptr}); + else if (Sec.getMemLifetime() != MemLifetime::NoAlloc) + NonDebugSections.push_back({&Sec, nullptr}); + } + + // Bail out early if no debug sections. + if (DebugSections.empty()) + return Error::success(); + + // Write MachO header and debug section load commands. + Builder.Header.filetype = MachO::MH_OBJECT; + switch (G.getTargetTriple().getArch()) { + case Triple::x86_64: + Builder.Header.cputype = MachO::CPU_TYPE_X86_64; + Builder.Header.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; + break; + case Triple::aarch64: + Builder.Header.cputype = MachO::CPU_TYPE_ARM64; + Builder.Header.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; + break; + default: + llvm_unreachable("Unsupported architecture"); + } + + Seg = &Builder.addSegment(""); + + StringMap<std::unique_ptr<MemoryBuffer>> DebugSectionMap; + StringRef DebugLineSectionData; + for (auto &DSec : DebugSections) { + auto [SegName, SecName] = DSec.GraphSec->getName().split(','); + DSec.BuilderSec = &Seg->addSection(SecName, SegName); + + SectionRange SR(*DSec.GraphSec); + DSec.BuilderSec->Content.Size = SR.getSize(); + if (!SR.empty()) { + DSec.BuilderSec->align = Log2_64(SR.getFirstBlock()->getAlignment()); + StringRef SectionData(SR.getFirstBlock()->getContent().data(), + SR.getFirstBlock()->getSize()); + DebugSectionMap[SecName] = + MemoryBuffer::getMemBuffer(SectionData, G.getName(), false); + if (SecName == "__debug_line") + DebugLineSectionData = SectionData; + } + } + + std::optional<StringRef> FileName; + if (!DebugLineSectionData.empty()) { + assert((G.getEndianness() == llvm::endianness::big || + G.getEndianness() == llvm::endianness::little) && + "G.getEndianness() must be either big or little"); + auto DWARFCtx = + DWARFContext::create(DebugSectionMap, G.getPointerSize(), + G.getEndianness() == llvm::endianness::little); + DWARFDataExtractor DebugLineData( + DebugLineSectionData, G.getEndianness() == llvm::endianness::little, + G.getPointerSize()); + uint64_t Offset = 0; + DWARFDebugLine::LineTable LineTable; + + // Try to parse line data. Consume error on failure. + if (auto Err = LineTable.parse(DebugLineData, &Offset, *DWARFCtx, nullptr, + consumeError)) { + handleAllErrors(std::move(Err), [&](ErrorInfoBase &EIB) { + LLVM_DEBUG({ + dbgs() << "Cannot parse line table for \"" << G.getName() << "\": "; + EIB.log(dbgs()); + dbgs() << "\n"; + }); + }); + } else { + if (!LineTable.Prologue.FileNames.empty()) + FileName = *dwarf::toString(LineTable.Prologue.FileNames[0].Name); + } + } + + // If no line table (or unable to use) then use graph name. + // FIXME: There are probably other debug sections we should look in first. + if (!FileName) + FileName = StringRef(G.getName()); + + Builder.addSymbol("", MachO::N_SO, 0, 0, 0); + Builder.addSymbol(*FileName, MachO::N_SO, 0, 0, 0); + auto TimeStamp = std::chrono::duration_cast<std::chrono::seconds>( + std::chrono::system_clock::now().time_since_epoch()) + .count(); + Builder.addSymbol("", MachO::N_OSO, 3, 1, TimeStamp); + + for (auto &NDSP : NonDebugSections) { + auto [SegName, SecName] = NDSP.GraphSec->getName().split(','); + NDSP.BuilderSec = &Seg->addSection(SecName, SegName); + SectionRange SR(*NDSP.GraphSec); + if (!SR.empty()) + NDSP.BuilderSec->align = Log2_64(SR.getFirstBlock()->getAlignment()); + + // Add stabs. + for (auto *Sym : NDSP.GraphSec->symbols()) { + // Skip anonymous symbols. + if (!Sym->hasName()) + continue; + + uint8_t SymType = Sym->isCallable() ? MachO::N_FUN : MachO::N_GSYM; + + Builder.addSymbol("", MachO::N_BNSYM, 1, 0, 0); + StabSymbols.push_back( + {*Sym, Builder.addSymbol(Sym->getName(), SymType, 1, 0, 0), + Builder.addSymbol(Sym->getName(), SymType, 0, 0, 0)}); + Builder.addSymbol("", MachO::N_ENSYM, 1, 0, 0); + } + } + + Builder.addSymbol("", MachO::N_SO, 1, 0, 0); + + // Lay out the debug object, create a section and block for it. + size_t DebugObjectSize = Builder.layout(); + + auto &SDOSec = G.createSection(SynthDebugSectionName, MemProt::Read); + MachOContainerBlock = &G.createMutableContentBlock( + SDOSec, G.allocateBuffer(DebugObjectSize), orc::ExecutorAddr(), 8, 0); + + return Error::success(); + } + + Error completeSynthesisAndRegister() override { + if (!MachOContainerBlock) { + LLVM_DEBUG({ + dbgs() << "Not writing MachO debug object header for " << G.getName() + << " since createDebugSection failed\n"; + }); + + return Error::success(); + } + ExecutorAddr MaxAddr; + for (auto &NDSec : NonDebugSections) { + SectionRange SR(*NDSec.GraphSec); + NDSec.BuilderSec->addr = SR.getStart().getValue(); + NDSec.BuilderSec->size = SR.getSize(); + NDSec.BuilderSec->offset = SR.getStart().getValue(); + if (SR.getEnd() > MaxAddr) + MaxAddr = SR.getEnd(); + } + + for (auto &DSec : DebugSections) { + if (DSec.GraphSec->blocks_size() != 1) + return make_error<StringError>( + "Unexpected number of blocks in debug info section", + inconvertibleErrorCode()); + + if (ExecutorAddr(DSec.BuilderSec->addr) + DSec.BuilderSec->size > MaxAddr) + MaxAddr = ExecutorAddr(DSec.BuilderSec->addr) + DSec.BuilderSec->size; + + auto &B = **DSec.GraphSec->blocks().begin(); + DSec.BuilderSec->Content.Data = B.getContent().data(); + DSec.BuilderSec->Content.Size = B.getContent().size(); + DSec.BuilderSec->flags |= MachO::S_ATTR_DEBUG; + } + + LLVM_DEBUG({ + dbgs() << "Writing MachO debug object header for " << G.getName() << "\n"; + }); + + // Update stab symbol addresses. + for (auto &SS : StabSymbols) { + SS.StartStab.nlist().n_value = SS.Sym.getAddress().getValue(); + SS.EndStab.nlist().n_value = SS.Sym.getSize(); + } + + Builder.write(MachOContainerBlock->getAlreadyMutableContent()); + + static constexpr bool AutoRegisterCode = true; + SectionRange R(MachOContainerBlock->getSection()); + G.allocActions().push_back( + {cantFail(shared::WrapperFunctionCall::Create< + shared::SPSArgList<shared::SPSExecutorAddrRange, bool>>( + RegisterActionAddr, R.getRange(), AutoRegisterCode)), + {}}); + + return Error::success(); + } + +private: + struct SectionPair { + Section *GraphSec = nullptr; + typename MachOBuilder<MachOTraits>::Section *BuilderSec = nullptr; + }; + + struct StabSymbolsEntry { + using RelocTarget = typename MachOBuilder<MachOTraits>::RelocTarget; + + StabSymbolsEntry(Symbol &Sym, RelocTarget StartStab, RelocTarget EndStab) + : Sym(Sym), StartStab(StartStab), EndStab(EndStab) {} + + Symbol &Sym; + RelocTarget StartStab, EndStab; + }; + + using BuilderType = MachOBuilder<MachOTraits>; + + Block *MachOContainerBlock = nullptr; + MachOBuilder<MachOTraits> Builder; + typename MachOBuilder<MachOTraits>::Segment *Seg = nullptr; + std::vector<StabSymbolsEntry> StabSymbols; + SmallVector<SectionPair, 16> DebugSections; + SmallVector<SectionPair, 16> NonDebugSections; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<GDBJITDebugInfoRegistrationPlugin>> +GDBJITDebugInfoRegistrationPlugin::Create(ExecutionSession &ES, + JITDylib &ProcessJD, + const Triple &TT) { + auto RegisterActionAddr = + TT.isOSBinFormatMachO() + ? ES.intern("_llvm_orc_registerJITLoaderGDBAllocAction") + : ES.intern("llvm_orc_registerJITLoaderGDBAllocAction"); + + if (auto RegisterSym = ES.lookup({&ProcessJD}, RegisterActionAddr)) + return std::make_unique<GDBJITDebugInfoRegistrationPlugin>( + RegisterSym->getAddress()); + else + return RegisterSym.takeError(); +} + +Error GDBJITDebugInfoRegistrationPlugin::notifyFailed( + MaterializationResponsibility &MR) { + return Error::success(); +} + +Error GDBJITDebugInfoRegistrationPlugin::notifyRemovingResources( + JITDylib &JD, ResourceKey K) { + return Error::success(); +} + +void GDBJITDebugInfoRegistrationPlugin::notifyTransferringResources( + JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) {} + +void GDBJITDebugInfoRegistrationPlugin::modifyPassConfig( + MaterializationResponsibility &MR, LinkGraph &LG, + PassConfiguration &PassConfig) { + + if (LG.getTargetTriple().getObjectFormat() == Triple::MachO) + modifyPassConfigForMachO(MR, LG, PassConfig); + else { + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unspported graph " + << LG.getName() << "(triple = " << LG.getTargetTriple().str() + << "\n"; + }); + } +} + +void GDBJITDebugInfoRegistrationPlugin::modifyPassConfigForMachO( + MaterializationResponsibility &MR, jitlink::LinkGraph &LG, + jitlink::PassConfiguration &PassConfig) { + + switch (LG.getTargetTriple().getArch()) { + case Triple::x86_64: + case Triple::aarch64: + // Supported, continue. + assert(LG.getPointerSize() == 8 && "Graph has incorrect pointer size"); + assert(LG.getEndianness() == llvm::endianness::little && + "Graph has incorrect endianness"); + break; + default: + // Unsupported. + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin skipping unsupported " + << "MachO graph " << LG.getName() + << "(triple = " << LG.getTargetTriple().str() + << ", pointer size = " << LG.getPointerSize() << ", endianness = " + << (LG.getEndianness() == llvm::endianness::big ? "big" : "little") + << ")\n"; + }); + return; + } + + // Scan for debug sections. If we find one then install passes. + bool HasDebugSections = false; + for (auto &Sec : LG.sections()) + if (MachODebugObjectSynthesizerBase::isDebugSection(Sec)) { + HasDebugSections = true; + break; + } + + if (HasDebugSections) { + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() + << " contains debug info. Installing debugger support passes.\n"; + }); + + auto MDOS = std::make_shared<MachODebugObjectSynthesizer<MachO64LE>>( + MR.getTargetJITDylib().getExecutionSession(), LG, RegisterActionAddr); + PassConfig.PrePrunePasses.push_back( + [=](LinkGraph &G) { return MDOS->preserveDebugSections(); }); + PassConfig.PostPrunePasses.push_back( + [=](LinkGraph &G) { return MDOS->startSynthesis(); }); + PassConfig.PostFixupPasses.push_back( + [=](LinkGraph &G) { return MDOS->completeSynthesisAndRegister(); }); + } else { + LLVM_DEBUG({ + dbgs() << "GDBJITDebugInfoRegistrationPlugin: Graph " << LG.getName() + << " contains no debug info. Skipping.\n"; + }); + } +} + +} // namespace orc +} // namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp new file mode 100644 index 000000000000..fffecfc97814 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.cpp @@ -0,0 +1,303 @@ +//===----- PerfSupportPlugin.cpp --- Utils for perf support -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Handles support for registering code with perf +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Debugging/PerfSupportPlugin.h" + +#include "llvm/ExecutionEngine/JITLink/x86_64.h" +#include "llvm/ExecutionEngine/Orc/Debugging/DebugInfoSupport.h" +#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/ExecutionEngine/Orc/Shared/WrapperFunctionUtils.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::jitlink; + +namespace { + +// Creates an EH frame header prepared for a 32-bit relative relocation +// to the start of the .eh_frame section. Absolute injects a 64-bit absolute +// address space offset 4 bytes from the start instead of 4 bytes +Expected<std::string> createX64EHFrameHeader(Section &EHFrame, + llvm::endianness endianness, + bool absolute) { + uint8_t Version = 1; + uint8_t EhFramePtrEnc = 0; + if (absolute) { + EhFramePtrEnc |= dwarf::DW_EH_PE_sdata8 | dwarf::DW_EH_PE_absptr; + } else { + EhFramePtrEnc |= dwarf::DW_EH_PE_sdata4 | dwarf::DW_EH_PE_datarel; + } + uint8_t FDECountEnc = dwarf::DW_EH_PE_omit; + uint8_t TableEnc = dwarf::DW_EH_PE_omit; + // X86_64_64 relocation to the start of the .eh_frame section + uint32_t EHFrameRelocation = 0; + // uint32_t FDECount = 0; + // Skip the FDE binary search table + // We'd have to reprocess the CIEs to get this information, + // which seems like more trouble than it's worth + // TODO consider implementing this. + // binary search table goes here + + size_t HeaderSize = + (sizeof(Version) + sizeof(EhFramePtrEnc) + sizeof(FDECountEnc) + + sizeof(TableEnc) + + (absolute ? sizeof(uint64_t) : sizeof(EHFrameRelocation))); + std::string HeaderContent(HeaderSize, '\0'); + BinaryStreamWriter Writer( + MutableArrayRef<uint8_t>( + reinterpret_cast<uint8_t *>(HeaderContent.data()), HeaderSize), + endianness); + if (auto Err = Writer.writeInteger(Version)) + return std::move(Err); + if (auto Err = Writer.writeInteger(EhFramePtrEnc)) + return std::move(Err); + if (auto Err = Writer.writeInteger(FDECountEnc)) + return std::move(Err); + if (auto Err = Writer.writeInteger(TableEnc)) + return std::move(Err); + if (absolute) { + uint64_t EHFrameAddr = SectionRange(EHFrame).getStart().getValue(); + if (auto Err = Writer.writeInteger(EHFrameAddr)) + return std::move(Err); + } else { + if (auto Err = Writer.writeInteger(EHFrameRelocation)) + return std::move(Err); + } + return HeaderContent; +} + +constexpr StringRef RegisterPerfStartSymbolName = + "llvm_orc_registerJITLoaderPerfStart"; +constexpr StringRef RegisterPerfEndSymbolName = + "llvm_orc_registerJITLoaderPerfEnd"; +constexpr StringRef RegisterPerfImplSymbolName = + "llvm_orc_registerJITLoaderPerfImpl"; + +static PerfJITCodeLoadRecord +getCodeLoadRecord(const Symbol &Sym, std::atomic<uint64_t> &CodeIndex) { + PerfJITCodeLoadRecord Record; + auto Name = Sym.getName(); + auto Addr = Sym.getAddress(); + auto Size = Sym.getSize(); + Record.Prefix.Id = PerfJITRecordType::JIT_CODE_LOAD; + // Runtime sets PID + Record.Pid = 0; + // Runtime sets TID + Record.Tid = 0; + Record.Vma = Addr.getValue(); + Record.CodeAddr = Addr.getValue(); + Record.CodeSize = Size; + Record.CodeIndex = CodeIndex++; + Record.Name = Name.str(); + // Initialize last, once all the other fields are filled + Record.Prefix.TotalSize = + (2 * sizeof(uint32_t) // id, total_size + + sizeof(uint64_t) // timestamp + + 2 * sizeof(uint32_t) // pid, tid + + 4 * sizeof(uint64_t) // vma, code_addr, code_size, code_index + + Name.size() + 1 // symbol name + + Record.CodeSize // code + ); + return Record; +} + +static std::optional<PerfJITDebugInfoRecord> +getDebugInfoRecord(const Symbol &Sym, DWARFContext &DC) { + auto &Section = Sym.getBlock().getSection(); + auto Addr = Sym.getAddress(); + auto Size = Sym.getSize(); + auto SAddr = object::SectionedAddress{Addr.getValue(), Section.getOrdinal()}; + LLVM_DEBUG(dbgs() << "Getting debug info for symbol " << Sym.getName() + << " at address " << Addr.getValue() << " with size " + << Size << "\n" + << "Section ordinal: " << Section.getOrdinal() << "\n"); + auto LInfo = DC.getLineInfoForAddressRange( + SAddr, Size, DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath); + if (LInfo.empty()) { + // No line info available + LLVM_DEBUG(dbgs() << "No line info available\n"); + return std::nullopt; + } + PerfJITDebugInfoRecord Record; + Record.Prefix.Id = PerfJITRecordType::JIT_CODE_DEBUG_INFO; + Record.CodeAddr = Addr.getValue(); + for (const auto &Entry : LInfo) { + auto Addr = Entry.first; + // The function re-created by perf is preceded by a elf + // header. Need to adjust for that, otherwise the results are + // wrong. + Addr += 0x40; + Record.Entries.push_back({Addr, Entry.second.Line, + Entry.second.Discriminator, + Entry.second.FileName}); + } + size_t EntriesBytes = (2 // record header + + 2 // record fields + ) * + sizeof(uint64_t); + for (const auto &Entry : Record.Entries) { + EntriesBytes += + sizeof(uint64_t) + 2 * sizeof(uint32_t); // Addr, Line/Discrim + EntriesBytes += Entry.Name.size() + 1; // Name + } + Record.Prefix.TotalSize = EntriesBytes; + LLVM_DEBUG(dbgs() << "Created debug info record\n" + << "Total size: " << Record.Prefix.TotalSize << "\n" + << "Nr entries: " << Record.Entries.size() << "\n"); + return Record; +} + +static Expected<PerfJITCodeUnwindingInfoRecord> +getUnwindingRecord(LinkGraph &G) { + PerfJITCodeUnwindingInfoRecord Record; + Record.Prefix.Id = PerfJITRecordType::JIT_CODE_UNWINDING_INFO; + Record.Prefix.TotalSize = 0; + auto Eh_frame = G.findSectionByName(".eh_frame"); + if (!Eh_frame) { + LLVM_DEBUG(dbgs() << "No .eh_frame section found\n"); + return Record; + } + if (!G.getTargetTriple().isOSBinFormatELF()) { + LLVM_DEBUG(dbgs() << "Not an ELF file, will not emit unwinding info\n"); + return Record; + } + auto SR = SectionRange(*Eh_frame); + auto EHFrameSize = SR.getSize(); + auto Eh_frame_hdr = G.findSectionByName(".eh_frame_hdr"); + if (!Eh_frame_hdr) { + if (G.getTargetTriple().getArch() == Triple::x86_64) { + auto Hdr = createX64EHFrameHeader(*Eh_frame, G.getEndianness(), true); + if (!Hdr) + return Hdr.takeError(); + Record.EHFrameHdr = std::move(*Hdr); + } else { + LLVM_DEBUG(dbgs() << "No .eh_frame_hdr section found\n"); + return Record; + } + Record.EHFrameHdrAddr = 0; + Record.EHFrameHdrSize = Record.EHFrameHdr.size(); + Record.UnwindDataSize = EHFrameSize + Record.EHFrameHdrSize; + Record.MappedSize = 0; // Because the EHFrame header was not mapped + } else { + auto SR = SectionRange(*Eh_frame_hdr); + Record.EHFrameHdrAddr = SR.getStart().getValue(); + Record.EHFrameHdrSize = SR.getSize(); + Record.UnwindDataSize = EHFrameSize + Record.EHFrameHdrSize; + Record.MappedSize = Record.UnwindDataSize; + } + Record.EHFrameAddr = SR.getStart().getValue(); + Record.Prefix.TotalSize = + (2 * sizeof(uint32_t) // id, total_size + + sizeof(uint64_t) // timestamp + + + 3 * sizeof(uint64_t) // unwind_data_size, eh_frame_hdr_size, mapped_size + + Record.UnwindDataSize // eh_frame_hdr, eh_frame + ); + LLVM_DEBUG(dbgs() << "Created unwind record\n" + << "Total size: " << Record.Prefix.TotalSize << "\n" + << "Unwind size: " << Record.UnwindDataSize << "\n" + << "EHFrame size: " << EHFrameSize << "\n" + << "EHFrameHdr size: " << Record.EHFrameHdrSize << "\n"); + return Record; +} + +static PerfJITRecordBatch getRecords(ExecutionSession &ES, LinkGraph &G, + std::atomic<uint64_t> &CodeIndex, + bool EmitDebugInfo, bool EmitUnwindInfo) { + std::unique_ptr<DWARFContext> DC; + StringMap<std::unique_ptr<MemoryBuffer>> DCBacking; + if (EmitDebugInfo) { + auto EDC = createDWARFContext(G); + if (!EDC) { + ES.reportError(EDC.takeError()); + EmitDebugInfo = false; + } else { + DC = std::move(EDC->first); + DCBacking = std::move(EDC->second); + } + } + PerfJITRecordBatch Batch; + for (auto Sym : G.defined_symbols()) { + if (!Sym->hasName() || !Sym->isCallable()) + continue; + if (EmitDebugInfo) { + auto DebugInfo = getDebugInfoRecord(*Sym, *DC); + if (DebugInfo) + Batch.DebugInfoRecords.push_back(std::move(*DebugInfo)); + } + Batch.CodeLoadRecords.push_back(getCodeLoadRecord(*Sym, CodeIndex)); + } + if (EmitUnwindInfo) { + auto UWR = getUnwindingRecord(G); + if (!UWR) { + ES.reportError(UWR.takeError()); + } else { + Batch.UnwindingRecord = std::move(*UWR); + } + } else { + Batch.UnwindingRecord.Prefix.TotalSize = 0; + } + return Batch; +} +} // namespace + +PerfSupportPlugin::PerfSupportPlugin(ExecutorProcessControl &EPC, + ExecutorAddr RegisterPerfStartAddr, + ExecutorAddr RegisterPerfEndAddr, + ExecutorAddr RegisterPerfImplAddr, + bool EmitDebugInfo, bool EmitUnwindInfo) + : EPC(EPC), RegisterPerfStartAddr(RegisterPerfStartAddr), + RegisterPerfEndAddr(RegisterPerfEndAddr), + RegisterPerfImplAddr(RegisterPerfImplAddr), CodeIndex(0), + EmitDebugInfo(EmitDebugInfo), EmitUnwindInfo(EmitUnwindInfo) { + cantFail(EPC.callSPSWrapper<void()>(RegisterPerfStartAddr)); +} +PerfSupportPlugin::~PerfSupportPlugin() { + cantFail(EPC.callSPSWrapper<void()>(RegisterPerfEndAddr)); +} + +void PerfSupportPlugin::modifyPassConfig(MaterializationResponsibility &MR, + LinkGraph &G, + PassConfiguration &Config) { + Config.PostFixupPasses.push_back([this](LinkGraph &G) { + auto Batch = getRecords(EPC.getExecutionSession(), G, CodeIndex, + EmitDebugInfo, EmitUnwindInfo); + G.allocActions().push_back( + {cantFail(shared::WrapperFunctionCall::Create< + shared::SPSArgList<shared::SPSPerfJITRecordBatch>>( + RegisterPerfImplAddr, Batch)), + {}}); + return Error::success(); + }); +} + +Expected<std::unique_ptr<PerfSupportPlugin>> +PerfSupportPlugin::Create(ExecutorProcessControl &EPC, JITDylib &JD, + bool EmitDebugInfo, bool EmitUnwindInfo) { + if (!EPC.getTargetTriple().isOSBinFormatELF()) { + return make_error<StringError>( + "Perf support only available for ELF LinkGraphs!", + inconvertibleErrorCode()); + } + auto &ES = EPC.getExecutionSession(); + ExecutorAddr StartAddr, EndAddr, ImplAddr; + if (auto Err = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder({&JD}), + {{ES.intern(RegisterPerfStartSymbolName), &StartAddr}, + {ES.intern(RegisterPerfEndSymbolName), &EndAddr}, + {ES.intern(RegisterPerfImplSymbolName), &ImplAddr}})) + return std::move(Err); + return std::make_unique<PerfSupportPlugin>(EPC, StartAddr, EndAddr, ImplAddr, + EmitDebugInfo, EmitUnwindInfo); +} diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp index 1bb4ecdff299..2b6c4b9e7f43 100644 --- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp @@ -11,6 +11,7 @@ #include "llvm/BinaryFormat/ELF.h" #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" #include "llvm/ExecutionEngine/JITLink/aarch64.h" +#include "llvm/ExecutionEngine/JITLink/ppc64.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" @@ -39,21 +40,31 @@ public: void materialize(std::unique_ptr<MaterializationResponsibility> R) override { unsigned PointerSize; - support::endianness Endianness; + llvm::endianness Endianness; jitlink::Edge::Kind EdgeKind; const auto &TT = ENP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::x86_64: PointerSize = 8; - Endianness = support::endianness::little; + Endianness = llvm::endianness::little; EdgeKind = jitlink::x86_64::Pointer64; break; case Triple::aarch64: PointerSize = 8; - Endianness = support::endianness::little; + Endianness = llvm::endianness::little; EdgeKind = jitlink::aarch64::Pointer64; break; + case Triple::ppc64: + PointerSize = 8; + Endianness = llvm::endianness::big; + EdgeKind = jitlink::ppc64::Pointer64; + break; + case Triple::ppc64le: + PointerSize = 8; + Endianness = llvm::endianness::little; + EdgeKind = jitlink::ppc64::Pointer64; + break; default: llvm_unreachable("Unrecognized architecture"); } @@ -238,6 +249,9 @@ bool ELFNixPlatform::supportedTarget(const Triple &TT) { switch (TT.getArch()) { case Triple::x86_64: case Triple::aarch64: + // FIXME: jitlink for ppc64 hasn't been well tested, leave it unsupported + // right now. + case Triple::ppc64le: return true; default: return false; diff --git a/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp b/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp index 56cd982cd5e1..f15315260ab0 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp @@ -9,67 +9,40 @@ #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" #include "llvm/ExecutionEngine/Orc/Core.h" -#include "llvm/Support/BinaryStreamWriter.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" using namespace llvm::orc::shared; namespace llvm { namespace orc { -Expected<std::unique_ptr<EPCEHFrameRegistrar>> EPCEHFrameRegistrar::Create( - ExecutionSession &ES, - std::optional<ExecutorAddr> RegistrationFunctionsDylib) { - // FIXME: Proper mangling here -- we really need to decouple linker mangling - // from DataLayout. +Expected<std::unique_ptr<EPCEHFrameRegistrar>> +EPCEHFrameRegistrar::Create(ExecutionSession &ES) { - // Find the addresses of the registration/deregistration functions in the - // executor process. - auto &EPC = ES.getExecutorProcessControl(); + // Lookup addresseses of the registration/deregistration functions in the + // bootstrap map. + ExecutorAddr RegisterEHFrameSectionWrapper; + ExecutorAddr DeregisterEHFrameSectionWrapper; + if (auto Err = ES.getExecutorProcessControl().getBootstrapSymbols( + {{RegisterEHFrameSectionWrapper, + rt::RegisterEHFrameSectionWrapperName}, + {DeregisterEHFrameSectionWrapper, + rt::DeregisterEHFrameSectionWrapperName}})) + return std::move(Err); - if (!RegistrationFunctionsDylib) { - if (auto D = EPC.loadDylib(nullptr)) - RegistrationFunctionsDylib = *D; - else - return D.takeError(); - } - - std::string RegisterWrapperName, DeregisterWrapperName; - if (EPC.getTargetTriple().isOSBinFormatMachO()) { - RegisterWrapperName += '_'; - DeregisterWrapperName += '_'; - } - RegisterWrapperName += "llvm_orc_registerEHFrameSectionWrapper"; - DeregisterWrapperName += "llvm_orc_deregisterEHFrameSectionWrapper"; - - SymbolLookupSet RegistrationSymbols; - RegistrationSymbols.add(EPC.intern(RegisterWrapperName)); - RegistrationSymbols.add(EPC.intern(DeregisterWrapperName)); - - auto Result = - EPC.lookupSymbols({{*RegistrationFunctionsDylib, RegistrationSymbols}}); - if (!Result) - return Result.takeError(); - - assert(Result->size() == 1 && "Unexpected number of dylibs in result"); - assert((*Result)[0].size() == 2 && - "Unexpected number of addresses in result"); - - auto RegisterEHFrameWrapperFnAddr = (*Result)[0][0]; - auto DeregisterEHFrameWrapperFnAddr = (*Result)[0][1]; - - return std::make_unique<EPCEHFrameRegistrar>(ES, RegisterEHFrameWrapperFnAddr, - DeregisterEHFrameWrapperFnAddr); + return std::make_unique<EPCEHFrameRegistrar>( + ES, RegisterEHFrameSectionWrapper, DeregisterEHFrameSectionWrapper); } Error EPCEHFrameRegistrar::registerEHFrames(ExecutorAddrRange EHFrameSection) { return ES.callSPSWrapper<void(SPSExecutorAddrRange)>( - RegisterEHFrameWrapperFnAddr, EHFrameSection); + RegisterEHFrameSectionWrapper, EHFrameSection); } Error EPCEHFrameRegistrar::deregisterEHFrames( ExecutorAddrRange EHFrameSection) { return ES.callSPSWrapper<void(SPSExecutorAddrRange)>( - DeregisterEHFrameWrapperFnAddr, EHFrameSection); + DeregisterEHFrameSectionWrapper, EHFrameSection); } } // end namespace orc diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index fb685e6c3727..7316b2dce8ab 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -284,7 +284,7 @@ StaticLibraryDefinitionGenerator::Load( // If this is a universal binary then search for a slice matching the given // Triple. - if (auto *UB = cast<object::MachOUniversalBinary>(B->getBinary())) { + if (auto *UB = dyn_cast<object::MachOUniversalBinary>(B->getBinary())) { const auto &TT = L.getExecutionSession().getTargetTriple(); @@ -347,7 +347,7 @@ StaticLibraryDefinitionGenerator::Create( // If this is a universal binary then search for a slice matching the given // Triple. - if (auto *UB = cast<object::MachOUniversalBinary>(B->get())) { + if (auto *UB = dyn_cast<object::MachOUniversalBinary>(B->get())) { const auto &TT = L.getExecutionSession().getTargetTriple(); @@ -538,11 +538,11 @@ DLLImportDefinitionGenerator::getTargetPointerSize(const Triple &TT) { } } -Expected<support::endianness> +Expected<llvm::endianness> DLLImportDefinitionGenerator::getTargetEndianness(const Triple &TT) { switch (TT.getArch()) { case Triple::x86_64: - return support::endianness::little; + return llvm::endianness::little; default: return make_error<StringError>( "architecture unsupported by DLLImportDefinitionGenerator", @@ -553,7 +553,7 @@ DLLImportDefinitionGenerator::getTargetEndianness(const Triple &TT) { Expected<std::unique_ptr<jitlink::LinkGraph>> DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) { Triple TT = ES.getTargetTriple(); - auto PointerSize = getTargetEndianness(TT); + auto PointerSize = getTargetPointerSize(TT); if (!PointerSize) return PointerSize.takeError(); auto Endianness = getTargetEndianness(TT); diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index b8b013f8a7a9..ad27deff38d9 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -9,6 +9,8 @@ #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" #include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Process.h" @@ -27,7 +29,8 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( std::shared_ptr<SymbolStringPool> SSP, std::unique_ptr<TaskDispatcher> D, Triple TargetTriple, unsigned PageSize, std::unique_ptr<jitlink::JITLinkMemoryManager> MemMgr) - : ExecutorProcessControl(std::move(SSP), std::move(D)) { + : ExecutorProcessControl(std::move(SSP), std::move(D)), + InProcessMemoryAccess(TargetTriple.isArch64Bit()) { OwnedMemMgr = std::move(MemMgr); if (!OwnedMemMgr) @@ -42,6 +45,11 @@ SelfExecutorProcessControl::SelfExecutorProcessControl( ExecutorAddr::fromPtr(this)}; if (this->TargetTriple.isOSBinFormatMachO()) GlobalManglingPrefix = '_'; + + this->BootstrapSymbols[rt::RegisterEHFrameSectionWrapperName] = + ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper); + this->BootstrapSymbols[rt::DeregisterEHFrameSectionWrapperName] = + ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper); } Expected<std::unique_ptr<SelfExecutorProcessControl>> @@ -139,41 +147,54 @@ Error SelfExecutorProcessControl::disconnect() { return Error::success(); } -void SelfExecutorProcessControl::writeUInt8sAsync( - ArrayRef<tpctypes::UInt8Write> Ws, WriteResultFn OnWriteComplete) { +void InProcessMemoryAccess::writeUInt8sAsync(ArrayRef<tpctypes::UInt8Write> Ws, + WriteResultFn OnWriteComplete) { for (auto &W : Ws) *W.Addr.toPtr<uint8_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeUInt16sAsync( +void InProcessMemoryAccess::writeUInt16sAsync( ArrayRef<tpctypes::UInt16Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) *W.Addr.toPtr<uint16_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeUInt32sAsync( +void InProcessMemoryAccess::writeUInt32sAsync( ArrayRef<tpctypes::UInt32Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) *W.Addr.toPtr<uint32_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeUInt64sAsync( +void InProcessMemoryAccess::writeUInt64sAsync( ArrayRef<tpctypes::UInt64Write> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) *W.Addr.toPtr<uint64_t *>() = W.Value; OnWriteComplete(Error::success()); } -void SelfExecutorProcessControl::writeBuffersAsync( +void InProcessMemoryAccess::writeBuffersAsync( ArrayRef<tpctypes::BufferWrite> Ws, WriteResultFn OnWriteComplete) { for (auto &W : Ws) memcpy(W.Addr.toPtr<char *>(), W.Buffer.data(), W.Buffer.size()); OnWriteComplete(Error::success()); } +void InProcessMemoryAccess::writePointersAsync( + ArrayRef<tpctypes::PointerWrite> Ws, WriteResultFn OnWriteComplete) { + if (IsArch64Bit) { + for (auto &W : Ws) + *W.Addr.toPtr<uint64_t *>() = W.Value.getValue(); + } else { + for (auto &W : Ws) + *W.Addr.toPtr<uint32_t *>() = static_cast<uint32_t>(W.Value.getValue()); + } + + OnWriteComplete(Error::success()); +} + shared::CWrapperFunctionResult SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager( void *Ctx, const void *FnTag, const char *Data, size_t Size) { diff --git a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index a0d81cdf2086..adf38659de3d 100644 --- a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -244,8 +244,7 @@ Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) { Constant *AddrIntVal = ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue()); Constant *AddrPtrVal = - ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, - PointerType::get(&FT, 0)); + ConstantExpr::getIntToPtr(AddrIntVal, PointerType::get(&FT, 0)); return AddrPtrVal; } diff --git a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index b66f52f1ec5d..17a96dee1000 100644 --- a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -126,16 +126,16 @@ void JITTargetMachineBuilderPrinter::print(raw_ostream &OS) const { OS << "\n" << Indent << " Optimization Level = "; switch (JTMB.OptLevel) { - case CodeGenOpt::None: + case CodeGenOptLevel::None: OS << "None"; break; - case CodeGenOpt::Less: + case CodeGenOptLevel::Less: OS << "Less"; break; - case CodeGenOpt::Default: + case CodeGenOptLevel::Default: OS << "Default"; break; - case CodeGenOpt::Aggressive: + case CodeGenOptLevel::Aggressive: OS << "Aggressive"; break; } diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index 7c7c2f000368..90123b3f0a96 100644 --- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -10,8 +10,6 @@ #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" #include "llvm/ExecutionEngine/Orc/COFFPlatform.h" -#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" -#include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" @@ -29,8 +27,6 @@ #include "llvm/IR/Module.h" #include "llvm/Support/DynamicLibrary.h" -#include <map> - #define DEBUG_TYPE "orc" using namespace llvm; @@ -752,6 +748,12 @@ Error LLJITBuilderState::prepareForConstruction() { case Triple::x86_64: UseJITLink = !TT.isOSBinFormatCOFF(); break; + case Triple::ppc64: + UseJITLink = TT.isPPC64ELFv2ABI(); + break; + case Triple::ppc64le: + UseJITLink = TT.isOSBinFormatELF(); + break; default: break; } @@ -775,25 +777,17 @@ Error LLJITBuilderState::prepareForConstruction() { // If we need a process JITDylib but no setup function has been given then // create a default one. - if (!SetupProcessSymbolsJITDylib && - (LinkProcessSymbolsByDefault || EnableDebuggerSupport)) { - - LLVM_DEBUG({ - dbgs() << "Creating default Process JD setup function (neeeded for"; - if (LinkProcessSymbolsByDefault) - dbgs() << " <link-process-syms-by-default>"; - if (EnableDebuggerSupport) - dbgs() << " <debugger-support>"; - dbgs() << ")\n"; - }); - - SetupProcessSymbolsJITDylib = [this](JITDylib &JD) -> Error { + if (!SetupProcessSymbolsJITDylib && LinkProcessSymbolsByDefault) { + LLVM_DEBUG(dbgs() << "Creating default Process JD setup function\n"); + SetupProcessSymbolsJITDylib = [this](LLJIT &J) -> Expected<JITDylibSP> { + auto &JD = + J.getExecutionSession().createBareJITDylib("<Process Symbols>"); auto G = orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( DL->getGlobalPrefix()); if (!G) return G.takeError(); JD.addGenerator(std::move(*G)); - return Error::success(); + return &JD; }; } @@ -998,50 +992,18 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) } if (S.SetupProcessSymbolsJITDylib) { - ProcessSymbols = &ES->createBareJITDylib("<Process Symbols>"); - if (auto Err2 = S.SetupProcessSymbolsJITDylib(*ProcessSymbols)) { - Err = std::move(Err2); + if (auto ProcSymsJD = S.SetupProcessSymbolsJITDylib(*this)) { + ProcessSymbols = ProcSymsJD->get(); + } else { + Err = ProcSymsJD.takeError(); return; } } - if (S.EnableDebuggerSupport) { - if (auto *OLL = dyn_cast<ObjectLinkingLayer>(ObjLinkingLayer.get())) { - switch (TT.getObjectFormat()) { - case Triple::ELF: { - auto Registrar = createJITLoaderGDBRegistrar(*ES); - if (!Registrar) { - Err = Registrar.takeError(); - return; - } - OLL->addPlugin(std::make_unique<DebugObjectManagerPlugin>( - *ES, std::move(*Registrar), true, true)); - break; - } - case Triple::MachO: { - assert(ProcessSymbols && "ProcessSymbols JD should be available when " - "EnableDebuggerSupport is set"); - auto DS = - GDBJITDebugInfoRegistrationPlugin::Create(*ES, *ProcessSymbols, TT); - if (!DS) { - Err = DS.takeError(); - return; - } - OLL->addPlugin(std::move(*DS)); - break; - } - default: - LLVM_DEBUG({ - dbgs() << "Cannot enable LLJIT debugger support: " - << Triple::getObjectFormatTypeName(TT.getObjectFormat()) - << " not supported.\n"; - }); - } - } else { - LLVM_DEBUG({ - dbgs() << "Cannot enable LLJIT debugger support: " - " debugger support is only available when using JITLink.\n"; - }); + if (S.PrePlatformSetup) { + if (auto Err2 = S.PrePlatformSetup(*this)) { + Err = std::move(Err2); + return; } } @@ -1131,7 +1093,7 @@ Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) { if (!ObjLinkingLayer) return make_error<StringError>( - "SetUpTargetPlatform requires ObjectLinkingLayer", + "ExecutorNativePlatform requires ObjectLinkingLayer", inconvertibleErrorCode()); std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer; diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index a3a766d602c1..a155b39ae263 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -34,6 +34,8 @@ using SPSMachOJITDylibDepInfo = SPSTuple<bool, SPSSequence<SPSExecutorAddr>>; using SPSMachOJITDylibDepInfoMap = SPSSequence<SPSTuple<SPSExecutorAddr, SPSMachOJITDylibDepInfo>>; +class SPSMachOExecutorSymbolFlags; + template <> class SPSSerializationTraits<SPSMachOJITDylibDepInfo, MachOPlatform::MachOJITDylibDepInfo> { @@ -55,23 +57,54 @@ public: } }; +template <> +class SPSSerializationTraits<SPSMachOExecutorSymbolFlags, + MachOPlatform::MachOExecutorSymbolFlags> { +private: + using UT = std::underlying_type_t<MachOPlatform::MachOExecutorSymbolFlags>; + +public: + static size_t size(const MachOPlatform::MachOExecutorSymbolFlags &SF) { + return sizeof(UT); + } + + static bool serialize(SPSOutputBuffer &OB, + const MachOPlatform::MachOExecutorSymbolFlags &SF) { + return SPSArgList<UT>::serialize(OB, static_cast<UT>(SF)); + } + + static bool deserialize(SPSInputBuffer &IB, + MachOPlatform::MachOExecutorSymbolFlags &SF) { + UT Tmp; + if (!SPSArgList<UT>::deserialize(IB, Tmp)) + return false; + SF = static_cast<MachOPlatform::MachOExecutorSymbolFlags>(Tmp); + return true; + } +}; + } // namespace shared } // namespace orc } // namespace llvm namespace { +using SPSRegisterSymbolsArgs = + SPSArgList<SPSExecutorAddr, + SPSSequence<SPSTuple<SPSExecutorAddr, SPSExecutorAddr, + SPSMachOExecutorSymbolFlags>>>; + std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP, std::string Name) { unsigned PointerSize; - support::endianness Endianness; + llvm::endianness Endianness; const auto &TT = MOP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::aarch64: case Triple::x86_64: PointerSize = 8; - Endianness = support::endianness::little; + Endianness = llvm::endianness::little; break; default: llvm_unreachable("Unrecognized architecture"); @@ -146,7 +179,7 @@ private: Hdr.flags = 0; Hdr.reserved = 0; - if (G.getEndianness() != support::endian::system_endianness()) + if (G.getEndianness() != llvm::endianness::native) MachO::swapStruct(Hdr); auto HeaderContent = G.allocateContent( @@ -180,21 +213,28 @@ constexpr MachOHeaderMaterializationUnit::HeaderSymbol class MachOPlatformCompleteBootstrapMaterializationUnit : public MaterializationUnit { public: + using SymbolTableVector = + SmallVector<std::tuple<ExecutorAddr, ExecutorAddr, + MachOPlatform::MachOExecutorSymbolFlags>>; + MachOPlatformCompleteBootstrapMaterializationUnit( MachOPlatform &MOP, StringRef PlatformJDName, - SymbolStringPtr CompleteBootstrapSymbol, shared::AllocActions DeferredAAs, + SymbolStringPtr CompleteBootstrapSymbol, SymbolTableVector SymTab, + shared::AllocActions DeferredAAs, ExecutorAddr MachOHeaderAddr, ExecutorAddr PlatformBootstrap, ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib, ExecutorAddr DeregisterJITDylib, - ExecutorAddr MachOHeaderAddr) + ExecutorAddr RegisterObjectSymbolTable, + ExecutorAddr DeregisterObjectSymbolTable) : MaterializationUnit( {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}), MOP(MOP), PlatformJDName(PlatformJDName), CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)), - DeferredAAs(std::move(DeferredAAs)), - PlatformBootstrap(PlatformBootstrap), + SymTab(std::move(SymTab)), DeferredAAs(std::move(DeferredAAs)), + MachOHeaderAddr(MachOHeaderAddr), PlatformBootstrap(PlatformBootstrap), PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib), DeregisterJITDylib(DeregisterJITDylib), - MachOHeaderAddr(MachOHeaderAddr) {} + RegisterObjectSymbolTable(RegisterObjectSymbolTable), + DeregisterObjectSymbolTable(DeregisterObjectSymbolTable) {} StringRef getName() const override { return "MachOPlatformCompleteBootstrap"; @@ -211,7 +251,7 @@ public: Linkage::Strong, Scope::Hidden, false, true); // Reserve space for the stolen actions, plus two extras. - G->allocActions().reserve(DeferredAAs.size() + 2); + G->allocActions().reserve(DeferredAAs.size() + 3); // 1. Bootstrap the platform support code. G->allocActions().push_back( @@ -227,7 +267,14 @@ public: cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( DeregisterJITDylib, MachOHeaderAddr))}); - // 3. Add the deferred actions to the graph. + // 3. Register deferred symbols. + G->allocActions().push_back( + {cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( + RegisterObjectSymbolTable, MachOHeaderAddr, SymTab)), + cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( + DeregisterObjectSymbolTable, MachOHeaderAddr, SymTab))}); + + // 4. Add the deferred actions to the graph. std::move(DeferredAAs.begin(), DeferredAAs.end(), std::back_inserter(G->allocActions())); @@ -240,12 +287,15 @@ private: MachOPlatform &MOP; StringRef PlatformJDName; SymbolStringPtr CompleteBootstrapSymbol; + SymbolTableVector SymTab; shared::AllocActions DeferredAAs; + ExecutorAddr MachOHeaderAddr; ExecutorAddr PlatformBootstrap; ExecutorAddr PlatformShutdown; ExecutorAddr RegisterJITDylib; ExecutorAddr DeregisterJITDylib; - ExecutorAddr MachOHeaderAddr; + ExecutorAddr RegisterObjectSymbolTable; + ExecutorAddr DeregisterObjectSymbolTable; }; static StringRef ObjCRuntimeObjectSectionsData[] = { @@ -266,6 +316,33 @@ static StringRef ObjCRuntimeObjectSectionName = static StringRef ObjCImageInfoSymbolName = "__llvm_jitlink_macho_objc_imageinfo"; +struct ObjCImageInfoFlags { + uint16_t SwiftABIVersion; + uint16_t SwiftVersion; + bool HasCategoryClassProperties; + bool HasSignedObjCClassROs; + + static constexpr uint32_t SIGNED_CLASS_RO = (1 << 4); + static constexpr uint32_t HAS_CATEGORY_CLASS_PROPERTIES = (1 << 6); + + explicit ObjCImageInfoFlags(uint32_t RawFlags) { + HasSignedObjCClassROs = RawFlags & SIGNED_CLASS_RO; + HasCategoryClassProperties = RawFlags & HAS_CATEGORY_CLASS_PROPERTIES; + SwiftABIVersion = (RawFlags >> 8) & 0xFF; + SwiftVersion = (RawFlags >> 16) & 0xFFFF; + } + + uint32_t rawFlags() const { + uint32_t Result = 0; + if (HasCategoryClassProperties) + Result |= HAS_CATEGORY_CLASS_PROPERTIES; + if (HasSignedObjCClassROs) + Result |= SIGNED_CLASS_RO; + Result |= (SwiftABIVersion << 8); + Result |= (SwiftVersion << 16); + return Result; + } +}; } // end anonymous namespace namespace llvm { @@ -419,6 +496,29 @@ bool MachOPlatform::supportedTarget(const Triple &TT) { } } +jitlink::Edge::Kind MachOPlatform::getPointerEdgeKind(jitlink::LinkGraph &G) { + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + return jitlink::aarch64::Pointer64; + case Triple::x86_64: + return jitlink::x86_64::Pointer64; + default: + llvm_unreachable("Unsupported architecture"); + } +} + +MachOPlatform::MachOExecutorSymbolFlags +MachOPlatform::flagsForSymbol(jitlink::Symbol &Sym) { + MachOPlatform::MachOExecutorSymbolFlags Flags{}; + if (Sym.getLinkage() == jitlink::Linkage::Weak) + Flags |= MachOExecutorSymbolFlags::Weak; + + if (Sym.isCallable()) + Flags |= MachOExecutorSymbolFlags::Callable; + + return Flags; +} + MachOPlatform::MachOPlatform( ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, @@ -442,11 +542,11 @@ MachOPlatform::MachOPlatform( // itself (to build the allocation actions that will call the registration // functions). Further complicating the situation (a) the graph containing // the registration functions is allowed to depend on other graphs (e.g. the - // graph containing the ORC runtime RTTI support) so we need to handle with - // an unknown set of dependencies during bootstrap, and (b) these graphs may + // graph containing the ORC runtime RTTI support) so we need to handle an + // unknown set of dependencies during bootstrap, and (b) these graphs may // be linked concurrently if the user has installed a concurrent dispatcher. // - // We satisfy these constraint by implementing a bootstrap phase during which + // We satisfy these constraints by implementing a bootstrap phase during which // allocation actions generated by MachOPlatform are appended to a list of // deferred allocation actions, rather than to the graphs themselves. At the // end of the bootstrap process the deferred actions are attached to a final @@ -498,6 +598,8 @@ MachOPlatform::MachOPlatform( SymbolLookupSet( {PlatformBootstrap.Name, PlatformShutdown.Name, RegisterJITDylib.Name, DeregisterJITDylib.Name, + RegisterObjectSymbolTable.Name, + DeregisterObjectSymbolTable.Name, RegisterObjectPlatformSections.Name, DeregisterObjectPlatformSections.Name, CreatePThreadKey.Name})) @@ -516,9 +618,11 @@ MachOPlatform::MachOPlatform( if ((Err = PlatformJD.define( std::make_unique<MachOPlatformCompleteBootstrapMaterializationUnit>( *this, PlatformJD.getName(), BootstrapCompleteSymbol, - std::move(BI.DeferredAAs), PlatformBootstrap.Addr, + std::move(BI.SymTab), std::move(BI.DeferredAAs), + BI.MachOHeaderAddr, PlatformBootstrap.Addr, PlatformShutdown.Addr, RegisterJITDylib.Addr, - DeregisterJITDylib.Addr, BI.MachOHeaderAddr)))) + DeregisterJITDylib.Addr, RegisterObjectSymbolTable.Addr, + DeregisterObjectSymbolTable.Addr)))) return; if ((Err = ES.lookup(makeJITDylibSearchOrder( &PlatformJD, JITDylibLookupFlags::MatchAllSymbols), @@ -540,11 +644,11 @@ Error MachOPlatform::associateRuntimeSupportFunctions() { ES.wrapAsyncWithSPS<PushInitializersSPSSig>( this, &MachOPlatform::rt_pushInitializers); - using LookupSymbolSPSSig = - SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString); - WFs[ES.intern("___orc_rt_macho_symbol_lookup_tag")] = - ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this, - &MachOPlatform::rt_lookupSymbol); + using PushSymbolsSPSSig = + SPSError(SPSExecutorAddr, SPSSequence<SPSTuple<SPSString, bool>>); + WFs[ES.intern("___orc_rt_macho_push_symbols_tag")] = + ES.wrapAsyncWithSPS<PushSymbolsSPSSig>(this, + &MachOPlatform::rt_pushSymbols); return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs)); } @@ -665,11 +769,9 @@ void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, pushInitializersLoop(std::move(SendResult), JD); } -void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, - ExecutorAddr Handle, StringRef SymbolName) { - LLVM_DEBUG({ - dbgs() << "MachOPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"; - }); +void MachOPlatform::rt_pushSymbols( + PushSymbolsInSendResultFn SendResult, ExecutorAddr Handle, + const std::vector<std::pair<StringRef, bool>> &SymbolNames) { JITDylib *JD = nullptr; @@ -679,39 +781,37 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, if (I != HeaderAddrToJITDylib.end()) JD = I->second; } + LLVM_DEBUG({ + dbgs() << "MachOPlatform::rt_pushSymbols("; + if (JD) + dbgs() << "\"" << JD->getName() << "\", [ "; + else + dbgs() << "<invalid handle " << Handle << ">, [ "; + for (auto &Name : SymbolNames) + dbgs() << "\"" << Name.first << "\" "; + dbgs() << "])\n"; + }); if (!JD) { - LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } - // Use functor class to work around XL build compiler issue on AIX. - class RtLookupNotifyComplete { - public: - RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult) - : SendResult(std::move(SendResult)) {} - void operator()(Expected<SymbolMap> Result) { - if (Result) { - assert(Result->size() == 1 && "Unexpected result map count"); - SendResult(Result->begin()->second.getAddress()); - } else { - SendResult(Result.takeError()); - } - } + SymbolLookupSet LS; + for (auto &[Name, Required] : SymbolNames) + LS.add(ES.intern(Name), Required + ? SymbolLookupFlags::RequiredSymbol + : SymbolLookupFlags::WeaklyReferencedSymbol); - private: - SendSymbolAddressFn SendResult; - }; - - // FIXME: Proper mangling. - auto MangledName = ("_" + SymbolName).str(); ES.lookup( LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, - SymbolLookupSet(ES.intern(MangledName)), SymbolState::Ready, - RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister); + std::move(LS), SymbolState::Ready, + [SendResult = std::move(SendResult)](Expected<SymbolMap> Result) mutable { + SendResult(Result.takeError()); + }, + NoDependenciesToRegister); } Expected<uint64_t> MachOPlatform::createPThreadKey() { @@ -781,6 +881,18 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( return fixTLVSectionsAndEdges(G, JD); }); + // Add symbol table prepare and register passes: These will add strings for + // all symbols to the c-strings section, and build a symbol table registration + // call. + auto JITSymTabInfo = std::make_shared<JITSymTabVector>(); + Config.PostPrunePasses.push_back([this, JITSymTabInfo](LinkGraph &G) { + return prepareSymbolTableRegistration(G, *JITSymTabInfo); + }); + Config.PostFixupPasses.push_back([this, &MR, JITSymTabInfo, + InBootstrapPhase](LinkGraph &G) { + return addSymbolTableRegistration(G, MR, *JITSymTabInfo, InBootstrapPhase); + }); + // Add a pass to register the final addresses of any special sections in the // object with the runtime. Config.PostAllocationPasses.push_back( @@ -826,6 +938,9 @@ Error MachOPlatform::MachOPlatformPlugin:: {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr}, {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr}, {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr}, + {*MP.RegisterObjectSymbolTable.Name, &MP.RegisterObjectSymbolTable.Addr}, + {*MP.DeregisterObjectSymbolTable.Name, + &MP.DeregisterObjectSymbolTable.Addr}, {*MP.RegisterObjectPlatformSections.Name, &MP.RegisterObjectPlatformSections.Addr}, {*MP.DeregisterObjectPlatformSections.Name, @@ -1029,15 +1144,19 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( " does not match first registered version", inconvertibleErrorCode()); if (ObjCImageInfoItr->second.Flags != Flags) - return make_error<StringError>("ObjC flags in " + G.getName() + - " do not match first registered flags", - inconvertibleErrorCode()); + if (Error E = mergeImageInfoFlags(G, MR, ObjCImageInfoItr->second, Flags)) + return E; // __objc_imageinfo is valid. Delete the block. for (auto *S : ObjCImageInfo->symbols()) G.removeDefinedSymbol(*S); G.removeBlock(ObjCImageInfoBlock); } else { + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Registered __objc_imageinfo for " + << MR.getTargetJITDylib().getName() << " in " << G.getName() + << "; flags = " << formatv("{0:x4}", Flags) << "\n"; + }); // We haven't registered an __objc_imageinfo section yet. Register and // move on. The section should already be marked no-dead-strip. G.addDefinedSymbol(ObjCImageInfoBlock, 0, ObjCImageInfoSymbolName, @@ -1047,12 +1166,66 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( {{MR.getExecutionSession().intern(ObjCImageInfoSymbolName), JITSymbolFlags()}})) return Err; - ObjCImageInfos[&MR.getTargetJITDylib()] = {Version, Flags}; + ObjCImageInfos[&MR.getTargetJITDylib()] = {Version, Flags, false}; } return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::mergeImageInfoFlags( + jitlink::LinkGraph &G, MaterializationResponsibility &MR, + ObjCImageInfo &Info, uint32_t NewFlags) { + if (Info.Flags == NewFlags) + return Error::success(); + + ObjCImageInfoFlags Old(Info.Flags); + ObjCImageInfoFlags New(NewFlags); + + // Check for incompatible flags. + if (Old.SwiftABIVersion && New.SwiftABIVersion && + Old.SwiftABIVersion != New.SwiftABIVersion) + return make_error<StringError>("Swift ABI version in " + G.getName() + + " does not match first registered flags", + inconvertibleErrorCode()); + + if (Old.HasCategoryClassProperties != New.HasCategoryClassProperties) + return make_error<StringError>("ObjC category class property support in " + + G.getName() + + " does not match first registered flags", + inconvertibleErrorCode()); + if (Old.HasSignedObjCClassROs != New.HasSignedObjCClassROs) + return make_error<StringError>("ObjC class_ro_t pointer signing in " + + G.getName() + + " does not match first registered flags", + inconvertibleErrorCode()); + + // If we cannot change the flags, ignore any remaining differences. Adding + // Swift or changing its version are unlikely to cause problems in practice. + if (Info.Finalized) + return Error::success(); + + // Use the minimum Swift version. + if (Old.SwiftVersion && New.SwiftVersion) + New.SwiftVersion = std::min(Old.SwiftVersion, New.SwiftVersion); + else if (Old.SwiftVersion) + New.SwiftVersion = Old.SwiftVersion; + // Add a Swift ABI version if it was pure objc before. + if (!New.SwiftABIVersion) + New.SwiftABIVersion = Old.SwiftABIVersion; + + LLVM_DEBUG({ + dbgs() << "MachOPlatform: Merging __objc_imageinfo flags for " + << MR.getTargetJITDylib().getName() << " (was " + << formatv("{0:x4}", Old.rawFlags()) << ")" + << " with " << G.getName() << " (" << formatv("{0:x4}", NewFlags) + << ")" + << " -> " << formatv("{0:x4}", New.rawFlags()) << "\n"; + }); + + Info.Flags = New.rawFlags(); + return Error::success(); +} + Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( jitlink::LinkGraph &G, JITDylib &JD) { @@ -1250,15 +1423,6 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( UI->CompactUnwindSection); if (!MachOPlatformSecs.empty() || UnwindInfo) { - ExecutorAddr HeaderAddr; - { - std::lock_guard<std::mutex> Lock(MP.PlatformMutex); - auto I = MP.JITDylibToHeaderAddr.find(&JD); - assert(I != MP.JITDylibToHeaderAddr.end() && - "Missing header for JITDylib"); - HeaderAddr = I->second; - } - // Dump the scraped inits. LLVM_DEBUG({ dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; @@ -1276,6 +1440,15 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( ? G.allocActions() : MP.Bootstrap.load()->DeferredAAs; + ExecutorAddr HeaderAddr; + { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + auto I = MP.JITDylibToHeaderAddr.find(&JD); + assert(I != MP.JITDylibToHeaderAddr.end() && + "No header registered for JD"); + assert(I->second && "Null header registered for JD"); + HeaderAddr = I->second; + } allocActions.push_back( {cantFail( WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>( @@ -1374,17 +1547,7 @@ Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( strcpy(SD.Sec.segname, "__DATA"); SD.Sec.size = 8; SD.AddFixups = [&](size_t RecordOffset) { - jitlink::Edge::Kind PointerEdge = jitlink::Edge::Invalid; - switch (G.getTargetTriple().getArch()) { - case Triple::aarch64: - PointerEdge = jitlink::aarch64::Pointer64; - break; - case Triple::x86_64: - PointerEdge = jitlink::x86_64::Pointer64; - break; - default: - llvm_unreachable("Unsupported architecture"); - } + auto PointerEdge = getPointerEdgeKind(G); // Look for an existing __objc_imageinfo symbol. jitlink::Symbol *ObjCImageInfoSym = nullptr; @@ -1403,6 +1566,24 @@ Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( for (auto *Sym : G.defined_symbols()) if (Sym->hasName() && Sym->getName() == ObjCImageInfoSymbolName) { ObjCImageInfoSym = Sym; + std::optional<uint32_t> Flags; + { + std::lock_guard<std::mutex> Lock(PluginMutex); + auto It = ObjCImageInfos.find(&MR.getTargetJITDylib()); + if (It != ObjCImageInfos.end()) { + It->second.Finalized = true; + Flags = It->second.Flags; + } + } + + if (Flags) { + // We own the definition of __objc_image_info; write the final + // merged flags value. + auto Content = Sym->getBlock().getMutableContent(G); + assert(Content.size() == 8 && + "__objc_image_info size should have been verified already"); + support::endian::write32(&Content[4], *Flags, G.getEndianness()); + } break; } if (!ObjCImageInfoSym) @@ -1460,7 +1641,7 @@ Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( auto SecContent = SecBlock.getAlreadyMutableContent(); char *P = SecContent.data(); auto WriteMachOStruct = [&](auto S) { - if (G.getEndianness() != support::endian::system_endianness()) + if (G.getEndianness() != llvm::endianness::native) MachO::swapStruct(S); memcpy(P, &S, sizeof(S)); P += sizeof(S); @@ -1492,5 +1673,87 @@ Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::prepareSymbolTableRegistration( + jitlink::LinkGraph &G, JITSymTabVector &JITSymTabInfo) { + + auto *CStringSec = G.findSectionByName(MachOCStringSectionName); + if (!CStringSec) + CStringSec = &G.createSection(MachOCStringSectionName, + MemProt::Read | MemProt::Exec); + + // Make a map of existing strings so that we can re-use them: + DenseMap<StringRef, jitlink::Symbol *> ExistingStrings; + for (auto *Sym : CStringSec->symbols()) { + + // The LinkGraph builder should have created single strings blocks, and all + // plugins should have maintained this invariant. + auto Content = Sym->getBlock().getContent(); + ExistingStrings.insert( + std::make_pair(StringRef(Content.data(), Content.size()), Sym)); + } + + // Add all symbol names to the string section, and record the symbols for + // those names. + { + SmallVector<jitlink::Symbol *> SymsToProcess; + for (auto *Sym : G.defined_symbols()) + SymsToProcess.push_back(Sym); + + for (auto *Sym : SymsToProcess) { + if (!Sym->hasName()) + continue; + + auto I = ExistingStrings.find(Sym->getName()); + if (I == ExistingStrings.end()) { + auto &NameBlock = G.createMutableContentBlock( + *CStringSec, G.allocateCString(Sym->getName()), orc::ExecutorAddr(), + 1, 0); + auto &SymbolNameSym = G.addAnonymousSymbol( + NameBlock, 0, NameBlock.getSize(), false, true); + JITSymTabInfo.push_back({Sym, &SymbolNameSym}); + } else + JITSymTabInfo.push_back({Sym, I->second}); + } + } + + return Error::success(); +} + +Error MachOPlatform::MachOPlatformPlugin::addSymbolTableRegistration( + jitlink::LinkGraph &G, MaterializationResponsibility &MR, + JITSymTabVector &JITSymTabInfo, bool InBootstrapPhase) { + + ExecutorAddr HeaderAddr; + { + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + auto I = MP.JITDylibToHeaderAddr.find(&MR.getTargetJITDylib()); + assert(I != MP.JITDylibToHeaderAddr.end() && "No header registered for JD"); + assert(I->second && "Null header registered for JD"); + HeaderAddr = I->second; + } + + SymbolTableVector LocalSymTab; + auto &SymTab = LLVM_LIKELY(!InBootstrapPhase) ? LocalSymTab + : MP.Bootstrap.load()->SymTab; + for (auto &[OriginalSymbol, NameSym] : JITSymTabInfo) + SymTab.push_back({NameSym->getAddress(), OriginalSymbol->getAddress(), + flagsForSymbol(*OriginalSymbol)}); + + // Bail out if we're in the bootstrap phase -- registration of thees symbols + // will be attached to the bootstrap graph. + if (LLVM_UNLIKELY(InBootstrapPhase)) + return Error::success(); + + shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase) + ? G.allocActions() + : MP.Bootstrap.load()->DeferredAAs; + allocActions.push_back( + {cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( + MP.RegisterObjectSymbolTable.Addr, HeaderAddr, SymTab)), + cantFail(WrapperFunctionCall::Create<SPSRegisterSymbolsArgs>( + MP.DeregisterObjectSymbolTable.Addr, HeaderAddr, SymTab))}); + + return Error::success(); +} } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp index ca4950077ffe..9cfe547c84c3 100644 --- a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -322,8 +322,8 @@ void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); tpctypes::SharedMemorySegFinalizeRequest SegReq; - SegReq.RAG = {Segment.AG.getMemProt(), Segment.AG.getMemLifetimePolicy() == - MemLifetimePolicy::Finalize}; + SegReq.RAG = {Segment.AG.getMemProt(), + Segment.AG.getMemLifetime() == MemLifetime::Finalize}; SegReq.Addr = AI.MappingBase + Segment.Offset; SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index a29f3d1c3aec..3d77f82e6569 100644 --- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -46,7 +46,7 @@ ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) { case Triple::armeb: case Triple::thumb: case Triple::thumbeb: - if (Sym.hasTargetFlags(aarch32::ThumbSymbol)) { + if (hasTargetFlags(Sym, aarch32::ThumbSymbol)) { // Set LSB to indicate thumb target assert(Sym.isCallable() && "Only callable symbols can have thumb flag"); assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear"); diff --git a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp index a73aec6d98c6..72314cceedf3 100644 --- a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp +++ b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -27,42 +27,6 @@ class InProgressLookupState; class OrcV2CAPIHelper { public: - using PoolEntry = SymbolStringPtr::PoolEntry; - using PoolEntryPtr = SymbolStringPtr::PoolEntryPtr; - - // Move from SymbolStringPtr to PoolEntryPtr (no change in ref count). - static PoolEntryPtr moveFromSymbolStringPtr(SymbolStringPtr S) { - PoolEntryPtr Result = nullptr; - std::swap(Result, S.S); - return Result; - } - - // Move from a PoolEntryPtr to a SymbolStringPtr (no change in ref count). - static SymbolStringPtr moveToSymbolStringPtr(PoolEntryPtr P) { - SymbolStringPtr S; - S.S = P; - return S; - } - - // Copy a pool entry to a SymbolStringPtr (increments ref count). - static SymbolStringPtr copyToSymbolStringPtr(PoolEntryPtr P) { - return SymbolStringPtr(P); - } - - static PoolEntryPtr getRawPoolEntryPtr(const SymbolStringPtr &S) { - return S.S; - } - - static void retainPoolEntry(PoolEntryPtr P) { - SymbolStringPtr S(P); - S.S = nullptr; - } - - static void releasePoolEntry(PoolEntryPtr P) { - SymbolStringPtr S; - S.S = P; - } - static InProgressLookupState *extractLookupState(LookupState &LS) { return LS.IPLS.release(); } @@ -75,10 +39,16 @@ public: } // namespace orc } // namespace llvm +inline LLVMOrcSymbolStringPoolEntryRef wrap(SymbolStringPoolEntryUnsafe E) { + return reinterpret_cast<LLVMOrcSymbolStringPoolEntryRef>(E.rawPtr()); +} + +inline SymbolStringPoolEntryUnsafe unwrap(LLVMOrcSymbolStringPoolEntryRef E) { + return reinterpret_cast<SymbolStringPoolEntryUnsafe::PoolEntry *>(E); +} + DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ExecutionSession, LLVMOrcExecutionSessionRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(SymbolStringPool, LLVMOrcSymbolStringPoolRef) -DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OrcV2CAPIHelper::PoolEntry, - LLVMOrcSymbolStringPoolEntryRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(MaterializationUnit, LLVMOrcMaterializationUnitRef) DEFINE_SIMPLE_CONVERSION_FUNCTIONS(MaterializationResponsibility, @@ -136,7 +106,7 @@ public: private: void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { - Discard(Ctx, wrap(&JD), wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); + Discard(Ctx, wrap(&JD), wrap(SymbolStringPoolEntryUnsafe::from(Name))); } std::string Name; @@ -184,7 +154,7 @@ static SymbolMap toSymbolMap(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs) { SymbolMap SM; for (size_t I = 0; I != NumPairs; ++I) { JITSymbolFlags Flags = toJITSymbolFlags(Syms[I].Sym.Flags); - SM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = { + SM[unwrap(Syms[I].Name).moveToSymbolStringPtr()] = { ExecutorAddr(Syms[I].Sym.Address), Flags}; } return SM; @@ -199,7 +169,7 @@ toSymbolDependenceMap(LLVMOrcCDependenceMapPairs Pairs, size_t NumPairs) { for (size_t J = 0; J != Pairs[I].Names.Length; ++J) { auto Sym = Pairs[I].Names.Symbols[J]; - Names.insert(OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Sym))); + Names.insert(unwrap(Sym).moveToSymbolStringPtr()); } SDM[JD] = Names; } @@ -309,7 +279,7 @@ public: CLookupSet.reserve(LookupSet.size()); for (auto &KV : LookupSet) { LLVMOrcSymbolStringPoolEntryRef Name = - ::wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(KV.first)); + ::wrap(SymbolStringPoolEntryUnsafe::from(KV.first)); LLVMOrcSymbolLookupFlags SLF = fromSymbolLookupFlags(KV.second); CLookupSet.push_back({Name, SLF}); } @@ -353,8 +323,7 @@ void LLVMOrcSymbolStringPoolClearDeadEntries(LLVMOrcSymbolStringPoolRef SSP) { LLVMOrcSymbolStringPoolEntryRef LLVMOrcExecutionSessionIntern(LLVMOrcExecutionSessionRef ES, const char *Name) { - return wrap( - OrcV2CAPIHelper::moveFromSymbolStringPtr(unwrap(ES)->intern(Name))); + return wrap(SymbolStringPoolEntryUnsafe::take(unwrap(ES)->intern(Name))); } void LLVMOrcExecutionSessionLookup( @@ -374,7 +343,7 @@ void LLVMOrcExecutionSessionLookup( SymbolLookupSet SLS; for (size_t I = 0; I != SymbolsSize; ++I) - SLS.add(OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Symbols[I].Name)), + SLS.add(unwrap(Symbols[I].Name).moveToSymbolStringPtr(), toSymbolLookupFlags(Symbols[I].LookupFlags)); unwrap(ES)->lookup( @@ -384,7 +353,7 @@ void LLVMOrcExecutionSessionLookup( SmallVector<LLVMOrcCSymbolMapPair> CResult; for (auto &KV : *Result) CResult.push_back(LLVMOrcCSymbolMapPair{ - wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(KV.first)), + wrap(SymbolStringPoolEntryUnsafe::from(KV.first)), fromExecutorSymbolDef(KV.second)}); HandleResult(LLVMErrorSuccess, CResult.data(), CResult.size(), Ctx); } else @@ -394,15 +363,15 @@ void LLVMOrcExecutionSessionLookup( } void LLVMOrcRetainSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { - OrcV2CAPIHelper::retainPoolEntry(unwrap(S)); + unwrap(S).retain(); } void LLVMOrcReleaseSymbolStringPoolEntry(LLVMOrcSymbolStringPoolEntryRef S) { - OrcV2CAPIHelper::releasePoolEntry(unwrap(S)); + unwrap(S).release(); } const char *LLVMOrcSymbolStringPoolEntryStr(LLVMOrcSymbolStringPoolEntryRef S) { - return unwrap(S)->getKey().data(); + return unwrap(S).rawPtr()->getKey().data(); } LLVMOrcResourceTrackerRef @@ -452,10 +421,10 @@ LLVMOrcMaterializationUnitRef LLVMOrcCreateCustomMaterializationUnit( LLVMOrcMaterializationUnitDestroyFunction Destroy) { SymbolFlagsMap SFM; for (size_t I = 0; I != NumSyms; ++I) - SFM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = + SFM[unwrap(Syms[I].Name).moveToSymbolStringPtr()] = toJITSymbolFlags(Syms[I].Flags); - auto IS = OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(InitSym)); + auto IS = unwrap(InitSym).moveToSymbolStringPtr(); return wrap(new OrcCAPIMaterializationUnit( Name, std::move(SFM), std::move(IS), Ctx, Materialize, Discard, Destroy)); @@ -476,9 +445,8 @@ LLVMOrcMaterializationUnitRef LLVMOrcLazyReexports( for (size_t I = 0; I != NumPairs; ++I) { auto pair = CallableAliases[I]; JITSymbolFlags Flags = toJITSymbolFlags(pair.Entry.Flags); - SymbolStringPtr Name = - OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(pair.Entry.Name)); - SAM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(pair.Name))] = + SymbolStringPtr Name = unwrap(pair.Entry.Name).moveToSymbolStringPtr(); + SAM[unwrap(pair.Name).moveToSymbolStringPtr()] = SymbolAliasMapEntry(Name, Flags); } @@ -511,7 +479,7 @@ LLVMOrcCSymbolFlagsMapPairs LLVMOrcMaterializationResponsibilityGetSymbols( safe_malloc(Symbols.size() * sizeof(LLVMOrcCSymbolFlagsMapPair))); size_t I = 0; for (auto const &pair : Symbols) { - auto Name = wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(pair.first)); + auto Name = wrap(SymbolStringPoolEntryUnsafe::from(pair.first)); auto Flags = pair.second; Result[I] = {Name, fromJITSymbolFlags(Flags)}; I++; @@ -528,7 +496,7 @@ LLVMOrcSymbolStringPoolEntryRef LLVMOrcMaterializationResponsibilityGetInitializerSymbol( LLVMOrcMaterializationResponsibilityRef MR) { auto Sym = unwrap(MR)->getInitializerSymbol(); - return wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Sym)); + return wrap(SymbolStringPoolEntryUnsafe::from(Sym)); } LLVMOrcSymbolStringPoolEntryRef * @@ -541,7 +509,7 @@ LLVMOrcMaterializationResponsibilityGetRequestedSymbols( Symbols.size() * sizeof(LLVMOrcSymbolStringPoolEntryRef))); size_t I = 0; for (auto &Name : Symbols) { - Result[I] = wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name)); + Result[I] = wrap(SymbolStringPoolEntryUnsafe::from(Name)); I++; } *NumSymbols = Symbols.size(); @@ -569,7 +537,7 @@ LLVMErrorRef LLVMOrcMaterializationResponsibilityDefineMaterializing( LLVMOrcCSymbolFlagsMapPairs Syms, size_t NumSyms) { SymbolFlagsMap SFM; for (size_t I = 0; I != NumSyms; ++I) - SFM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = + SFM[unwrap(Syms[I].Name).moveToSymbolStringPtr()] = toJITSymbolFlags(Syms[I].Flags); return wrap(unwrap(MR)->defineMaterializing(std::move(SFM))); @@ -588,7 +556,7 @@ LLVMErrorRef LLVMOrcMaterializationResponsibilityDelegate( LLVMOrcMaterializationResponsibilityRef *Result) { SymbolNameSet Syms; for (size_t I = 0; I != NumSymbols; I++) { - Syms.insert(OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Symbols[I]))); + Syms.insert(unwrap(Symbols[I]).moveToSymbolStringPtr()); } auto OtherMR = unwrap(MR)->delegate(Syms); @@ -605,7 +573,7 @@ void LLVMOrcMaterializationResponsibilityAddDependencies( LLVMOrcCDependenceMapPairs Dependencies, size_t NumPairs) { SymbolDependenceMap SDM = toSymbolDependenceMap(Dependencies, NumPairs); - auto Sym = OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Name)); + auto Sym = unwrap(Name).moveToSymbolStringPtr(); unwrap(MR)->addDependencies(Sym, SDM); } @@ -698,7 +666,7 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForProcess( DynamicLibrarySearchGenerator::SymbolPredicate Pred; if (Filter) Pred = [=](const SymbolStringPtr &Name) -> bool { - return Filter(FilterCtx, wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); + return Filter(FilterCtx, wrap(SymbolStringPoolEntryUnsafe::from(Name))); }; auto ProcessSymsGenerator = @@ -724,7 +692,7 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForPath( DynamicLibrarySearchGenerator::SymbolPredicate Pred; if (Filter) Pred = [=](const SymbolStringPtr &Name) -> bool { - return Filter(FilterCtx, wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(Name))); + return Filter(FilterCtx, wrap(SymbolStringPoolEntryUnsafe::from(Name))); }; auto LibrarySymsGenerator = @@ -992,7 +960,7 @@ char LLVMOrcLLJITGetGlobalPrefix(LLVMOrcLLJITRef J) { LLVMOrcSymbolStringPoolEntryRef LLVMOrcLLJITMangleAndIntern(LLVMOrcLLJITRef J, const char *UnmangledName) { - return wrap(OrcV2CAPIHelper::moveFromSymbolStringPtr( + return wrap(SymbolStringPoolEntryUnsafe::take( unwrap(J)->mangleAndIntern(UnmangledName))); } diff --git a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 9ef333222028..f9630161b95e 100644 --- a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -233,7 +233,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( if (auto *COFFObj = dyn_cast<object::COFFObjectFile>(&Obj)) { auto &ES = getExecutionSession(); - // For all resolved symbols that are not already in the responsibilty set: + // For all resolved symbols that are not already in the responsibility set: // check whether the symbol is in a comdat section and if so mark it as // weak. for (auto &Sym : COFFObj->symbols()) { diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp index ecf5e2915773..1d9d23e64158 100644 --- a/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp @@ -19,6 +19,7 @@ StringRef MachODataCommonSectionName = "__DATA,__common"; StringRef MachODataDataSectionName = "__DATA,__data"; StringRef MachOEHFrameSectionName = "__TEXT,__eh_frame"; StringRef MachOCompactUnwindInfoSectionName = "__TEXT,__unwind_info"; +StringRef MachOCStringSectionName = "__TEXT,__cstring"; StringRef MachOModInitFuncSectionName = "__DATA,__mod_init_func"; StringRef MachOObjCCatListSectionName = "__DATA,__objc_catlist"; StringRef MachOObjCCatList2SectionName = "__DATA,__objc_catlist2"; @@ -56,7 +57,19 @@ StringRef MachOInitSectionNames[19] = { }; StringRef ELFEHFrameSectionName = ".eh_frame"; + StringRef ELFInitArrayFuncSectionName = ".init_array"; +StringRef ELFInitFuncSectionName = ".init"; +StringRef ELFFiniArrayFuncSectionName = ".fini_array"; +StringRef ELFFiniFuncSectionName = ".fini"; +StringRef ELFCtorArrayFuncSectionName = ".ctors"; +StringRef ELFDtorArrayFuncSectionName = ".dtors"; + +StringRef ELFInitSectionNames[3]{ + ELFInitArrayFuncSectionName, + ELFInitFuncSectionName, + ELFCtorArrayFuncSectionName, +}; StringRef ELFThreadBSSSectionName = ".tbss"; StringRef ELFThreadDataSectionName = ".tdata"; @@ -80,9 +93,11 @@ bool isMachOInitializerSection(StringRef QualifiedName) { } bool isELFInitializerSection(StringRef SecName) { - if (SecName.consume_front(ELFInitArrayFuncSectionName) && - (SecName.empty() || SecName[0] == '.')) - return true; + for (StringRef InitSection : ELFInitSectionNames) { + StringRef Name = SecName; + if (Name.consume_front(InitSection) && (Name.empty() || Name[0] == '.')) + return true; + } return false; } diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp index 86e31c52100e..ae39b1d1bfaa 100644 --- a/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -51,9 +51,9 @@ const char *MemoryWriteBuffersWrapperName = "__llvm_orc_bootstrap_mem_write_buffers_wrapper"; const char *RegisterEHFrameSectionWrapperName = - "__llvm_orc_bootstrap_register_ehframe_section_wrapper"; + "llvm_orc_registerEHFrameSectionWrapper"; const char *DeregisterEHFrameSectionWrapperName = - "__llvm_orc_bootstrap_deregister_ehframe_section_wrapper"; + "llvm_orc_deregisterEHFrameSectionWrapper"; const char *RunAsMainWrapperName = "__llvm_orc_bootstrap_run_as_main_wrapper"; const char *RunAsVoidFunctionWrapperName = diff --git a/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp b/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp index 0388725dfb63..8f42de91b5bb 100644 --- a/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp +++ b/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp @@ -10,7 +10,6 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" @@ -227,7 +226,7 @@ void SequenceBBQuery::traverseToExitBlock(const BasicBlock *AtBB, VisitedBlocks); } -// Get Block frequencies for blocks and take most frquently executed block, +// Get Block frequencies for blocks and take most frequently executed block, // walk towards the entry block from those blocks and discover the basic blocks // with call. SequenceBBQuery::BlockListTy diff --git a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp index d4cbd1970d8f..70b536d2feda 100644 --- a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp @@ -67,7 +67,7 @@ void IRSpeculationLayer::emit(std::unique_ptr<MaterializationResponsibility> R, auto SpeculatorVTy = StructType::create(MContext, "Class.Speculator"); auto RuntimeCallTy = FunctionType::get( Type::getVoidTy(MContext), - {SpeculatorVTy->getPointerTo(), Type::getInt64Ty(MContext)}, false); + {PointerType::getUnqual(MContext), Type::getInt64Ty(MContext)}, false); auto RuntimeCall = Function::Create(RuntimeCallTy, Function::LinkageTypes::ExternalLinkage, "__orc_speculate_for", &M); diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp index 3f70dbf60437..e8b0e240ac1f 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp @@ -194,9 +194,7 @@ Error ExecutorSharedMemoryMapperService::deinitialize( // Remove the allocation from the allocation list of its reservation for (auto &Reservation : Reservations) { - auto AllocationIt = - std::find(Reservation.second.Allocations.begin(), - Reservation.second.Allocations.end(), Base); + auto AllocationIt = llvm::find(Reservation.second.Allocations, Base); if (AllocationIt != Reservation.second.Allocations.end()) { Reservation.second.Allocations.erase(AllocationIt); break; diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.cpp new file mode 100644 index 000000000000..5e0623102d33 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.cpp @@ -0,0 +1,457 @@ +//===------- JITLoaderPerf.cpp - Register profiler objects ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Register objects for access by profilers via the perf JIT interface. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderPerf.h" + +#include "llvm/ExecutionEngine/Orc/Shared/PerfSharedStructs.h" + +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/Threading.h" + +#include <mutex> +#include <optional> + +#ifdef __linux__ + +#include <sys/mman.h> // mmap() +#include <time.h> // clock_gettime(), time(), localtime_r() */ +#include <unistd.h> // for read(), close() + +#define DEBUG_TYPE "orc" + +// language identifier (XXX: should we generate something better from debug +// info?) +#define JIT_LANG "llvm-IR" +#define LLVM_PERF_JIT_MAGIC \ + ((uint32_t)'J' << 24 | (uint32_t)'i' << 16 | (uint32_t)'T' << 8 | \ + (uint32_t)'D') +#define LLVM_PERF_JIT_VERSION 1 + +using namespace llvm; +using namespace llvm::orc; + +struct PerfState { + // cache lookups + uint32_t Pid; + + // base directory for output data + std::string JitPath; + + // output data stream, closed via Dumpstream + int DumpFd = -1; + + // output data stream + std::unique_ptr<raw_fd_ostream> Dumpstream; + + // perf mmap marker + void *MarkerAddr = NULL; +}; + +// prevent concurrent dumps from messing up the output file +static std::mutex Mutex; +static std::optional<PerfState> State; + +struct RecHeader { + uint32_t Id; + uint32_t TotalSize; + uint64_t Timestamp; +}; + +struct DIR { + RecHeader Prefix; + uint64_t CodeAddr; + uint64_t NrEntry; +}; + +struct DIE { + uint64_t CodeAddr; + uint32_t Line; + uint32_t Discrim; +}; + +struct CLR { + RecHeader Prefix; + uint32_t Pid; + uint32_t Tid; + uint64_t Vma; + uint64_t CodeAddr; + uint64_t CodeSize; + uint64_t CodeIndex; +}; + +struct UWR { + RecHeader Prefix; + uint64_t UnwindDataSize; + uint64_t EhFrameHeaderSize; + uint64_t MappedSize; +}; + +static inline uint64_t timespec_to_ns(const struct timespec *TS) { + const uint64_t NanoSecPerSec = 1000000000; + return ((uint64_t)TS->tv_sec * NanoSecPerSec) + TS->tv_nsec; +} + +static inline uint64_t perf_get_timestamp() { + timespec TS; + if (clock_gettime(CLOCK_MONOTONIC, &TS)) + return 0; + + return timespec_to_ns(&TS); +} + +static void writeDebugRecord(const PerfJITDebugInfoRecord &DebugRecord) { + assert(State && "PerfState not initialized"); + LLVM_DEBUG(dbgs() << "Writing debug record with " + << DebugRecord.Entries.size() << " entries\n"); + [[maybe_unused]] size_t Written = 0; + DIR Dir{RecHeader{static_cast<uint32_t>(DebugRecord.Prefix.Id), + DebugRecord.Prefix.TotalSize, perf_get_timestamp()}, + DebugRecord.CodeAddr, DebugRecord.Entries.size()}; + State->Dumpstream->write(reinterpret_cast<const char *>(&Dir), sizeof(Dir)); + Written += sizeof(Dir); + for (auto &Die : DebugRecord.Entries) { + DIE d{Die.Addr, Die.Lineno, Die.Discrim}; + State->Dumpstream->write(reinterpret_cast<const char *>(&d), sizeof(d)); + State->Dumpstream->write(Die.Name.data(), Die.Name.size() + 1); + Written += sizeof(d) + Die.Name.size() + 1; + } + LLVM_DEBUG(dbgs() << "wrote " << Written << " bytes of debug info\n"); +} + +static void writeCodeRecord(const PerfJITCodeLoadRecord &CodeRecord) { + assert(State && "PerfState not initialized"); + uint32_t Tid = get_threadid(); + LLVM_DEBUG(dbgs() << "Writing code record with code size " + << CodeRecord.CodeSize << " and code index " + << CodeRecord.CodeIndex << "\n"); + CLR Clr{RecHeader{static_cast<uint32_t>(CodeRecord.Prefix.Id), + CodeRecord.Prefix.TotalSize, perf_get_timestamp()}, + State->Pid, + Tid, + CodeRecord.Vma, + CodeRecord.CodeAddr, + CodeRecord.CodeSize, + CodeRecord.CodeIndex}; + LLVM_DEBUG(dbgs() << "wrote " << sizeof(Clr) << " bytes of CLR, " + << CodeRecord.Name.size() + 1 << " bytes of name, " + << CodeRecord.CodeSize << " bytes of code\n"); + State->Dumpstream->write(reinterpret_cast<const char *>(&Clr), sizeof(Clr)); + State->Dumpstream->write(CodeRecord.Name.data(), CodeRecord.Name.size() + 1); + State->Dumpstream->write((const char *)CodeRecord.CodeAddr, + CodeRecord.CodeSize); +} + +static void +writeUnwindRecord(const PerfJITCodeUnwindingInfoRecord &UnwindRecord) { + assert(State && "PerfState not initialized"); + dbgs() << "Writing unwind record with unwind data size " + << UnwindRecord.UnwindDataSize << " and EH frame header size " + << UnwindRecord.EHFrameHdrSize << " and mapped size " + << UnwindRecord.MappedSize << "\n"; + UWR Uwr{RecHeader{static_cast<uint32_t>(UnwindRecord.Prefix.Id), + UnwindRecord.Prefix.TotalSize, perf_get_timestamp()}, + UnwindRecord.UnwindDataSize, UnwindRecord.EHFrameHdrSize, + UnwindRecord.MappedSize}; + LLVM_DEBUG(dbgs() << "wrote " << sizeof(Uwr) << " bytes of UWR, " + << UnwindRecord.EHFrameHdrSize + << " bytes of EH frame header, " + << UnwindRecord.UnwindDataSize - UnwindRecord.EHFrameHdrSize + << " bytes of EH frame\n"); + State->Dumpstream->write(reinterpret_cast<const char *>(&Uwr), sizeof(Uwr)); + if (UnwindRecord.EHFrameHdrAddr) + State->Dumpstream->write((const char *)UnwindRecord.EHFrameHdrAddr, + UnwindRecord.EHFrameHdrSize); + else + State->Dumpstream->write(UnwindRecord.EHFrameHdr.data(), + UnwindRecord.EHFrameHdrSize); + State->Dumpstream->write((const char *)UnwindRecord.EHFrameAddr, + UnwindRecord.UnwindDataSize - + UnwindRecord.EHFrameHdrSize); +} + +static Error registerJITLoaderPerfImpl(const PerfJITRecordBatch &Batch) { + if (!State) + return make_error<StringError>("PerfState not initialized", + inconvertibleErrorCode()); + + // Serialize the batch + std::lock_guard<std::mutex> Lock(Mutex); + if (Batch.UnwindingRecord.Prefix.TotalSize > 0) + writeUnwindRecord(Batch.UnwindingRecord); + + for (const auto &DebugInfo : Batch.DebugInfoRecords) + writeDebugRecord(DebugInfo); + + for (const auto &CodeLoad : Batch.CodeLoadRecords) + writeCodeRecord(CodeLoad); + + State->Dumpstream->flush(); + + return Error::success(); +} + +struct Header { + uint32_t Magic; // characters "JiTD" + uint32_t Version; // header version + uint32_t TotalSize; // total size of header + uint32_t ElfMach; // elf mach target + uint32_t Pad1; // reserved + uint32_t Pid; + uint64_t Timestamp; // timestamp + uint64_t Flags; // flags +}; + +static Error OpenMarker(PerfState &State) { + // We mmap the jitdump to create an MMAP RECORD in perf.data file. The mmap + // is captured either live (perf record running when we mmap) or in deferred + // mode, via /proc/PID/maps. The MMAP record is used as a marker of a jitdump + // file for more meta data info about the jitted code. Perf report/annotate + // detect this special filename and process the jitdump file. + // + // Mapping must be PROT_EXEC to ensure it is captured by perf record + // even when not using -d option. + State.MarkerAddr = + ::mmap(NULL, sys::Process::getPageSizeEstimate(), PROT_READ | PROT_EXEC, + MAP_PRIVATE, State.DumpFd, 0); + + if (State.MarkerAddr == MAP_FAILED) + return make_error<llvm::StringError>("could not mmap JIT marker", + inconvertibleErrorCode()); + + return Error::success(); +} + +void CloseMarker(PerfState &State) { + if (!State.MarkerAddr) + return; + + munmap(State.MarkerAddr, sys::Process::getPageSizeEstimate()); + State.MarkerAddr = nullptr; +} + +static Expected<Header> FillMachine(PerfState &State) { + Header Hdr; + Hdr.Magic = LLVM_PERF_JIT_MAGIC; + Hdr.Version = LLVM_PERF_JIT_VERSION; + Hdr.TotalSize = sizeof(Hdr); + Hdr.Pid = State.Pid; + Hdr.Timestamp = perf_get_timestamp(); + + char Id[16]; + struct { + uint16_t e_type; + uint16_t e_machine; + } Info; + + size_t RequiredMemory = sizeof(Id) + sizeof(Info); + + ErrorOr<std::unique_ptr<MemoryBuffer>> MB = + MemoryBuffer::getFileSlice("/proc/self/exe", RequiredMemory, 0); + + // This'll not guarantee that enough data was actually read from the + // underlying file. Instead the trailing part of the buffer would be + // zeroed. Given the ELF signature check below that seems ok though, + // it's unlikely that the file ends just after that, and the + // consequence would just be that perf wouldn't recognize the + // signature. + if (!MB) + return make_error<llvm::StringError>("could not open /proc/self/exe", + MB.getError()); + + memcpy(&Id, (*MB)->getBufferStart(), sizeof(Id)); + memcpy(&Info, (*MB)->getBufferStart() + sizeof(Id), sizeof(Info)); + + // check ELF signature + if (Id[0] != 0x7f || Id[1] != 'E' || Id[2] != 'L' || Id[3] != 'F') + return make_error<llvm::StringError>("invalid ELF signature", + inconvertibleErrorCode()); + + Hdr.ElfMach = Info.e_machine; + + return Hdr; +} + +static Error InitDebuggingDir(PerfState &State) { + time_t Time; + struct tm LocalTime; + char TimeBuffer[sizeof("YYYYMMDD")]; + SmallString<64> Path; + + // search for location to dump data to + if (const char *BaseDir = getenv("JITDUMPDIR")) + Path.append(BaseDir); + else if (!sys::path::home_directory(Path)) + Path = "."; + + // create debug directory + Path += "/.debug/jit/"; + if (auto EC = sys::fs::create_directories(Path)) { + std::string ErrStr; + raw_string_ostream ErrStream(ErrStr); + ErrStream << "could not create jit cache directory " << Path << ": " + << EC.message() << "\n"; + return make_error<StringError>(std::move(ErrStr), inconvertibleErrorCode()); + } + + // create unique directory for dump data related to this process + time(&Time); + localtime_r(&Time, &LocalTime); + strftime(TimeBuffer, sizeof(TimeBuffer), "%Y%m%d", &LocalTime); + Path += JIT_LANG "-jit-"; + Path += TimeBuffer; + + SmallString<128> UniqueDebugDir; + + using sys::fs::createUniqueDirectory; + if (auto EC = createUniqueDirectory(Path, UniqueDebugDir)) { + std::string ErrStr; + raw_string_ostream ErrStream(ErrStr); + ErrStream << "could not create unique jit cache directory " + << UniqueDebugDir << ": " << EC.message() << "\n"; + return make_error<StringError>(std::move(ErrStr), inconvertibleErrorCode()); + } + + State.JitPath = std::string(UniqueDebugDir.str()); + + return Error::success(); +} + +static Error registerJITLoaderPerfStartImpl() { + PerfState Tentative; + Tentative.Pid = sys::Process::getProcessId(); + // check if clock-source is supported + if (!perf_get_timestamp()) + return make_error<StringError>("kernel does not support CLOCK_MONOTONIC", + inconvertibleErrorCode()); + + if (auto Err = InitDebuggingDir(Tentative)) + return Err; + + std::string Filename; + raw_string_ostream FilenameBuf(Filename); + FilenameBuf << Tentative.JitPath << "/jit-" << Tentative.Pid << ".dump"; + + // Need to open ourselves, because we need to hand the FD to OpenMarker() and + // raw_fd_ostream doesn't expose the FD. + using sys::fs::openFileForWrite; + if (auto EC = openFileForReadWrite(FilenameBuf.str(), Tentative.DumpFd, + sys::fs::CD_CreateNew, sys::fs::OF_None)) { + std::string ErrStr; + raw_string_ostream ErrStream(ErrStr); + ErrStream << "could not open JIT dump file " << FilenameBuf.str() << ": " + << EC.message() << "\n"; + return make_error<StringError>(std::move(ErrStr), inconvertibleErrorCode()); + } + + Tentative.Dumpstream = + std::make_unique<raw_fd_ostream>(Tentative.DumpFd, true); + + auto Header = FillMachine(Tentative); + if (!Header) + return Header.takeError(); + + // signal this process emits JIT information + if (auto Err = OpenMarker(Tentative)) + return Err; + + Tentative.Dumpstream->write(reinterpret_cast<const char *>(&Header.get()), + sizeof(*Header)); + + // Everything initialized, can do profiling now. + if (Tentative.Dumpstream->has_error()) + return make_error<StringError>("could not write JIT dump header", + inconvertibleErrorCode()); + + State = std::move(Tentative); + return Error::success(); +} + +static Error registerJITLoaderPerfEndImpl() { + if (!State) + return make_error<StringError>("PerfState not initialized", + inconvertibleErrorCode()); + + RecHeader Close; + Close.Id = static_cast<uint32_t>(PerfJITRecordType::JIT_CODE_CLOSE); + Close.TotalSize = sizeof(Close); + Close.Timestamp = perf_get_timestamp(); + State->Dumpstream->write(reinterpret_cast<const char *>(&Close), + sizeof(Close)); + if (State->MarkerAddr) + CloseMarker(*State); + + State.reset(); + return Error::success(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfImpl(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction<SPSError(SPSPerfJITRecordBatch)>::handle( + Data, Size, registerJITLoaderPerfImpl) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfStart(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction<SPSError()>::handle(Data, Size, + registerJITLoaderPerfStartImpl) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfEnd(const char *Data, uint64_t Size) { + using namespace orc::shared; + return WrapperFunction<SPSError()>::handle(Data, Size, + registerJITLoaderPerfEndImpl) + .release(); +} + +#else + +using namespace llvm; +using namespace llvm::orc; + +static Error badOS() { + using namespace llvm; + return llvm::make_error<StringError>( + "unsupported OS (perf support is only available on linux!)", + inconvertibleErrorCode()); +} + +static Error badOSBatch(PerfJITRecordBatch &Batch) { return badOS(); } + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfImpl(const char *Data, uint64_t Size) { + using namespace shared; + return WrapperFunction<SPSError(SPSPerfJITRecordBatch)>::handle(Data, Size, + badOSBatch) + .release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfStart(const char *Data, uint64_t Size) { + using namespace shared; + return WrapperFunction<SPSError()>::handle(Data, Size, badOS).release(); +} + +extern "C" llvm::orc::shared::CWrapperFunctionResult +llvm_orc_registerJITLoaderPerfEnd(const char *Data, uint64_t Size) { + using namespace shared; + return WrapperFunction<SPSError()>::handle(Data, Size, badOS).release(); +} + +#endif diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp index 67bc379f9821..a585767bf474 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp @@ -8,7 +8,9 @@ #include "llvm/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.h" +#include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/Support/FormatVariadic.h" #include "llvm/Support/Process.h" #include "llvm/TargetParser/Host.h" @@ -206,6 +208,10 @@ Error SimpleRemoteEPCServer::sendSetupMessage( "Dispatch function name should not be set"); EI.BootstrapSymbols[ExecutorSessionObjectName] = ExecutorAddr::fromPtr(this); EI.BootstrapSymbols[DispatchFnName] = ExecutorAddr::fromPtr(jitDispatchEntry); + EI.BootstrapSymbols[rt::RegisterEHFrameSectionWrapperName] = + ExecutorAddr::fromPtr(&llvm_orc_registerEHFrameSectionWrapper); + EI.BootstrapSymbols[rt::DeregisterEHFrameSectionWrapperName] = + ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper); using SPSSerialize = shared::SPSArgList<shared::SPSSimpleRemoteEPCExecutorInfo>; diff --git a/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp index 62cab22a1c45..e2b5ce49ba2e 100644 --- a/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp +++ b/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp @@ -275,7 +275,7 @@ void PerfJITEventListener::notifyObjectLoaded( SectionIndex = SectOrErr.get()->getIndex(); // According to spec debugging info has to come before loading the - // corresonding code load. + // corresponding code load. DILineInfoTable Lines = Context->getLineInfoForAddressRange( {*AddrOrErr, SectionIndex}, Size, FileLineInfoKind::AbsoluteFilePath); @@ -447,7 +447,7 @@ void PerfJITEventListener::NotifyDebug(uint64_t CodeAddr, rec.CodeAddr = CodeAddr; rec.NrEntry = Lines.size(); - // compute total size size of record (variable due to filenames) + // compute total size of record (variable due to filenames) DILineInfoTable::iterator Begin = Lines.begin(); DILineInfoTable::iterator End = Lines.end(); for (DILineInfoTable::iterator It = Begin; It != End; ++It) { diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp index bc42eebf3fec..fd11450b635b 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp @@ -269,7 +269,7 @@ RTDyldMemoryManager::getSymbolAddressInProcess(const std::string &Name) { const char *NameStr = Name.c_str(); - // DynamicLibrary::SearchForAddresOfSymbol expects an unmangled 'C' symbol + // DynamicLibrary::SearchForAddressOfSymbol expects an unmangled 'C' symbol // name so ff we're on Darwin, strip the leading '_' off. #ifdef __APPLE__ if (NameStr[0] == '_') diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index ae1bb5a1da4b..68497305f06b 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -10,9 +10,16 @@ #include "RuntimeDyldCheckerImpl.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" +#include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" #include "llvm/MC/MCInst.h" +#include "llvm/MC/MCInstPrinter.h" +#include "llvm/MC/MCInstrInfo.h" +#include "llvm/MC/MCRegisterInfo.h" +#include "llvm/MC/MCSubtargetInfo.h" +#include "llvm/MC/MCTargetOptions.h" +#include "llvm/MC/TargetRegistry.h" #include "llvm/Support/Endian.h" #include "llvm/Support/MSVCErrorWorkarounds.h" #include "llvm/Support/MemoryBuffer.h" @@ -25,6 +32,19 @@ using namespace llvm; +namespace { +struct TargetInfo { + const Target *TheTarget; + std::unique_ptr<MCSubtargetInfo> STI; + std::unique_ptr<MCRegisterInfo> MRI; + std::unique_ptr<MCAsmInfo> MAI; + std::unique_ptr<MCContext> Ctx; + std::unique_ptr<MCDisassembler> Disassembler; + std::unique_ptr<MCInstrInfo> MII; + std::unique_ptr<MCInstPrinter> InstPrinter; +}; +} // anonymous namespace + namespace llvm { // Helper class that implements the language evaluated by RuntimeDyldChecker. @@ -276,6 +296,20 @@ private: ""); unsigned OpIdx = OpIdxExpr.getValue(); + + auto printInst = [this](StringRef Symbol, MCInst Inst, + raw_string_ostream &ErrMsgStream) { + auto TT = Checker.getTripleForSymbol(Checker.getTargetFlag(Symbol)); + auto TI = getTargetInfo(TT, Checker.getCPU(), Checker.getFeatures()); + if (auto E = TI.takeError()) { + errs() << "Error obtaining instruction printer: " + << toString(std::move(E)) << "\n"; + return std::make_pair(EvalResult(ErrMsgStream.str()), ""); + } + Inst.dump_pretty(ErrMsgStream, TI->InstPrinter.get()); + return std::make_pair(EvalResult(ErrMsgStream.str()), ""); + }; + if (OpIdx >= Inst.getNumOperands()) { std::string ErrMsg; raw_string_ostream ErrMsgStream(ErrMsg); @@ -284,8 +318,8 @@ private: << "'. Instruction has only " << format("%i", Inst.getNumOperands()) << " operands.\nInstruction is:\n "; - Inst.dump_pretty(ErrMsgStream, Checker.InstPrinter); - return std::make_pair(EvalResult(ErrMsgStream.str()), ""); + + return printInst(Symbol, Inst, ErrMsgStream); } const MCOperand &Op = Inst.getOperand(OpIdx); @@ -294,9 +328,8 @@ private: raw_string_ostream ErrMsgStream(ErrMsg); ErrMsgStream << "Operand '" << format("%i", OpIdx) << "' of instruction '" << Symbol << "' is not an immediate.\nInstruction is:\n "; - Inst.dump_pretty(ErrMsgStream, Checker.InstPrinter); - return std::make_pair(EvalResult(ErrMsgStream.str()), ""); + return printInst(Symbol, Inst, ErrMsgStream); } return std::make_pair(EvalResult(Op.getImm()), RemainingExpr); @@ -422,7 +455,7 @@ private: return std::make_pair(EvalResult(StubAddr), RemainingExpr); } - // Evaluate an identiefer expr, which may be a symbol, or a call to + // Evaluate an identifier expr, which may be a symbol, or a call to // one of the builtin functions: get_insn_opcode or get_insn_length. // Return the result, plus the expression remaining to be parsed. std::pair<EvalResult, StringRef> evalIdentifierExpr(StringRef Expr, @@ -662,7 +695,7 @@ private: if (LHSResult.hasError() || RemainingExpr == "") return std::make_pair(LHSResult, RemainingExpr); - // Otherwise check if this is a binary expressioan. + // Otherwise check if this is a binary expression. BinOpToken BinOp; std::tie(BinOp, RemainingExpr) = parseBinOpToken(RemainingExpr); @@ -687,31 +720,100 @@ private: bool decodeInst(StringRef Symbol, MCInst &Inst, uint64_t &Size, int64_t Offset) const { - MCDisassembler *Dis = Checker.Disassembler; + auto TT = Checker.getTripleForSymbol(Checker.getTargetFlag(Symbol)); + auto TI = getTargetInfo(TT, Checker.getCPU(), Checker.getFeatures()); + + if (auto E = TI.takeError()) { + errs() << "Error obtaining disassembler: " << toString(std::move(E)) + << "\n"; + return false; + } + StringRef SymbolMem = Checker.getSymbolContent(Symbol); ArrayRef<uint8_t> SymbolBytes(SymbolMem.bytes_begin() + Offset, SymbolMem.size() - Offset); MCDisassembler::DecodeStatus S = - Dis->getInstruction(Inst, Size, SymbolBytes, 0, nulls()); + TI->Disassembler->getInstruction(Inst, Size, SymbolBytes, 0, nulls()); return (S == MCDisassembler::Success); } + + Expected<TargetInfo> getTargetInfo(const Triple &TT, const StringRef &CPU, + const SubtargetFeatures &TF) const { + + auto TripleName = TT.str(); + std::string ErrorStr; + const Target *TheTarget = + TargetRegistry::lookupTarget(TripleName, ErrorStr); + if (!TheTarget) + return make_error<StringError>("Error accessing target '" + TripleName + + "': " + ErrorStr, + inconvertibleErrorCode()); + + std::unique_ptr<MCSubtargetInfo> STI( + TheTarget->createMCSubtargetInfo(TripleName, CPU, TF.getString())); + if (!STI) + return make_error<StringError>("Unable to create subtarget for " + + TripleName, + inconvertibleErrorCode()); + + std::unique_ptr<MCRegisterInfo> MRI(TheTarget->createMCRegInfo(TripleName)); + if (!MRI) + return make_error<StringError>("Unable to create target register info " + "for " + + TripleName, + inconvertibleErrorCode()); + + MCTargetOptions MCOptions; + std::unique_ptr<MCAsmInfo> MAI( + TheTarget->createMCAsmInfo(*MRI, TripleName, MCOptions)); + if (!MAI) + return make_error<StringError>("Unable to create target asm info " + + TripleName, + inconvertibleErrorCode()); + + auto Ctx = std::make_unique<MCContext>(Triple(TripleName), MAI.get(), + MRI.get(), STI.get()); + + std::unique_ptr<MCDisassembler> Disassembler( + TheTarget->createMCDisassembler(*STI, *Ctx)); + if (!Disassembler) + return make_error<StringError>("Unable to create disassembler for " + + TripleName, + inconvertibleErrorCode()); + + std::unique_ptr<MCInstrInfo> MII(TheTarget->createMCInstrInfo()); + if (!MII) + return make_error<StringError>("Unable to create instruction info for" + + TripleName, + inconvertibleErrorCode()); + + std::unique_ptr<MCInstPrinter> InstPrinter(TheTarget->createMCInstPrinter( + Triple(TripleName), 0, *MAI, *MII, *MRI)); + if (!InstPrinter) + return make_error<StringError>( + "Unable to create instruction printer for" + TripleName, + inconvertibleErrorCode()); + + return TargetInfo({TheTarget, std::move(STI), std::move(MRI), + std::move(MAI), std::move(Ctx), std::move(Disassembler), + std::move(MII), std::move(InstPrinter)}); + } }; } // namespace llvm RuntimeDyldCheckerImpl::RuntimeDyldCheckerImpl( IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo, GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo, - GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, - MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, - raw_ostream &ErrStream) + GetGOTInfoFunction GetGOTInfo, llvm::endianness Endianness, Triple TT, + StringRef CPU, SubtargetFeatures TF, raw_ostream &ErrStream) : IsSymbolValid(std::move(IsSymbolValid)), GetSymbolInfo(std::move(GetSymbolInfo)), GetSectionInfo(std::move(GetSectionInfo)), GetStubInfo(std::move(GetStubInfo)), GetGOTInfo(std::move(GetGOTInfo)), - Endianness(Endianness), Disassembler(Disassembler), - InstPrinter(InstPrinter), ErrStream(ErrStream) {} + Endianness(Endianness), TT(std::move(TT)), CPU(std::move(CPU)), + TF(std::move(TF)), ErrStream(ErrStream) {} bool RuntimeDyldCheckerImpl::check(StringRef CheckExpr) const { CheckExpr = CheckExpr.trim(); @@ -822,6 +924,36 @@ StringRef RuntimeDyldCheckerImpl::getSymbolContent(StringRef Symbol) const { return {SymInfo->getContent().data(), SymInfo->getContent().size()}; } +TargetFlagsType RuntimeDyldCheckerImpl::getTargetFlag(StringRef Symbol) const { + auto SymInfo = GetSymbolInfo(Symbol); + if (!SymInfo) { + logAllUnhandledErrors(SymInfo.takeError(), errs(), "RTDyldChecker: "); + return TargetFlagsType{}; + } + return SymInfo->getTargetFlags(); +} + +Triple +RuntimeDyldCheckerImpl::getTripleForSymbol(TargetFlagsType Flag) const { + Triple TheTriple = TT; + + switch (TT.getArch()) { + case Triple::ArchType::arm: + if (~Flag & 0x1) + return TT; + TheTriple.setArchName((Twine("thumb") + TT.getArchName().substr(3)).str()); + return TheTriple; + case Triple::ArchType::thumb: + if (Flag & 0x1) + return TT; + TheTriple.setArchName((Twine("arm") + TT.getArchName().substr(5)).str()); + return TheTriple; + + default: + return TT; + } +} + std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getSectionAddr( StringRef FileName, StringRef SectionName, bool IsInsideLoad) const { @@ -884,14 +1016,13 @@ std::pair<uint64_t, std::string> RuntimeDyldCheckerImpl::getStubOrGOTAddrFor( RuntimeDyldChecker::RuntimeDyldChecker( IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo, GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo, - GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, - MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, - raw_ostream &ErrStream) + GetGOTInfoFunction GetGOTInfo, llvm::endianness Endianness, Triple TT, + StringRef CPU, SubtargetFeatures TF, raw_ostream &ErrStream) : Impl(::std::make_unique<RuntimeDyldCheckerImpl>( std::move(IsSymbolValid), std::move(GetSymbolInfo), std::move(GetSectionInfo), std::move(GetStubInfo), - std::move(GetGOTInfo), Endianness, Disassembler, InstPrinter, - ErrStream)) {} + std::move(GetGOTInfo), Endianness, std::move(TT), std::move(CPU), + std::move(TF), ErrStream)) {} RuntimeDyldChecker::~RuntimeDyldChecker() = default; diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h index f564b0035bff..9f44a9389f47 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h @@ -13,6 +13,9 @@ namespace llvm { +/// Holds target-specific properties for a symbol. +using TargetFlagsType = uint8_t; + class RuntimeDyldCheckerImpl { friend class RuntimeDyldChecker; friend class RuntimeDyldCheckerExprEval; @@ -25,12 +28,13 @@ class RuntimeDyldCheckerImpl { using GetGOTInfoFunction = RuntimeDyldChecker::GetGOTInfoFunction; public: - RuntimeDyldCheckerImpl( - IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo, - GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo, - GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, - MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, - llvm::raw_ostream &ErrStream); + RuntimeDyldCheckerImpl(IsSymbolValidFunction IsSymbolValid, + GetSymbolInfoFunction GetSymbolInfo, + GetSectionInfoFunction GetSectionInfo, + GetStubInfoFunction GetStubInfo, + GetGOTInfoFunction GetGOTInfo, + llvm::endianness Endianness, Triple TT, StringRef CPU, + SubtargetFeatures TF, llvm::raw_ostream &ErrStream); bool check(StringRef CheckExpr) const; bool checkAllRulesInBuffer(StringRef RulePrefix, MemoryBuffer *MemBuf) const; @@ -49,6 +53,11 @@ private: StringRef getSymbolContent(StringRef Symbol) const; + TargetFlagsType getTargetFlag(StringRef Symbol) const; + Triple getTripleForSymbol(TargetFlagsType Flag) const; + StringRef getCPU() const { return CPU; } + SubtargetFeatures getFeatures() const { return TF; } + std::pair<uint64_t, std::string> getSectionAddr(StringRef FileName, StringRef SectionName, bool IsInsideLoad) const; @@ -64,9 +73,10 @@ private: GetSectionInfoFunction GetSectionInfo; GetStubInfoFunction GetStubInfo; GetGOTInfoFunction GetGOTInfo; - support::endianness Endianness; - MCDisassembler *Disassembler; - MCInstPrinter *InstPrinter; + llvm::endianness Endianness; + Triple TT; + std::string CPU; + SubtargetFeatures TF; llvm::raw_ostream &ErrStream; }; } diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index d439b1b4ebfb..9fdabf310d6e 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -35,7 +35,8 @@ static void or32AArch64Imm(void *L, uint64_t Imm) { } template <class T> static void write(bool isBE, void *P, T V) { - isBE ? write<T, support::big>(P, V) : write<T, support::little>(P, V); + isBE ? write<T, llvm::endianness::big>(P, V) + : write<T, llvm::endianness::little>(P, V); } static void write32AArch64Addr(void *L, uint64_t Imm) { diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h index dfdd98cb3a34..b73d2af8c0c4 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -108,7 +108,7 @@ private: uint64_t findOrAllocGOTEntry(const RelocationValueRef &Value, unsigned GOTRelType); - // Resolve the relvative address of GOTOffset in Section ID and place + // Resolve the relative address of GOTOffset in Section ID and place // it at the given Offset void resolveGOTOffsetRelocation(unsigned SectionID, uint64_t Offset, uint64_t GOTOffset, uint32_t Type); @@ -121,8 +121,8 @@ private: // Compute the address in memory where we can find the placeholder void *computePlaceholderAddress(unsigned SectionID, uint64_t Offset) const; - // Split out common case for createing the RelocationEntry for when the relocation requires - // no particular advanced processing. + // Split out common case for creating the RelocationEntry for when the + // relocation requires no particular advanced processing. void processSimpleRelocation(unsigned SectionID, uint64_t Offset, unsigned RelType, RelocationValueRef Value); // Return matching *LO16 relocation (Mips specific) diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index 501417db421a..73e2b365f109 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -301,7 +301,7 @@ protected: // won't be interleaved between modules. It is also used in mapSectionAddress // and resolveRelocations to protect write access to internal data structures. // - // loadObject may be called on the same thread during the handling of of + // loadObject may be called on the same thread during the handling of // processRelocations, and that's OK. The handling of the relocation lists // is written in such a way as to work correctly if new elements are added to // the end of the list while the list is being processed. @@ -318,18 +318,24 @@ protected: std::string ErrorStr; void writeInt16BE(uint8_t *Addr, uint16_t Value) { - llvm::support::endian::write<uint16_t, llvm::support::unaligned>( - Addr, Value, IsTargetLittleEndian ? support::little : support::big); + llvm::support::endian::write<uint16_t>(Addr, Value, + IsTargetLittleEndian + ? llvm::endianness::little + : llvm::endianness::big); } void writeInt32BE(uint8_t *Addr, uint32_t Value) { - llvm::support::endian::write<uint32_t, llvm::support::unaligned>( - Addr, Value, IsTargetLittleEndian ? support::little : support::big); + llvm::support::endian::write<uint32_t>(Addr, Value, + IsTargetLittleEndian + ? llvm::endianness::little + : llvm::endianness::big); } void writeInt64BE(uint8_t *Addr, uint64_t Value) { - llvm::support::endian::write<uint64_t, llvm::support::unaligned>( - Addr, Value, IsTargetLittleEndian ? support::little : support::big); + llvm::support::endian::write<uint64_t>(Addr, Value, + IsTargetLittleEndian + ? llvm::endianness::little + : llvm::endianness::big); } virtual void setMipsABI(const ObjectFile &Obj) { diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h index da381986e9de..a1151b81d141 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h @@ -27,7 +27,7 @@ using namespace llvm::support::endian; namespace llvm { // This relocation type is used for handling long branch instruction -// throught the Stub. +// through the Stub. enum InternalRelocationType : unsigned { INTERNAL_REL_ARM64_LONG_BRANCH26 = 0x111, }; diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h index 22f1cf33158c..a3e66c6bc0ec 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h @@ -54,6 +54,28 @@ public: return 16; // 8-byte load instructions, 4-byte jump, 4-byte padding } + Expected<JITSymbolFlags> getJITSymbolFlags(const SymbolRef &SR) override { + + auto Flags = RuntimeDyldImpl::getJITSymbolFlags(SR); + + if (!Flags) { + return Flags.takeError(); + } + auto SectionIterOrErr = SR.getSection(); + if (!SectionIterOrErr) { + return SectionIterOrErr.takeError(); + } + SectionRef Sec = *SectionIterOrErr.get(); + const object::COFFObjectFile *COFFObjPtr = + cast<object::COFFObjectFile>(Sec.getObject()); + const coff_section *CoffSec = COFFObjPtr->getCOFFSection(Sec); + bool isThumb = CoffSec->Characteristics & COFF::IMAGE_SCN_MEM_16BIT; + + Flags->getTargetFlags() = isThumb; + + return Flags; + } + Align getStubAlignment() override { return Align(1); } Expected<object::relocation_iterator> diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h index 89156b992d87..43ce64af8685 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h @@ -134,6 +134,13 @@ public: break; } + case COFF::IMAGE_REL_AMD64_SECTION: { + assert(static_cast<int16_t>(RE.SectionID) <= INT16_MAX && "Relocation overflow"); + assert(static_cast<int16_t>(RE.SectionID) >= INT16_MIN && "Relocation underflow"); + writeBytesUnaligned(RE.SectionID, Target, 2); + break; + } + default: llvm_unreachable("Relocation type not implemented yet!"); break; |
