diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine')
76 files changed, 3711 insertions, 1147 deletions
diff --git a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp index 2a90b67bee4b..768d84501337 100644 --- a/llvm/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/llvm/lib/ExecutionEngine/ExecutionEngine.cpp @@ -34,9 +34,9 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Host.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/TargetParser/Host.h" #include <cmath> #include <cstring> #include <mutex> @@ -878,6 +878,12 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { report_fatal_error(OS.str()); } + if (auto *TETy = dyn_cast<TargetExtType>(C->getType())) { + assert(TETy->hasProperty(TargetExtType::HasZeroInit) && C->isNullValue() && + "TargetExtType only supports null constant value"); + C = Constant::getNullValue(TETy->getLayoutType()); + } + // Otherwise, we have a simple constant. GenericValue Result; switch (C->getType()->getTypeID()) { @@ -1017,6 +1023,11 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { void ExecutionEngine::StoreValueToMemory(const GenericValue &Val, GenericValue *Ptr, Type *Ty) { + // It is safe to treat TargetExtType as its layout type since the underlying + // bits are only copied and are not inspected. + if (auto *TETy = dyn_cast<TargetExtType>(Ty)) + Ty = TETy->getLayoutType(); + const unsigned StoreBytes = getDataLayout().getTypeStoreSize(Ty); switch (Ty->getTypeID()) { @@ -1068,6 +1079,9 @@ void ExecutionEngine::StoreValueToMemory(const GenericValue &Val, void ExecutionEngine::LoadValueFromMemory(GenericValue &Result, GenericValue *Ptr, Type *Ty) { + if (auto *TETy = dyn_cast<TargetExtType>(Ty)) + Ty = TETy->getLayoutType(); + const unsigned LoadBytes = getDataLayout().getTypeStoreSize(Ty); switch (Ty->getTypeID()) { diff --git a/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp b/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp index f1eeee3b3599..b5b76130c55e 100644 --- a/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp +++ b/llvm/lib/ExecutionEngine/GDBRegistrationListener.cpp @@ -176,7 +176,7 @@ void GDBJITRegistrationListener::notifyObjectLoaded( size_t Size = DebugObj.getBinary()->getMemoryBufferRef().getBufferSize(); std::lock_guard<llvm::sys::Mutex> locked(JITDebugLock); - assert(ObjectBufferMap.find(K) == ObjectBufferMap.end() && + assert(!ObjectBufferMap.contains(K) && "Second attempt to perform debug registration."); jit_code_entry* JITCodeEntry = new jit_code_entry(); diff --git a/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index 29f481a1e4e8..4f8f883a75f3 100644 --- a/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -344,6 +344,12 @@ static GenericValue lle_X_abort(FunctionType *FT, ArrayRef<GenericValue> Args) { return GenericValue(); } +// Silence warnings about sprintf. (See also +// https://github.com/llvm/llvm-project/issues/58086) +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif // int sprintf(char *, const char *, ...) - a very rough implementation to make // output useful. static GenericValue lle_X_sprintf(FunctionType *FT, @@ -425,6 +431,9 @@ static GenericValue lle_X_sprintf(FunctionType *FT, } return GV; } +#if defined(__clang__) +#pragma clang diagnostic pop +#endif // int printf(const char *, ...) - a very rough implementation to make output // useful. diff --git a/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h b/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h index 3dfe736dc5be..41a0389442d3 100644 --- a/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h +++ b/llvm/lib/ExecutionEngine/Interpreter/Interpreter.h @@ -224,8 +224,6 @@ private: // Helper functions ExecutionContext &SF); GenericValue executeBitCastInst(Value *SrcVal, Type *DstTy, ExecutionContext &SF); - GenericValue executeCastOperation(Instruction::CastOps opcode, Value *SrcVal, - Type *Ty, ExecutionContext &SF); void popStackAndReturnValueToCaller(Type *RetTy, GenericValue Result); }; diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h b/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h index 8d5e0f7314dd..5c953da7581f 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h +++ b/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h @@ -13,13 +13,13 @@ #ifndef LLVM_EXECUTIONENGINE_JITLINK_COFFDIRECTIVEPARSER_H #define LLVM_EXECUTIONENGINE_JITLINK_COFFDIRECTIVEPARSER_H -#include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/Option/Arg.h" #include "llvm/Option/ArgList.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Error.h" #include "llvm/Support/StringSaver.h" +#include "llvm/TargetParser/Triple.h" namespace llvm { namespace jitlink { diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp index 782928c26084..6668854e1a6a 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp @@ -24,13 +24,12 @@ static Triple createTripleWithCOFFFormat(Triple T) { } COFFLinkGraphBuilder::COFFLinkGraphBuilder( - const object::COFFObjectFile &Obj, Triple TT, + const object::COFFObjectFile &Obj, Triple TT, SubtargetFeatures Features, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName) - : Obj(Obj), - G(std::make_unique<LinkGraph>(Obj.getFileName().str(), - createTripleWithCOFFFormat(TT), - getPointerSize(Obj), getEndianness(Obj), - std::move(GetEdgeKindName))) { + : Obj(Obj), G(std::make_unique<LinkGraph>( + Obj.getFileName().str(), createTripleWithCOFFFormat(TT), + std::move(Features), getPointerSize(Obj), + getEndianness(Obj), std::move(GetEdgeKindName))) { LLVM_DEBUG({ dbgs() << "Created COFFLinkGraphBuilder for \"" << Obj.getFileName() << "\"\n"; @@ -135,6 +134,13 @@ Error COFFLinkGraphBuilder::graphifySections() { SectionName = *SecNameOrErr; // FIXME: Skip debug info sections + if (SectionName == ".voltbl") { + LLVM_DEBUG({ + dbgs() << " " + << "Skipping section \"" << SectionName << "\"\n"; + }); + continue; + } LLVM_DEBUG({ dbgs() << " " @@ -152,8 +158,11 @@ Error COFFLinkGraphBuilder::graphifySections() { // Look for existing sections first. auto *GraphSec = G->findSectionByName(SectionName); - if (!GraphSec) + if (!GraphSec) { GraphSec = &G->createSection(SectionName, Prot); + if ((*Sec)->Characteristics & COFF::IMAGE_SCN_LNK_REMOVE) + GraphSec->setMemLifetimePolicy(orc::MemLifetimePolicy::NoAlloc); + } if (GraphSec->getMemProt() != Prot) return make_error<JITLinkError>("MemProt should match"); @@ -287,7 +296,7 @@ Error COFFLinkGraphBuilder::handleDirectiveSection(StringRef Str) { break; } case COFF_OPT_incl: { - auto DataCopy = G->allocateString(S); + auto DataCopy = G->allocateContent(S); StringRef StrCopy(DataCopy.data(), DataCopy.size()); ExternalSymbols[StrCopy] = &G->addExternalSymbol(StrCopy, 0, false); ExternalSymbols[StrCopy]->setLive(true); diff --git a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h index 0c0a1a536deb..e64823759540 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h @@ -39,6 +39,7 @@ protected: using COFFSymbolIndex = int32_t; COFFLinkGraphBuilder(const object::COFFObjectFile &Obj, Triple TT, + SubtargetFeatures Features, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName); LinkGraph &getGraph() const { return *G; } @@ -192,6 +193,10 @@ Error COFFLinkGraphBuilder::forEachRelocation(const object::SectionRef &RelSec, Expected<StringRef> Name = Obj.getSectionName(COFFRelSect); if (!Name) return Name.takeError(); + + // Skip the unhandled metadata sections. + if (*Name == ".voltbl") + return Error::success(); LLVM_DEBUG(dbgs() << " " << *Name << ":\n"); // Lookup the link-graph node corresponding to the target section name. diff --git a/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp index b09dc769b81c..3257a2ae94f6 100644 --- a/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp @@ -181,8 +181,10 @@ private: } public: - COFFLinkGraphBuilder_x86_64(const object::COFFObjectFile &Obj, const Triple T) - : COFFLinkGraphBuilder(Obj, std::move(T), getCOFFX86RelocationKindName) {} + COFFLinkGraphBuilder_x86_64(const object::COFFObjectFile &Obj, const Triple T, + const SubtargetFeatures Features) + : COFFLinkGraphBuilder(Obj, std::move(T), std::move(Features), + getCOFFX86RelocationKindName) {} }; class COFFLinkGraphLowering_x86_64 { @@ -196,7 +198,7 @@ public: auto ImageBase = getImageBaseAddress(G, Ctx); if (!ImageBase) return ImageBase.takeError(); - E.setAddend(E.getAddend() - *ImageBase); + E.setAddend(E.getAddend() - ImageBase->getValue()); E.setKind(x86_64::Pointer32); break; } @@ -238,19 +240,19 @@ private: return SectionStartCache[&Sec]; } - Expected<JITTargetAddress> getImageBaseAddress(LinkGraph &G, - JITLinkContext &Ctx) { + Expected<orc::ExecutorAddr> getImageBaseAddress(LinkGraph &G, + JITLinkContext &Ctx) { if (this->ImageBase) return this->ImageBase; for (auto *S : G.defined_symbols()) if (S->getName() == getImageBaseSymbolName()) { - this->ImageBase = S->getAddress().getValue(); + this->ImageBase = S->getAddress(); return this->ImageBase; } JITLinkContext::LookupMap Symbols; Symbols[getImageBaseSymbolName()] = SymbolLookupFlags::RequiredSymbol; - JITTargetAddress ImageBase; + orc::ExecutorAddr ImageBase; Error Err = Error::success(); Ctx.lookup(Symbols, createLookupContinuation([&](Expected<AsyncLookupResult> LR) { @@ -259,8 +261,7 @@ private: Err = LR.takeError(); return; } - auto &ImageBaseSymbol = LR->begin()->second; - ImageBase = ImageBaseSymbol.getAddress(); + ImageBase = LR->begin()->second.getAddress(); })); if (Err) return std::move(Err); @@ -269,7 +270,7 @@ private: } DenseMap<Section *, orc::ExecutorAddr> SectionStartCache; - JITTargetAddress ImageBase = 0; + orc::ExecutorAddr ImageBase; }; Error lowerEdges_COFF_x86_64(LinkGraph &G, JITLinkContext *Ctx) { @@ -315,7 +316,12 @@ createLinkGraphFromCOFFObject_x86_64(MemoryBufferRef ObjectBuffer) { if (!COFFObj) return COFFObj.takeError(); - return COFFLinkGraphBuilder_x86_64(**COFFObj, (*COFFObj)->makeTriple()) + auto Features = (*COFFObj)->getFeatures(); + if (!Features) + return Features.takeError(); + + return COFFLinkGraphBuilder_x86_64(**COFFObj, (*COFFObj)->makeTriple(), + std::move(*Features)) .buildGraph(); } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp index ef0f19a78571..dd08a23306ff 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF.cpp @@ -13,9 +13,11 @@ #include "llvm/ExecutionEngine/JITLink/ELF.h" #include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/ELF_aarch32.h" #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h" #include "llvm/ExecutionEngine/JITLink/ELF_i386.h" #include "llvm/ExecutionEngine/JITLink/ELF_loongarch.h" +#include "llvm/ExecutionEngine/JITLink/ELF_ppc64.h" #include "llvm/ExecutionEngine/JITLink/ELF_riscv.h" #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" #include "llvm/Object/ELF.h" @@ -56,12 +58,13 @@ Expected<uint16_t> readTargetMachineArch(StringRef Buffer) { Expected<std::unique_ptr<LinkGraph>> createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer) { StringRef Buffer = ObjectBuffer.getBuffer(); - if (Buffer.size() < ELF::EI_MAG3 + 1) + if (Buffer.size() < ELF::EI_NIDENT) return make_error<JITLinkError>("Truncated ELF buffer"); if (memcmp(Buffer.data(), ELF::ElfMagic, strlen(ELF::ElfMagic)) != 0) return make_error<JITLinkError>("ELF magic not valid"); + uint8_t DataEncoding = Buffer.data()[ELF::EI_DATA]; Expected<uint16_t> TargetMachineArch = readTargetMachineArch(Buffer); if (!TargetMachineArch) return TargetMachineArch.takeError(); @@ -69,8 +72,16 @@ createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer) { switch (*TargetMachineArch) { case ELF::EM_AARCH64: return createLinkGraphFromELFObject_aarch64(ObjectBuffer); + case ELF::EM_ARM: + return createLinkGraphFromELFObject_aarch32(ObjectBuffer); case ELF::EM_LOONGARCH: return createLinkGraphFromELFObject_loongarch(ObjectBuffer); + case ELF::EM_PPC64: { + if (DataEncoding == ELF::ELFDATA2LSB) + return createLinkGraphFromELFObject_ppc64le(ObjectBuffer); + else + return createLinkGraphFromELFObject_ppc64(ObjectBuffer); + } case ELF::EM_RISCV: return createLinkGraphFromELFObject_riscv(ObjectBuffer); case ELF::EM_X86_64: @@ -90,10 +101,22 @@ void link_ELF(std::unique_ptr<LinkGraph> G, case Triple::aarch64: link_ELF_aarch64(std::move(G), std::move(Ctx)); return; + case Triple::arm: + case Triple::armeb: + case Triple::thumb: + case Triple::thumbeb: + link_ELF_aarch32(std::move(G), std::move(Ctx)); + return; case Triple::loongarch32: case Triple::loongarch64: link_ELF_loongarch(std::move(G), std::move(Ctx)); return; + case Triple::ppc64: + link_ELF_ppc64(std::move(G), std::move(Ctx)); + return; + case Triple::ppc64le: + link_ELF_ppc64le(std::move(G), std::move(Ctx)); + return; case Triple::riscv32: case Triple::riscv64: link_ELF_riscv(std::move(G), std::move(Ctx)); diff --git a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h index 953a9f512784..e72645798349 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h @@ -59,9 +59,17 @@ class ELFLinkGraphBuilder : public ELFLinkGraphBuilderBase { public: ELFLinkGraphBuilder(const object::ELFFile<ELFT> &Obj, Triple TT, - StringRef FileName, + SubtargetFeatures Features, StringRef FileName, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName); + /// Debug sections are included in the graph by default. Use + /// setProcessDebugSections(false) to ignore them if debug info is not + /// needed. + ELFLinkGraphBuilder &setProcessDebugSections(bool ProcessDebugSections) { + this->ProcessDebugSections = ProcessDebugSections; + return *this; + } + /// Attempt to construct and return the LinkGraph. Expected<std::unique_ptr<LinkGraph>> buildGraph(); @@ -83,10 +91,7 @@ protected: } Block *getGraphBlock(ELFSectionIndex SecIndex) { - auto I = GraphBlocks.find(SecIndex); - if (I == GraphBlocks.end()) - return nullptr; - return I->second; + return GraphBlocks.lookup(SecIndex); } void setGraphSymbol(ELFSymbolIndex SymIndex, Symbol &Sym) { @@ -95,19 +100,33 @@ protected: } Symbol *getGraphSymbol(ELFSymbolIndex SymIndex) { - auto I = GraphSymbols.find(SymIndex); - if (I == GraphSymbols.end()) - return nullptr; - return I->second; + return GraphSymbols.lookup(SymIndex); } Expected<std::pair<Linkage, Scope>> getSymbolLinkageAndScope(const typename ELFT::Sym &Sym, StringRef Name); + /// Set the target flags on the given Symbol. + virtual TargetFlagsType makeTargetFlags(const typename ELFT::Sym &Sym) { + return TargetFlagsType{}; + } + + /// Get the physical offset of the symbol on the target platform. + virtual orc::ExecutorAddrDiff getRawOffset(const typename ELFT::Sym &Sym, + TargetFlagsType Flags) { + return Sym.getValue(); + } + Error prepare(); Error graphifySections(); Error graphifySymbols(); + /// Override in derived classes to suppress certain sections in the link + /// graph. + virtual bool excludeSection(const typename ELFT::Shdr &Sect) const { + return false; + } + /// Traverse all matching ELFT::Rela relocation records in the given section. /// The handler function Func should be callable with this signature: /// Error(const typename ELFT::Rela &, @@ -115,8 +134,7 @@ protected: /// template <typename RelocHandlerMethod> Error forEachRelaRelocation(const typename ELFT::Shdr &RelSect, - RelocHandlerMethod &&Func, - bool ProcessDebugSections = false); + RelocHandlerMethod &&Func); /// Traverse all matching ELFT::Rel relocation records in the given section. /// The handler function Func should be callable with this signature: @@ -125,22 +143,19 @@ protected: /// template <typename RelocHandlerMethod> Error forEachRelRelocation(const typename ELFT::Shdr &RelSect, - RelocHandlerMethod &&Func, - bool ProcessDebugSections = false); + RelocHandlerMethod &&Func); /// Traverse all matching rela relocation records in the given section. /// Convenience wrapper to allow passing a member function for the handler. /// template <typename ClassT, typename RelocHandlerMethod> Error forEachRelaRelocation(const typename ELFT::Shdr &RelSect, - ClassT *Instance, RelocHandlerMethod &&Method, - bool ProcessDebugSections = false) { + ClassT *Instance, RelocHandlerMethod &&Method) { return forEachRelaRelocation( RelSect, [Instance, Method](const auto &Rel, const auto &Target, auto &GS) { return (Instance->*Method)(Rel, Target, GS); - }, - ProcessDebugSections); + }); } /// Traverse all matching rel relocation records in the given section. @@ -148,14 +163,12 @@ protected: /// template <typename ClassT, typename RelocHandlerMethod> Error forEachRelRelocation(const typename ELFT::Shdr &RelSect, - ClassT *Instance, RelocHandlerMethod &&Method, - bool ProcessDebugSections = false) { + ClassT *Instance, RelocHandlerMethod &&Method) { return forEachRelRelocation( RelSect, [Instance, Method](const auto &Rel, const auto &Target, auto &GS) { return (Instance->*Method)(Rel, Target, GS); - }, - ProcessDebugSections); + }); } const ELFFile &Obj; @@ -163,6 +176,7 @@ protected: typename ELFFile::Elf_Shdr_Range Sections; const typename ELFFile::Elf_Shdr *SymTabSec = nullptr; StringRef SectionStringTab; + bool ProcessDebugSections = true; // Maps ELF section indexes to LinkGraph Blocks. // Only SHF_ALLOC sections will have graph blocks. @@ -175,11 +189,11 @@ protected: template <typename ELFT> ELFLinkGraphBuilder<ELFT>::ELFLinkGraphBuilder( - const ELFFile &Obj, Triple TT, StringRef FileName, - LinkGraph::GetEdgeKindNameFunction GetEdgeKindName) + const ELFFile &Obj, Triple TT, SubtargetFeatures Features, + StringRef FileName, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName) : ELFLinkGraphBuilderBase(std::make_unique<LinkGraph>( - FileName.str(), Triple(std::move(TT)), ELFT::Is64Bits ? 8 : 4, - support::endianness(ELFT::TargetEndianness), + FileName.str(), Triple(std::move(TT)), std::move(Features), + ELFT::Is64Bits ? 8 : 4, support::endianness(ELFT::TargetEndianness), std::move(GetEdgeKindName))), Obj(Obj) { LLVM_DEBUG( @@ -307,23 +321,28 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySections() { auto Name = Obj.getSectionName(Sec, SectionStringTab); if (!Name) return Name.takeError(); + if (excludeSection(Sec)) { + LLVM_DEBUG({ + dbgs() << " " << SecIndex << ": Skipping section \"" << *Name + << "\" explicitly\n"; + }); + continue; + } - // If the name indicates that it's a debug section then skip it: We don't - // support those yet. - if (isDwarfSection(*Name)) { + // Skip null sections. + if (Sec.sh_type == ELF::SHT_NULL) { LLVM_DEBUG({ - dbgs() << " " << SecIndex << ": \"" << *Name - << "\" is a debug section: " - "No graph section will be created.\n"; + dbgs() << " " << SecIndex << ": has type SHT_NULL. Skipping.\n"; }); continue; } - // Skip non-SHF_ALLOC sections - if (!(Sec.sh_flags & ELF::SHF_ALLOC)) { + // If the name indicates that it's a debug section then skip it: We don't + // support those yet. + if (!ProcessDebugSections && isDwarfSection(*Name)) { LLVM_DEBUG({ dbgs() << " " << SecIndex << ": \"" << *Name - << "\" is not an SHF_ALLOC section: " + << "\" is a debug section: " "No graph section will be created.\n"; }); continue; @@ -335,16 +354,26 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySections() { }); // Get the section's memory protection flags. - orc::MemProt Prot; + orc::MemProt Prot = orc::MemProt::Read; if (Sec.sh_flags & ELF::SHF_EXECINSTR) - Prot = orc::MemProt::Read | orc::MemProt::Exec; - else - Prot = orc::MemProt::Read | orc::MemProt::Write; + Prot |= orc::MemProt::Exec; + if (Sec.sh_flags & ELF::SHF_WRITE) + Prot |= orc::MemProt::Write; // Look for existing sections first. auto *GraphSec = G->findSectionByName(*Name); - if (!GraphSec) + if (!GraphSec) { GraphSec = &G->createSection(*Name, Prot); + // Non-SHF_ALLOC sections get NoAlloc memory lifetimes. + if (!(Sec.sh_flags & ELF::SHF_ALLOC)) { + GraphSec->setMemLifetimePolicy(orc::MemLifetimePolicy::NoAlloc); + LLVM_DEBUG({ + dbgs() << " " << SecIndex << ": \"" << *Name + << "\" is not a SHF_ALLOC section. Using NoAlloc lifetime.\n"; + }); + } + } + assert(GraphSec->getMemProt() == Prot && "MemProt should match"); Block *B = nullptr; @@ -467,6 +496,9 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { << "\"\n"; }); + TargetFlagsType Flags = makeTargetFlags(Sym); + orc::ExecutorAddrDiff Offset = getRawOffset(Sym, Flags); + // In RISCV, temporary symbols (Used to generate dwarf, eh_frame // sections...) will appear in object code's symbol table, and LLVM does // not use names on these temporary symbols (RISCV gnu toolchain uses @@ -474,10 +506,13 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { // anonymous symbol. auto &GSym = Name->empty() - ? G->addAnonymousSymbol(*B, Sym.getValue(), Sym.st_size, + ? G->addAnonymousSymbol(*B, Offset, Sym.st_size, false, false) - : G->addDefinedSymbol(*B, Sym.getValue(), *Name, Sym.st_size, L, - S, Sym.getType() == ELF::STT_FUNC, false); + : G->addDefinedSymbol(*B, Offset, *Name, Sym.st_size, L, + S, Sym.getType() == ELF::STT_FUNC, + false); + + GSym.setTargetFlags(Flags); setGraphSymbol(SymIndex, GSym); } } else if (Sym.isUndefined() && Sym.isExternal()) { @@ -499,6 +534,21 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { auto &GSym = G->addExternalSymbol(*Name, Sym.st_size, Sym.getBinding() == ELF::STB_WEAK); setGraphSymbol(SymIndex, GSym); + } else if (Sym.isUndefined() && Sym.st_value == 0 && Sym.st_size == 0 && + Sym.getType() == ELF::STT_NOTYPE && + Sym.getBinding() == ELF::STB_LOCAL && Name->empty()) { + // Some relocations (e.g., R_RISCV_ALIGN) don't have a target symbol and + // use this kind of null symbol as a placeholder. + LLVM_DEBUG({ + dbgs() << " " << SymIndex << ": Creating null graph symbol\n"; + }); + + auto SymName = + G->allocateContent("__jitlink_ELF_SYM_UND_" + Twine(SymIndex)); + auto SymNameRef = StringRef(SymName.data(), SymName.size()); + auto &GSym = G->addAbsoluteSymbol(SymNameRef, orc::ExecutorAddr(0), 0, + Linkage::Strong, Scope::Local, false); + setGraphSymbol(SymIndex, GSym); } else { LLVM_DEBUG({ dbgs() << " " << SymIndex @@ -514,8 +564,7 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { template <typename ELFT> template <typename RelocHandlerFunction> Error ELFLinkGraphBuilder<ELFT>::forEachRelaRelocation( - const typename ELFT::Shdr &RelSect, RelocHandlerFunction &&Func, - bool ProcessDebugSections) { + const typename ELFT::Shdr &RelSect, RelocHandlerFunction &&Func) { // Only look into sections that store relocation entries. if (RelSect.sh_type != ELF::SHT_RELA) return Error::success(); @@ -537,6 +586,10 @@ Error ELFLinkGraphBuilder<ELFT>::forEachRelaRelocation( LLVM_DEBUG(dbgs() << " skipped (dwarf section)\n\n"); return Error::success(); } + if (excludeSection(**FixupSection)) { + LLVM_DEBUG(dbgs() << " skipped (fixup section excluded explicitly)\n\n"); + return Error::success(); + } // Lookup the link-graph node corresponding to the target section name. auto *BlockToFix = getGraphBlock(RelSect.sh_info); @@ -561,8 +614,7 @@ Error ELFLinkGraphBuilder<ELFT>::forEachRelaRelocation( template <typename ELFT> template <typename RelocHandlerFunction> Error ELFLinkGraphBuilder<ELFT>::forEachRelRelocation( - const typename ELFT::Shdr &RelSect, RelocHandlerFunction &&Func, - bool ProcessDebugSections) { + const typename ELFT::Shdr &RelSect, RelocHandlerFunction &&Func) { // Only look into sections that store relocation entries. if (RelSect.sh_type != ELF::SHT_REL) return Error::success(); @@ -584,6 +636,10 @@ Error ELFLinkGraphBuilder<ELFT>::forEachRelRelocation( LLVM_DEBUG(dbgs() << " skipped (dwarf section)\n\n"); return Error::success(); } + if (excludeSection(**FixupSection)) { + LLVM_DEBUG(dbgs() << " skipped (fixup section excluded explicitly)\n\n"); + return Error::success(); + } // Lookup the link-graph node corresponding to the target section name. auto *BlockToFix = getGraphBlock(RelSect.sh_info); diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp new file mode 100644 index 000000000000..a1bc4c853323 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch32.cpp @@ -0,0 +1,311 @@ +//===----- ELF_aarch32.cpp - JIT linker implementation for arm/thumb ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ELF/aarch32 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ELF_aarch32.h" + +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITLink/aarch32.h" +#include "llvm/Object/ELF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/TargetParser/ARMTargetParser.h" + +#include "ELFLinkGraphBuilder.h" +#include "JITLinkGeneric.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm::object; + +namespace llvm { +namespace jitlink { + +/// Translate from ELF relocation type to JITLink-internal edge kind. +Expected<aarch32::EdgeKind_aarch32> getJITLinkEdgeKind(uint32_t ELFType) { + switch (ELFType) { + case ELF::R_ARM_ABS32: + return aarch32::Data_Pointer32; + case ELF::R_ARM_REL32: + return aarch32::Data_Delta32; + case ELF::R_ARM_CALL: + return aarch32::Arm_Call; + case ELF::R_ARM_THM_CALL: + return aarch32::Thumb_Call; + case ELF::R_ARM_THM_JUMP24: + return aarch32::Thumb_Jump24; + case ELF::R_ARM_THM_MOVW_ABS_NC: + return aarch32::Thumb_MovwAbsNC; + case ELF::R_ARM_THM_MOVT_ABS: + return aarch32::Thumb_MovtAbs; + } + + return make_error<JITLinkError>( + "Unsupported aarch32 relocation " + formatv("{0:d}: ", ELFType) + + object::getELFRelocationTypeName(ELF::EM_ARM, ELFType)); +} + +/// Translate from JITLink-internal edge kind back to ELF relocation type. +Expected<uint32_t> getELFRelocationType(Edge::Kind Kind) { + switch (static_cast<aarch32::EdgeKind_aarch32>(Kind)) { + case aarch32::Data_Delta32: + return ELF::R_ARM_REL32; + case aarch32::Data_Pointer32: + return ELF::R_ARM_ABS32; + case aarch32::Arm_Call: + return ELF::R_ARM_CALL; + case aarch32::Thumb_Call: + return ELF::R_ARM_THM_CALL; + case aarch32::Thumb_Jump24: + return ELF::R_ARM_THM_JUMP24; + case aarch32::Thumb_MovwAbsNC: + return ELF::R_ARM_THM_MOVW_ABS_NC; + case aarch32::Thumb_MovtAbs: + return ELF::R_ARM_THM_MOVT_ABS; + } + + return make_error<JITLinkError>(formatv("Invalid aarch32 edge {0:d}: ", + Kind)); +} + +/// Get a human-readable name for the given ELF AArch32 edge kind. +const char *getELFAArch32EdgeKindName(Edge::Kind R) { + // No ELF-specific edge kinds yet + return aarch32::getEdgeKindName(R); +} + +class ELFJITLinker_aarch32 : public JITLinker<ELFJITLinker_aarch32> { + friend class JITLinker<ELFJITLinker_aarch32>; + +public: + ELFJITLinker_aarch32(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, PassConfiguration PassCfg, + aarch32::ArmConfig ArmCfg) + : JITLinker(std::move(Ctx), std::move(G), std::move(PassCfg)), + ArmCfg(std::move(ArmCfg)) {} + +private: + aarch32::ArmConfig ArmCfg; + + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { + return aarch32::applyFixup(G, B, E, ArmCfg); + } +}; + +template <support::endianness DataEndianness> +class ELFLinkGraphBuilder_aarch32 + : public ELFLinkGraphBuilder<ELFType<DataEndianness, false>> { +private: + using ELFT = ELFType<DataEndianness, false>; + using Base = ELFLinkGraphBuilder<ELFT>; + + bool excludeSection(const typename ELFT::Shdr &Sect) const override { + // TODO: An .ARM.exidx (Exception Index table) entry is 8-bytes in size and + // consists of 2 words. It might be sufficient to process only relocations + // in the the second word (offset 4). Please find more details in: Exception + // Handling ABI for the Arm® Architecture -> Index table entries + if (Sect.sh_type == ELF::SHT_ARM_EXIDX) + return true; + return false; + } + + Error addRelocations() override { + LLVM_DEBUG(dbgs() << "Processing relocations:\n"); + using Self = ELFLinkGraphBuilder_aarch32<DataEndianness>; + for (const auto &RelSect : Base::Sections) { + if (Error Err = Base::forEachRelRelocation(RelSect, this, + &Self::addSingleRelRelocation)) + return Err; + } + return Error::success(); + } + + Error addSingleRelRelocation(const typename ELFT::Rel &Rel, + const typename ELFT::Shdr &FixupSect, + Block &BlockToFix) { + uint32_t SymbolIndex = Rel.getSymbol(false); + auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); + if (!ObjSymbol) + return ObjSymbol.takeError(); + + Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); + if (!GraphSymbol) + return make_error<StringError>( + formatv("Could not find symbol at given index, did you add it to " + "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", + SymbolIndex, (*ObjSymbol)->st_shndx, + Base::GraphSymbols.size()), + inconvertibleErrorCode()); + + uint32_t Type = Rel.getType(false); + Expected<aarch32::EdgeKind_aarch32> Kind = getJITLinkEdgeKind(Type); + if (!Kind) + return Kind.takeError(); + + auto FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; + Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); + Edge E(*Kind, Offset, *GraphSymbol, 0); + + Expected<int64_t> Addend = + aarch32::readAddend(*Base::G, BlockToFix, E, ArmCfg); + if (!Addend) + return Addend.takeError(); + + E.setAddend(*Addend); + LLVM_DEBUG({ + dbgs() << " "; + printEdge(dbgs(), BlockToFix, E, getELFAArch32EdgeKindName(*Kind)); + dbgs() << "\n"; + }); + + BlockToFix.addEdge(std::move(E)); + return Error::success(); + } + + aarch32::ArmConfig ArmCfg; + +protected: + TargetFlagsType makeTargetFlags(const typename ELFT::Sym &Sym) override { + if (Sym.getValue() & 0x01) + return aarch32::ThumbSymbol; + return TargetFlagsType{}; + } + + orc::ExecutorAddrDiff getRawOffset(const typename ELFT::Sym &Sym, + TargetFlagsType Flags) override { + assert((makeTargetFlags(Sym) & Flags) == Flags); + static constexpr uint64_t ThumbBit = 0x01; + return Sym.getValue() & ~ThumbBit; + } + +public: + ELFLinkGraphBuilder_aarch32(StringRef FileName, + const llvm::object::ELFFile<ELFT> &Obj, Triple TT, + SubtargetFeatures Features, + aarch32::ArmConfig ArmCfg) + : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features), + FileName, getELFAArch32EdgeKindName), + ArmCfg(std::move(ArmCfg)) {} +}; + +template <aarch32::StubsFlavor Flavor> +Error buildTables_ELF_aarch32(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); + + aarch32::StubsManager<Flavor> PLT; + visitExistingEdges(G, PLT); + return Error::success(); +} + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_aarch32(MemoryBufferRef ObjectBuffer) { + LLVM_DEBUG({ + dbgs() << "Building jitlink graph for new input " + << ObjectBuffer.getBufferIdentifier() << "...\n"; + }); + + auto ELFObj = ObjectFile::createELFObjectFile(ObjectBuffer); + if (!ELFObj) + return ELFObj.takeError(); + + auto Features = (*ELFObj)->getFeatures(); + if (!Features) + return Features.takeError(); + + // Find out what exact AArch32 instruction set and features we target. + auto TT = (*ELFObj)->makeTriple(); + ARM::ArchKind AK = ARM::parseArch(TT.getArchName()); + if (AK == ARM::ArchKind::INVALID) + return make_error<JITLinkError>( + "Failed to build ELF link graph: Invalid ARM ArchKind"); + + // Resolve our internal configuration for the target. If at some point the + // CPUArch alone becomes too unprecise, we can find more details in the + // Tag_CPU_arch_profile. + aarch32::ArmConfig ArmCfg; + using namespace ARMBuildAttrs; + auto Arch = static_cast<CPUArch>(ARM::getArchAttr(AK)); + switch (Arch) { + case v7: + case v8_A: + ArmCfg = aarch32::getArmConfigForCPUArch(Arch); + assert(ArmCfg.Stubs != aarch32::Unsupported && + "Provide a config for each supported CPU"); + break; + default: + return make_error<JITLinkError>( + "Failed to build ELF link graph: Unsupported CPU arch " + + StringRef(aarch32::getCPUArchName(Arch))); + } + + // Populate the link-graph. + switch (TT.getArch()) { + case Triple::arm: + case Triple::thumb: { + auto &ELFFile = cast<ELFObjectFile<ELF32LE>>(**ELFObj).getELFFile(); + return ELFLinkGraphBuilder_aarch32<support::little>( + (*ELFObj)->getFileName(), ELFFile, TT, std::move(*Features), + ArmCfg) + .buildGraph(); + } + case Triple::armeb: + case Triple::thumbeb: { + auto &ELFFile = cast<ELFObjectFile<ELF32BE>>(**ELFObj).getELFFile(); + return ELFLinkGraphBuilder_aarch32<support::big>( + (*ELFObj)->getFileName(), ELFFile, TT, std::move(*Features), + ArmCfg) + .buildGraph(); + } + default: + return make_error<JITLinkError>( + "Failed to build ELF/aarch32 link graph: Invalid target triple " + + TT.getTriple()); + } +} + +void link_ELF_aarch32(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + const Triple &TT = G->getTargetTriple(); + + using namespace ARMBuildAttrs; + ARM::ArchKind AK = ARM::parseArch(TT.getArchName()); + auto CPU = static_cast<CPUArch>(ARM::getArchAttr(AK)); + aarch32::ArmConfig ArmCfg = aarch32::getArmConfigForCPUArch(CPU); + + PassConfiguration PassCfg; + if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + PassCfg.PrePrunePasses.push_back(std::move(MarkLive)); + else + PassCfg.PrePrunePasses.push_back(markAllSymbolsLive); + + switch (ArmCfg.Stubs) { + case aarch32::Thumbv7: + PassCfg.PostPrunePasses.push_back( + buildTables_ELF_aarch32<aarch32::Thumbv7>); + break; + case aarch32::Unsupported: + llvm_unreachable("Check before building graph"); + } + } + + if (auto Err = Ctx->modifyPassConfig(*G, PassCfg)) + return Ctx->notifyFailed(std::move(Err)); + + ELFJITLinker_aarch32::link(std::move(Ctx), std::move(G), std::move(PassCfg), + std::move(ArmCfg)); +} + +} // namespace jitlink +} // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 567d5a4dd47a..652eb931190e 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -47,6 +47,7 @@ class ELFLinkGraphBuilder_aarch64 : public ELFLinkGraphBuilder<ELFT> { private: enum ELFAArch64RelocationKind : Edge::Kind { ELFCall26 = Edge::FirstRelocation, + ELFAdrLo21, ELFAdrPage21, ELFAddAbs12, ELFLdSt8Abs12, @@ -58,6 +59,9 @@ private: ELFMovwAbsG1, ELFMovwAbsG2, ELFMovwAbsG3, + ELFTstBr14, + ELFCondBr19, + ELFAbs32, ELFAbs64, ELFPrel32, ELFPrel64, @@ -76,6 +80,8 @@ private: case ELF::R_AARCH64_CALL26: case ELF::R_AARCH64_JUMP26: return ELFCall26; + case ELF::R_AARCH64_ADR_PREL_LO21: + return ELFAdrLo21; case ELF::R_AARCH64_ADR_PREL_PG_HI21: return ELFAdrPage21; case ELF::R_AARCH64_ADD_ABS_LO12_NC: @@ -98,6 +104,12 @@ private: return ELFMovwAbsG2; case ELF::R_AARCH64_MOVW_UABS_G3: return ELFMovwAbsG3; + case ELF::R_AARCH64_TSTBR14: + return ELFTstBr14; + case ELF::R_AARCH64_CONDBR19: + return ELFCondBr19; + case ELF::R_AARCH64_ABS32: + return ELFAbs32; case ELF::R_AARCH64_ABS64: return ELFAbs64; case ELF::R_AARCH64_PREL32: @@ -177,6 +189,15 @@ private: Kind = aarch64::Branch26PCRel; break; } + case ELFAdrLo21: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isADR(Instr)) + return make_error<JITLinkError>( + "R_AARCH64_ADR_PREL_LO21 target is not an ADR instruction"); + + Kind = aarch64::ADRLiteral21; + break; + } case ELFAdrPage21: { Kind = aarch64::Page21; break; @@ -284,6 +305,29 @@ private: Kind = aarch64::MoveWide16; break; } + case ELFTstBr14: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isTestAndBranchImm14(Instr)) + return make_error<JITLinkError>("R_AARCH64_TSTBR14 target is not a " + "test and branch instruction"); + + Kind = aarch64::TestAndBranch14PCRel; + break; + } + case ELFCondBr19: { + uint32_t Instr = *(const ulittle32_t *)FixupContent; + if (!aarch64::isCondBranchImm19(Instr) && + !aarch64::isCompAndBranchImm19(Instr)) + return make_error<JITLinkError>("R_AARCH64_CONDBR19 target is not a " + "conditional branch instruction"); + + Kind = aarch64::CondBranch19PCRel; + break; + } + case ELFAbs32: { + Kind = aarch64::Pointer32; + break; + } case ELFAbs64: { Kind = aarch64::Pointer64; break; @@ -357,6 +401,8 @@ private: return "ELFMovwAbsG2"; case ELFMovwAbsG3: return "ELFMovwAbsG3"; + case ELFAbs32: + return "ELFAbs32"; case ELFAbs64: return "ELFAbs64"; case ELFPrel32: @@ -382,9 +428,10 @@ private: public: ELFLinkGraphBuilder_aarch64(StringRef FileName, - const object::ELFFile<ELFT> &Obj, const Triple T) - : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, - aarch64::getEdgeKindName) {} + const object::ELFFile<ELFT> &Obj, Triple TT, + SubtargetFeatures Features) + : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features), + FileName, aarch64::getEdgeKindName) {} }; // TLS Info Builder. @@ -532,13 +579,17 @@ createLinkGraphFromELFObject_aarch64(MemoryBufferRef ObjectBuffer) { if (!ELFObj) return ELFObj.takeError(); + auto Features = (*ELFObj)->getFeatures(); + if (!Features) + return Features.takeError(); + assert((*ELFObj)->getArch() == Triple::aarch64 && "Only AArch64 (little endian) is supported for now"); auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); - return ELFLinkGraphBuilder_aarch64<object::ELF64LE>((*ELFObj)->getFileName(), - ELFObjFile.getELFFile(), - (*ELFObj)->makeTriple()) + return ELFLinkGraphBuilder_aarch64<object::ELF64LE>( + (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), + (*ELFObj)->makeTriple(), std::move(*Features)) .buildGraph(); } @@ -552,6 +603,7 @@ void link_ELF_aarch64(std::unique_ptr<LinkGraph> G, Config.PrePrunePasses.push_back(EHFrameEdgeFixer( ".eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); + Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); // Add a mark-live pass. if (auto MarkLive = Ctx->getMarkLivePass(TT)) diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp index 1fee1b24b6bd..860165365a7e 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp @@ -30,7 +30,8 @@ Error buildTables_ELF_i386(LinkGraph &G) { LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); i386::GOTTableManager GOT; - visitExistingEdges(G, GOT); + i386::PLTTableManager PLT(GOT); + visitExistingEdges(G, GOT, PLT); return Error::success(); } } // namespace @@ -130,6 +131,8 @@ private: return EdgeKind_i386::Delta32; case ELF::R_386_GOTOFF: return EdgeKind_i386::Delta32FromGOT; + case ELF::R_386_PLT32: + return EdgeKind_i386::BranchPCRel32; } return make_error<JITLinkError>("Unsupported i386 relocation:" + @@ -207,9 +210,9 @@ private: public: ELFLinkGraphBuilder_i386(StringRef FileName, const object::ELFFile<ELFT> &Obj, - const Triple T) - : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, - i386::getEdgeKindName) {} + Triple TT, SubtargetFeatures Features) + : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features), + FileName, i386::getEdgeKindName) {} }; Expected<std::unique_ptr<LinkGraph>> @@ -223,13 +226,17 @@ createLinkGraphFromELFObject_i386(MemoryBufferRef ObjectBuffer) { if (!ELFObj) return ELFObj.takeError(); + auto Features = (*ELFObj)->getFeatures(); + if (!Features) + return Features.takeError(); + assert((*ELFObj)->getArch() == Triple::x86 && "Only i386 (little endian) is supported for now"); auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj); - return ELFLinkGraphBuilder_i386<object::ELF32LE>((*ELFObj)->getFileName(), - ELFObjFile.getELFFile(), - (*ELFObj)->makeTriple()) + return ELFLinkGraphBuilder_i386<object::ELF32LE>( + (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), + (*ELFObj)->makeTriple(), std::move(*Features)) .buildGraph(); } @@ -243,8 +250,11 @@ void link_ELF_i386(std::unique_ptr<LinkGraph> G, else Config.PrePrunePasses.push_back(markAllSymbolsLive); - // Add an in-place GOT build pass. + // Add an in-place GOT and PLT build pass. Config.PostPrunePasses.push_back(buildTables_ELF_i386); + + // Add GOT/Stubs optimizer pass. + Config.PreFixupPasses.push_back(i386::optimizeGOTAndStubAccesses); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp index cd70217b4c0a..7f76b45aecbb 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp @@ -129,10 +129,10 @@ private: public: ELFLinkGraphBuilder_loongarch(StringRef FileName, - const object::ELFFile<ELFT> &Obj, - const Triple T) - : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, - loongarch::getEdgeKindName) {} + const object::ELFFile<ELFT> &Obj, Triple TT, + SubtargetFeatures Features) + : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features), + FileName, loongarch::getEdgeKindName) {} }; Error buildTables_ELF_loongarch(LinkGraph &G) { @@ -160,11 +160,15 @@ createLinkGraphFromELFObject_loongarch(MemoryBufferRef ObjectBuffer) { if (!ELFObj) return ELFObj.takeError(); + auto Features = (*ELFObj)->getFeatures(); + if (!Features) + return Features.takeError(); + if ((*ELFObj)->getArch() == Triple::loongarch64) { auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); return ELFLinkGraphBuilder_loongarch<object::ELF64LE>( (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), - (*ELFObj)->makeTriple()) + (*ELFObj)->makeTriple(), std::move(*Features)) .buildGraph(); } @@ -173,7 +177,7 @@ createLinkGraphFromELFObject_loongarch(MemoryBufferRef ObjectBuffer) { auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj); return ELFLinkGraphBuilder_loongarch<object::ELF32LE>( (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), - (*ELFObj)->makeTriple()) + (*ELFObj)->makeTriple(), std::move(*Features)) .buildGraph(); } diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp new file mode 100644 index 000000000000..a30b9ce51c84 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_ppc64.cpp @@ -0,0 +1,396 @@ +//===------- ELF_ppc64.cpp -JIT linker implementation for ELF/ppc64 -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ELF/ppc64 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ELF_ppc64.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" +#include "llvm/ExecutionEngine/JITLink/TableManager.h" +#include "llvm/ExecutionEngine/JITLink/ppc64.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Endian.h" + +#include "EHFrameSupportImpl.h" +#include "ELFLinkGraphBuilder.h" +#include "JITLinkGeneric.h" + +#define DEBUG_TYPE "jitlink" + +namespace { + +using namespace llvm; +using namespace llvm::jitlink; + +constexpr StringRef ELFTOCSymbolName = ".TOC."; +constexpr StringRef TOCSymbolAliasIdent = "__TOC__"; +constexpr uint64_t ELFTOCBaseOffset = 0x8000; + +template <support::endianness Endianness> +Symbol &createELFGOTHeader(LinkGraph &G, + ppc64::TOCTableManager<Endianness> &TOC) { + Symbol *TOCSymbol = nullptr; + + for (Symbol *Sym : G.defined_symbols()) + if (LLVM_UNLIKELY(Sym->getName() == ELFTOCSymbolName)) { + TOCSymbol = Sym; + break; + } + + if (LLVM_LIKELY(TOCSymbol == nullptr)) { + for (Symbol *Sym : G.external_symbols()) + if (Sym->getName() == ELFTOCSymbolName) { + TOCSymbol = Sym; + break; + } + } + + if (!TOCSymbol) + TOCSymbol = &G.addExternalSymbol(ELFTOCSymbolName, 0, false); + + return TOC.getEntryForTarget(G, *TOCSymbol); +} + +// Register preexisting GOT entries with TOC table manager. +template <support::endianness Endianness> +inline void +registerExistingGOTEntries(LinkGraph &G, + ppc64::TOCTableManager<Endianness> &TOC) { + auto isGOTEntry = [](const Edge &E) { + return E.getKind() == ppc64::Pointer64 && E.getTarget().isExternal(); + }; + if (Section *dotTOCSection = G.findSectionByName(".toc")) { + for (Block *B : dotTOCSection->blocks()) + for (Edge &E : B->edges()) + if (isGOTEntry(E)) + TOC.registerPreExistingEntry(E.getTarget(), + G.addAnonymousSymbol(*B, E.getOffset(), + G.getPointerSize(), + false, false)); + } +} + +template <support::endianness Endianness> +Error buildTables_ELF_ppc64(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); + ppc64::TOCTableManager<Endianness> TOC; + // Before visiting edges, we create a header containing the address of TOC + // base as ELFABIv2 suggests: + // > The GOT consists of an 8-byte header that contains the TOC base (the + // first TOC base when multiple TOCs are present), followed by an array of + // 8-byte addresses. + createELFGOTHeader(G, TOC); + + // There might be compiler-generated GOT entries in ELF relocatable file. + registerExistingGOTEntries(G, TOC); + + ppc64::PLTTableManager<Endianness> PLT(TOC); + visitExistingEdges(G, TOC, PLT); + // TODO: Add TLS support. + + // After visiting edges in LinkGraph, we have GOT entries built in the + // synthesized section. + // Merge sections included in TOC into synthesized TOC section, + // thus TOC is compact and reducing chances of relocation + // overflow. + if (Section *TOCSection = G.findSectionByName(TOC.getSectionName())) { + // .got and .plt are not normally present in a relocatable object file + // because they are linker generated. + if (Section *gotSection = G.findSectionByName(".got")) + G.mergeSections(*TOCSection, *gotSection); + if (Section *tocSection = G.findSectionByName(".toc")) + G.mergeSections(*TOCSection, *tocSection); + if (Section *sdataSection = G.findSectionByName(".sdata")) + G.mergeSections(*TOCSection, *sdataSection); + if (Section *sbssSection = G.findSectionByName(".sbss")) + G.mergeSections(*TOCSection, *sbssSection); + // .tocbss no longer appears in ELFABIv2. Leave it here to be compatible + // with rtdyld. + if (Section *tocbssSection = G.findSectionByName(".tocbss")) + G.mergeSections(*TOCSection, *tocbssSection); + if (Section *pltSection = G.findSectionByName(".plt")) + G.mergeSections(*TOCSection, *pltSection); + } + + return Error::success(); +} + +} // namespace + +namespace llvm::jitlink { + +template <support::endianness Endianness> +class ELFLinkGraphBuilder_ppc64 + : public ELFLinkGraphBuilder<object::ELFType<Endianness, true>> { +private: + using ELFT = object::ELFType<Endianness, true>; + using Base = ELFLinkGraphBuilder<ELFT>; + + using Base::G; // Use LinkGraph pointer from base class. + + Error addRelocations() override { + LLVM_DEBUG(dbgs() << "Processing relocations:\n"); + + using Self = ELFLinkGraphBuilder_ppc64<Endianness>; + for (const auto &RelSect : Base::Sections) { + // Validate the section to read relocation entries from. + if (RelSect.sh_type == ELF::SHT_REL) + return make_error<StringError>("No SHT_REL in valid " + + G->getTargetTriple().getArchName() + + " ELF object files", + inconvertibleErrorCode()); + + if (Error Err = Base::forEachRelaRelocation(RelSect, this, + &Self::addSingleRelocation)) + return Err; + } + + return Error::success(); + } + + Error addSingleRelocation(const typename ELFT::Rela &Rel, + const typename ELFT::Shdr &FixupSection, + Block &BlockToFix) { + using Base = ELFLinkGraphBuilder<ELFT>; + auto ELFReloc = Rel.getType(false); + + // R_PPC64_NONE is a no-op. + if (LLVM_UNLIKELY(ELFReloc == ELF::R_PPC64_NONE)) + return Error::success(); + + auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); + if (!ObjSymbol) + return ObjSymbol.takeError(); + + uint32_t SymbolIndex = Rel.getSymbol(false); + Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); + if (!GraphSymbol) + return make_error<StringError>( + formatv("Could not find symbol at given index, did you add it to " + "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", + SymbolIndex, (*ObjSymbol)->st_shndx, + Base::GraphSymbols.size()), + inconvertibleErrorCode()); + + int64_t Addend = Rel.r_addend; + orc::ExecutorAddr FixupAddress = + orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset; + Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); + Edge::Kind Kind = Edge::Invalid; + + switch (ELFReloc) { + default: + return make_error<JITLinkError>( + "In " + G->getName() + ": Unsupported ppc64 relocation type " + + object::getELFRelocationTypeName(ELF::EM_PPC64, ELFReloc)); + case ELF::R_PPC64_ADDR64: + Kind = ppc64::Pointer64; + break; + case ELF::R_PPC64_TOC16_HA: + Kind = ppc64::TOCDelta16HA; + break; + case ELF::R_PPC64_TOC16_DS: + Kind = ppc64::TOCDelta16DS; + break; + case ELF::R_PPC64_TOC16_LO: + Kind = ppc64::TOCDelta16LO; + break; + case ELF::R_PPC64_TOC16_LO_DS: + Kind = ppc64::TOCDelta16LODS; + break; + case ELF::R_PPC64_REL16: + Kind = ppc64::Delta16; + break; + case ELF::R_PPC64_REL16_HA: + Kind = ppc64::Delta16HA; + break; + case ELF::R_PPC64_REL16_LO: + Kind = ppc64::Delta16LO; + break; + case ELF::R_PPC64_REL32: + Kind = ppc64::Delta32; + break; + case ELF::R_PPC64_REL24_NOTOC: + case ELF::R_PPC64_REL24: { + bool isLocal = !GraphSymbol->isExternal(); + if (isLocal) { + // TODO: There are cases a local function call need a call stub. + // 1. Caller uses TOC, the callee doesn't, need a r2 save stub. + // 2. Caller doesn't use TOC, the callee does, need a r12 setup stub. + // FIXME: For a local call, we might need a thunk if branch target is + // out of range. + Kind = ppc64::CallBranchDelta; + // Branch to local entry. + Addend += ELF::decodePPC64LocalEntryOffset((*ObjSymbol)->st_other); + } else { + Kind = ELFReloc == ELF::R_PPC64_REL24 ? ppc64::RequestPLTCallStubSaveTOC + : ppc64::RequestPLTCallStubNoTOC; + } + break; + } + case ELF::R_PPC64_REL64: + Kind = ppc64::Delta64; + break; + } + + Edge GE(Kind, Offset, *GraphSymbol, Addend); + BlockToFix.addEdge(std::move(GE)); + return Error::success(); + } + +public: + ELFLinkGraphBuilder_ppc64(StringRef FileName, + const object::ELFFile<ELFT> &Obj, Triple TT, + SubtargetFeatures Features) + : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features), + FileName, ppc64::getEdgeKindName) {} +}; + +template <support::endianness Endianness> +class ELFJITLinker_ppc64 : public JITLinker<ELFJITLinker_ppc64<Endianness>> { + using JITLinkerBase = JITLinker<ELFJITLinker_ppc64<Endianness>>; + friend JITLinkerBase; + +public: + ELFJITLinker_ppc64(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) + : JITLinkerBase(std::move(Ctx), std::move(G), std::move(PassConfig)) { + JITLinkerBase::getPassConfig().PostAllocationPasses.push_back( + [this](LinkGraph &G) { return defineTOCBase(G); }); + } + +private: + Symbol *TOCSymbol = nullptr; + + Error defineTOCBase(LinkGraph &G) { + for (Symbol *Sym : G.defined_symbols()) { + if (LLVM_UNLIKELY(Sym->getName() == ELFTOCSymbolName)) { + TOCSymbol = Sym; + return Error::success(); + } + } + + assert(TOCSymbol == nullptr && + "TOCSymbol should not be defined at this point"); + + for (Symbol *Sym : G.external_symbols()) { + if (Sym->getName() == ELFTOCSymbolName) { + TOCSymbol = Sym; + break; + } + } + + if (Section *TOCSection = G.findSectionByName( + ppc64::TOCTableManager<Endianness>::getSectionName())) { + assert(!TOCSection->empty() && "TOC section should have reserved an " + "entry for containing the TOC base"); + + SectionRange SR(*TOCSection); + orc::ExecutorAddr TOCBaseAddr(SR.getFirstBlock()->getAddress() + + ELFTOCBaseOffset); + assert(TOCSymbol && TOCSymbol->isExternal() && + ".TOC. should be a external symbol at this point"); + G.makeAbsolute(*TOCSymbol, TOCBaseAddr); + // Create an alias of .TOC. so that rtdyld checker can recognize. + G.addAbsoluteSymbol(TOCSymbolAliasIdent, TOCSymbol->getAddress(), + TOCSymbol->getSize(), TOCSymbol->getLinkage(), + TOCSymbol->getScope(), TOCSymbol->isLive()); + return Error::success(); + } + + // If TOC section doesn't exist, which means no TOC relocation is found, we + // don't need a TOCSymbol. + return Error::success(); + } + + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { + return ppc64::applyFixup<Endianness>(G, B, E, TOCSymbol); + } +}; + +template <support::endianness Endianness> +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) { + LLVM_DEBUG({ + dbgs() << "Building jitlink graph for new input " + << ObjectBuffer.getBufferIdentifier() << "...\n"; + }); + + auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); + if (!ELFObj) + return ELFObj.takeError(); + + auto Features = (*ELFObj)->getFeatures(); + if (!Features) + return Features.takeError(); + + using ELFT = object::ELFType<Endianness, true>; + auto &ELFObjFile = cast<object::ELFObjectFile<ELFT>>(**ELFObj); + return ELFLinkGraphBuilder_ppc64<Endianness>( + (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), + (*ELFObj)->makeTriple(), std::move(*Features)) + .buildGraph(); +} + +template <support::endianness Endianness> +void link_ELF_ppc64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; + + if (Ctx->shouldAddDefaultTargetPasses(G->getTargetTriple())) { + // Construct a JITLinker and run the link function. + + // Add eh-frame passses. + Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); + Config.PrePrunePasses.push_back(EHFrameEdgeFixer( + ".eh_frame", G->getPointerSize(), ppc64::Pointer32, ppc64::Pointer64, + ppc64::Delta32, ppc64::Delta64, ppc64::NegDelta32)); + Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); + + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(G->getTargetTriple())) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + } + + Config.PostPrunePasses.push_back(buildTables_ELF_ppc64<Endianness>); + + if (auto Err = Ctx->modifyPassConfig(*G, Config)) + return Ctx->notifyFailed(std::move(Err)); + + ELFJITLinker_ppc64<Endianness>::link(std::move(Ctx), std::move(G), + std::move(Config)); +} + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_ppc64(MemoryBufferRef ObjectBuffer) { + return createLinkGraphFromELFObject_ppc64<support::big>( + std::move(ObjectBuffer)); +} + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_ppc64le(MemoryBufferRef ObjectBuffer) { + return createLinkGraphFromELFObject_ppc64<support::little>( + std::move(ObjectBuffer)); +} + +/// jit-link the given object buffer, which must be a ELF ppc64 object file. +void link_ELF_ppc64(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + return link_ELF_ppc64<support::big>(std::move(G), std::move(Ctx)); +} + +/// jit-link the given object buffer, which must be a ELF ppc64le object file. +void link_ELF_ppc64le(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + return link_ELF_ppc64<support::little>(std::move(G), std::move(Ctx)); +} + +} // end namespace llvm::jitlink diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp index 90d3bbe6a276..410dd7fedad1 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp @@ -70,13 +70,17 @@ public: } void fixPLTEdge(Edge &E, Symbol &PLTStubs) { - assert(E.getKind() == R_RISCV_CALL_PLT && "Not a R_RISCV_CALL_PLT edge?"); + assert((E.getKind() == R_RISCV_CALL || E.getKind() == R_RISCV_CALL_PLT || + E.getKind() == CallRelaxable) && + "Not a PLT edge?"); E.setKind(R_RISCV_CALL); E.setTarget(PLTStubs); } bool isExternalBranchEdge(Edge &E) const { - return E.getKind() == R_RISCV_CALL_PLT; + return (E.getKind() == R_RISCV_CALL || E.getKind() == R_RISCV_CALL_PLT || + E.getKind() == CallRelaxable) && + !E.getTarget().isDefined(); } private: @@ -229,6 +233,9 @@ private: (RawInstr & 0xFFF) | Imm20 | Imm10_1 | Imm11 | Imm19_12; break; } + case CallRelaxable: + // Treat as R_RISCV_CALL when the relaxation pass did not run + case R_RISCV_CALL_PLT: case R_RISCV_CALL: { int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; int64_t Hi = Value + 0x800; @@ -322,63 +329,52 @@ private: case R_RISCV_ADD8: { int64_t Value = (E.getTarget().getAddress() + - *(reinterpret_cast<const uint8_t *>(FixupAddress.getValue())) + - E.getAddend()) + *(reinterpret_cast<const uint8_t *>(FixupPtr)) + E.getAddend()) .getValue(); *FixupPtr = static_cast<uint8_t>(Value); break; } case R_RISCV_ADD16: { int64_t Value = (E.getTarget().getAddress() + - support::endian::read16le(reinterpret_cast<const void *>( - FixupAddress.getValue())) + - E.getAddend()) + support::endian::read16le(FixupPtr) + E.getAddend()) .getValue(); *(little16_t *)FixupPtr = static_cast<uint16_t>(Value); break; } case R_RISCV_ADD32: { int64_t Value = (E.getTarget().getAddress() + - support::endian::read32le(reinterpret_cast<const void *>( - FixupAddress.getValue())) + - E.getAddend()) + support::endian::read32le(FixupPtr) + E.getAddend()) .getValue(); *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); break; } case R_RISCV_ADD64: { int64_t Value = (E.getTarget().getAddress() + - support::endian::read64le(reinterpret_cast<const void *>( - FixupAddress.getValue())) + - E.getAddend()) + support::endian::read64le(FixupPtr) + E.getAddend()) .getValue(); *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); break; } case R_RISCV_SUB8: { - int64_t Value = - *(reinterpret_cast<const uint8_t *>(FixupAddress.getValue())) - - E.getTarget().getAddress().getValue() - E.getAddend(); + int64_t Value = *(reinterpret_cast<const uint8_t *>(FixupPtr)) - + E.getTarget().getAddress().getValue() - E.getAddend(); *FixupPtr = static_cast<uint8_t>(Value); break; } case R_RISCV_SUB16: { - int64_t Value = support::endian::read16le(reinterpret_cast<const void *>( - FixupAddress.getValue())) - + int64_t Value = support::endian::read16le(FixupPtr) - E.getTarget().getAddress().getValue() - E.getAddend(); *(little16_t *)FixupPtr = static_cast<uint32_t>(Value); break; } case R_RISCV_SUB32: { - int64_t Value = support::endian::read32le(reinterpret_cast<const void *>( - FixupAddress.getValue())) - + int64_t Value = support::endian::read32le(FixupPtr) - E.getTarget().getAddress().getValue() - E.getAddend(); *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); break; } case R_RISCV_SUB64: { - int64_t Value = support::endian::read64le(reinterpret_cast<const void *>( - FixupAddress.getValue())) - + int64_t Value = support::endian::read64le(FixupPtr) - E.getTarget().getAddress().getValue() - E.getAddend(); *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); break; @@ -419,8 +415,7 @@ private: break; } case R_RISCV_SUB6: { - int64_t Value = - *(reinterpret_cast<const uint8_t *>(FixupAddress.getValue())) & 0x3f; + int64_t Value = *(reinterpret_cast<const uint8_t *>(FixupPtr)) & 0x3f; Value -= E.getTarget().getAddress().getValue() - E.getAddend(); *FixupPtr = (*FixupPtr & 0xc0) | (static_cast<uint8_t>(Value) & 0x3f); break; @@ -458,11 +453,318 @@ private: *(little32_t *)FixupPtr = Word32; break; } + case AlignRelaxable: + // Ignore when the relaxation pass did not run + break; } return Error::success(); } }; +namespace { + +struct SymbolAnchor { + uint64_t Offset; + Symbol *Sym; + bool End; // true for the anchor of getOffset() + getSize() +}; + +struct BlockRelaxAux { + // This records symbol start and end offsets which will be adjusted according + // to the nearest RelocDeltas element. + SmallVector<SymbolAnchor, 0> Anchors; + // All edges that either 1) are R_RISCV_ALIGN or 2) have a R_RISCV_RELAX edge + // at the same offset. + SmallVector<Edge *, 0> RelaxEdges; + // For RelaxEdges[I], the actual offset is RelaxEdges[I]->getOffset() - (I ? + // RelocDeltas[I - 1] : 0). + SmallVector<uint32_t, 0> RelocDeltas; + // For RelaxEdges[I], the actual type is EdgeKinds[I]. + SmallVector<Edge::Kind, 0> EdgeKinds; + // List of rewritten instructions. Contains one raw encoded instruction per + // element in EdgeKinds that isn't Invalid or R_RISCV_ALIGN. + SmallVector<uint32_t, 0> Writes; +}; + +struct RelaxConfig { + bool IsRV32; + bool HasRVC; +}; + +struct RelaxAux { + RelaxConfig Config; + DenseMap<Block *, BlockRelaxAux> Blocks; +}; + +} // namespace + +static bool shouldRelax(const Section &S) { + return (S.getMemProt() & orc::MemProt::Exec) != orc::MemProt::None; +} + +static bool isRelaxable(const Edge &E) { + switch (E.getKind()) { + default: + return false; + case CallRelaxable: + case AlignRelaxable: + return true; + } +} + +static RelaxAux initRelaxAux(LinkGraph &G) { + RelaxAux Aux; + Aux.Config.IsRV32 = G.getTargetTriple().isRISCV32(); + const auto &Features = G.getFeatures().getFeatures(); + Aux.Config.HasRVC = + std::find(Features.begin(), Features.end(), "+c") != Features.end(); + + for (auto &S : G.sections()) { + if (!shouldRelax(S)) + continue; + for (auto *B : S.blocks()) { + auto BlockEmplaceResult = Aux.Blocks.try_emplace(B); + assert(BlockEmplaceResult.second && "Block encountered twice"); + auto &BlockAux = BlockEmplaceResult.first->second; + + for (auto &E : B->edges()) + if (isRelaxable(E)) + BlockAux.RelaxEdges.push_back(&E); + + if (BlockAux.RelaxEdges.empty()) { + Aux.Blocks.erase(BlockEmplaceResult.first); + continue; + } + + const auto NumEdges = BlockAux.RelaxEdges.size(); + BlockAux.RelocDeltas.resize(NumEdges, 0); + BlockAux.EdgeKinds.resize_for_overwrite(NumEdges); + + // Store anchors (offset and offset+size) for symbols. + for (auto *Sym : S.symbols()) { + if (!Sym->isDefined() || &Sym->getBlock() != B) + continue; + + BlockAux.Anchors.push_back({Sym->getOffset(), Sym, false}); + BlockAux.Anchors.push_back( + {Sym->getOffset() + Sym->getSize(), Sym, true}); + } + } + } + + // Sort anchors by offset so that we can find the closest relocation + // efficiently. For a zero size symbol, ensure that its start anchor precedes + // its end anchor. For two symbols with anchors at the same offset, their + // order does not matter. + for (auto &BlockAuxIter : Aux.Blocks) { + llvm::sort(BlockAuxIter.second.Anchors, [](auto &A, auto &B) { + return std::make_pair(A.Offset, A.End) < std::make_pair(B.Offset, B.End); + }); + } + + return Aux; +} + +static void relaxAlign(orc::ExecutorAddr Loc, const Edge &E, uint32_t &Remove, + Edge::Kind &NewEdgeKind) { + // E points to the start of the padding bytes. + // E + Addend points to the instruction to be aligned by removing padding. + // Alignment is the smallest power of 2 strictly greater than Addend. + const auto Align = NextPowerOf2(E.getAddend()); + const auto DestLoc = alignTo(Loc.getValue(), Align); + const auto SrcLoc = Loc.getValue() + E.getAddend(); + Remove = SrcLoc - DestLoc; + assert(static_cast<int32_t>(Remove) >= 0 && + "R_RISCV_ALIGN needs expanding the content"); + NewEdgeKind = AlignRelaxable; +} + +static void relaxCall(const Block &B, BlockRelaxAux &Aux, + const RelaxConfig &Config, orc::ExecutorAddr Loc, + const Edge &E, uint32_t &Remove, + Edge::Kind &NewEdgeKind) { + const auto JALR = + support::endian::read32le(B.getContent().data() + E.getOffset() + 4); + const auto RD = extractBits(JALR, 7, 5); + const auto Dest = E.getTarget().getAddress() + E.getAddend(); + const auto Displace = Dest - Loc; + + if (Config.HasRVC && isInt<12>(Displace) && RD == 0) { + NewEdgeKind = R_RISCV_RVC_JUMP; + Aux.Writes.push_back(0xa001); // c.j + Remove = 6; + } else if (Config.HasRVC && Config.IsRV32 && isInt<12>(Displace) && RD == 1) { + NewEdgeKind = R_RISCV_RVC_JUMP; + Aux.Writes.push_back(0x2001); // c.jal + Remove = 6; + } else if (isInt<21>(Displace)) { + NewEdgeKind = R_RISCV_JAL; + Aux.Writes.push_back(0x6f | RD << 7); // jal + Remove = 4; + } else { + // Not relaxable + NewEdgeKind = R_RISCV_CALL_PLT; + Remove = 0; + } +} + +static bool relaxBlock(LinkGraph &G, Block &Block, BlockRelaxAux &Aux, + const RelaxConfig &Config) { + const auto BlockAddr = Block.getAddress(); + bool Changed = false; + ArrayRef<SymbolAnchor> SA = ArrayRef(Aux.Anchors); + uint32_t Delta = 0; + + Aux.EdgeKinds.assign(Aux.EdgeKinds.size(), Edge::Invalid); + Aux.Writes.clear(); + + for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) { + const auto Loc = BlockAddr + E->getOffset() - Delta; + auto &Cur = Aux.RelocDeltas[I]; + uint32_t Remove = 0; + switch (E->getKind()) { + case AlignRelaxable: + relaxAlign(Loc, *E, Remove, Aux.EdgeKinds[I]); + break; + case CallRelaxable: + relaxCall(Block, Aux, Config, Loc, *E, Remove, Aux.EdgeKinds[I]); + break; + default: + llvm_unreachable("Unexpected relaxable edge kind"); + } + + // For all anchors whose offsets are <= E->getOffset(), they are preceded by + // the previous relocation whose RelocDeltas value equals Delta. + // Decrease their offset and update their size. + for (; SA.size() && SA[0].Offset <= E->getOffset(); SA = SA.slice(1)) { + if (SA[0].End) + SA[0].Sym->setSize(SA[0].Offset - Delta - SA[0].Sym->getOffset()); + else + SA[0].Sym->setOffset(SA[0].Offset - Delta); + } + + Delta += Remove; + if (Delta != Cur) { + Cur = Delta; + Changed = true; + } + } + + for (const SymbolAnchor &A : SA) { + if (A.End) + A.Sym->setSize(A.Offset - Delta - A.Sym->getOffset()); + else + A.Sym->setOffset(A.Offset - Delta); + } + + return Changed; +} + +static bool relaxOnce(LinkGraph &G, RelaxAux &Aux) { + bool Changed = false; + + for (auto &[B, BlockAux] : Aux.Blocks) + Changed |= relaxBlock(G, *B, BlockAux, Aux.Config); + + return Changed; +} + +static void finalizeBlockRelax(LinkGraph &G, Block &Block, BlockRelaxAux &Aux) { + auto Contents = Block.getAlreadyMutableContent(); + auto *Dest = Contents.data(); + auto NextWrite = Aux.Writes.begin(); + uint32_t Offset = 0; + uint32_t Delta = 0; + + // Update section content: remove NOPs for R_RISCV_ALIGN and rewrite + // instructions for relaxed relocations. + for (auto [I, E] : llvm::enumerate(Aux.RelaxEdges)) { + uint32_t Remove = Aux.RelocDeltas[I] - Delta; + Delta = Aux.RelocDeltas[I]; + if (Remove == 0 && Aux.EdgeKinds[I] == Edge::Invalid) + continue; + + // Copy from last location to the current relocated location. + const auto Size = E->getOffset() - Offset; + std::memmove(Dest, Contents.data() + Offset, Size); + Dest += Size; + + uint32_t Skip = 0; + switch (Aux.EdgeKinds[I]) { + case Edge::Invalid: + break; + case AlignRelaxable: + // For R_RISCV_ALIGN, we will place Offset in a location (among NOPs) to + // satisfy the alignment requirement. If both Remove and E->getAddend() + // are multiples of 4, it is as if we have skipped some NOPs. Otherwise we + // are in the middle of a 4-byte NOP, and we need to rewrite the NOP + // sequence. + if (Remove % 4 || E->getAddend() % 4) { + Skip = E->getAddend() - Remove; + uint32_t J = 0; + for (; J + 4 <= Skip; J += 4) + support::endian::write32le(Dest + J, 0x00000013); // nop + if (J != Skip) { + assert(J + 2 == Skip); + support::endian::write16le(Dest + J, 0x0001); // c.nop + } + } + break; + case R_RISCV_RVC_JUMP: + Skip = 2; + support::endian::write16le(Dest, *NextWrite++); + break; + case R_RISCV_JAL: + Skip = 4; + support::endian::write32le(Dest, *NextWrite++); + break; + } + + Dest += Skip; + Offset = E->getOffset() + Skip + Remove; + } + + std::memmove(Dest, Contents.data() + Offset, Contents.size() - Offset); + + // Fixup edge offsets and kinds. + Delta = 0; + size_t I = 0; + for (auto &E : Block.edges()) { + E.setOffset(E.getOffset() - Delta); + + if (I < Aux.RelaxEdges.size() && Aux.RelaxEdges[I] == &E) { + if (Aux.EdgeKinds[I] != Edge::Invalid) + E.setKind(Aux.EdgeKinds[I]); + + Delta = Aux.RelocDeltas[I]; + ++I; + } + } + + // Remove AlignRelaxable edges: all other relaxable edges got modified and + // will be used later while linking. Alignment is entirely handled here so we + // don't need these edges anymore. + for (auto IE = Block.edges().begin(); IE != Block.edges().end();) { + if (IE->getKind() == AlignRelaxable) + IE = Block.removeEdge(IE); + else + ++IE; + } +} + +static void finalizeRelax(LinkGraph &G, RelaxAux &Aux) { + for (auto &[B, BlockAux] : Aux.Blocks) + finalizeBlockRelax(G, *B, BlockAux); +} + +static Error relax(LinkGraph &G) { + auto Aux = initRelaxAux(G); + while (relaxOnce(G, Aux)) { + } + finalizeRelax(G, Aux); + return Error::success(); +} + template <typename ELFT> class ELFLinkGraphBuilder_riscv : public ELFLinkGraphBuilder<ELFT> { private: @@ -528,6 +830,8 @@ private: return EdgeKind_riscv::R_RISCV_SET32; case ELF::R_RISCV_32_PCREL: return EdgeKind_riscv::R_RISCV_32_PCREL; + case ELF::R_RISCV_ALIGN: + return EdgeKind_riscv::AlignRelaxable; } return make_error<JITLinkError>( @@ -535,6 +839,17 @@ private: object::getELFRelocationTypeName(ELF::EM_RISCV, Type)); } + EdgeKind_riscv getRelaxableRelocationKind(EdgeKind_riscv Kind) { + switch (Kind) { + default: + // Just ignore unsupported relaxations + return Kind; + case R_RISCV_CALL: + case R_RISCV_CALL_PLT: + return CallRelaxable; + } + } + Error addRelocations() override { LLVM_DEBUG(dbgs() << "Processing relocations:\n"); @@ -554,22 +869,17 @@ private: using Base = ELFLinkGraphBuilder<ELFT>; uint32_t Type = Rel.getType(false); - // We do not implement linker relaxation, except what is required for - // alignment (see below). - if (Type == llvm::ELF::R_RISCV_RELAX) - return Error::success(); - int64_t Addend = Rel.r_addend; - if (Type == llvm::ELF::R_RISCV_ALIGN) { - uint64_t Alignment = PowerOf2Ceil(Addend); - // FIXME: Implement support for ensuring alignment together with linker - // relaxation; 2 bytes are guaranteed by the length of compressed - // instructions, so this does not need any action from our side. - if (Alignment > 2) - return make_error<JITLinkError>( - formatv("Unsupported relocation R_RISCV_ALIGN with alignment {0} " - "larger than 2 (addend: {1})", - Alignment, Addend)); + + if (Type == ELF::R_RISCV_RELAX) { + if (BlockToFix.edges_empty()) + return make_error<StringError>( + "R_RISCV_RELAX without preceding relocation", + inconvertibleErrorCode()); + + auto &PrevEdge = *std::prev(BlockToFix.edges().end()); + auto Kind = static_cast<EdgeKind_riscv>(PrevEdge.getKind()); + PrevEdge.setKind(getRelaxableRelocationKind(Kind)); return Error::success(); } @@ -606,9 +916,10 @@ private: public: ELFLinkGraphBuilder_riscv(StringRef FileName, - const object::ELFFile<ELFT> &Obj, const Triple T) - : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, - riscv::getEdgeKindName) {} + const object::ELFFile<ELFT> &Obj, Triple TT, + SubtargetFeatures Features) + : ELFLinkGraphBuilder<ELFT>(Obj, std::move(TT), std::move(Features), + FileName, riscv::getEdgeKindName) {} }; Expected<std::unique_ptr<LinkGraph>> @@ -622,11 +933,15 @@ createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer) { if (!ELFObj) return ELFObj.takeError(); + auto Features = (*ELFObj)->getFeatures(); + if (!Features) + return Features.takeError(); + if ((*ELFObj)->getArch() == Triple::riscv64) { auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); return ELFLinkGraphBuilder_riscv<object::ELF64LE>( (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), - (*ELFObj)->makeTriple()) + (*ELFObj)->makeTriple(), std::move(*Features)) .buildGraph(); } else { assert((*ELFObj)->getArch() == Triple::riscv32 && @@ -634,7 +949,7 @@ createLinkGraphFromELFObject_riscv(MemoryBufferRef ObjectBuffer) { auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj); return ELFLinkGraphBuilder_riscv<object::ELF32LE>( (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), - (*ELFObj)->makeTriple()) + (*ELFObj)->makeTriple(), std::move(*Features)) .buildGraph(); } } @@ -650,6 +965,7 @@ void link_ELF_riscv(std::unique_ptr<LinkGraph> G, Config.PrePrunePasses.push_back(markAllSymbolsLive); Config.PostPrunePasses.push_back( PerGraphGOTAndPLTStubsBuilder_ELF_riscv::asPass); + Config.PostAllocationPasses.push_back(relax); } if (auto Err = Ctx->modifyPassConfig(*G, Config)) return Ctx->notifyFailed(std::move(Err)); @@ -657,5 +973,7 @@ void link_ELF_riscv(std::unique_ptr<LinkGraph> G, ELFJITLinker_riscv::link(std::move(Ctx), std::move(G), std::move(Config)); } +LinkGraphPassFunction createRelaxationPass_ELF_riscv() { return relax; } + } // namespace jitlink } // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index c9359522c248..1bdddd4c722b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -100,60 +100,10 @@ Error buildTables_ELF_x86_64(LinkGraph &G) { namespace llvm { namespace jitlink { -// This should become a template as the ELFFile is so a lot of this could become -// generic class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> { private: using ELFT = object::ELF64LE; - enum ELFX86RelocationKind : Edge::Kind { - Branch32 = Edge::FirstRelocation, - Pointer32Signed, - Pointer64, - PCRel32, - PCRel32GOTLoad, - PCRel32GOTLoadRelaxable, - PCRel32REXGOTLoadRelaxable, - PCRel32TLV, - PCRel64GOT, - GOTOFF64, - GOT64, - Delta64, - }; - - static Expected<ELFX86RelocationKind> getRelocationKind(const uint32_t Type) { - switch (Type) { - case ELF::R_X86_64_32S: - return ELFX86RelocationKind::Pointer32Signed; - case ELF::R_X86_64_PC32: - return ELFX86RelocationKind::PCRel32; - case ELF::R_X86_64_PC64: - case ELF::R_X86_64_GOTPC64: - return ELFX86RelocationKind::Delta64; - case ELF::R_X86_64_64: - return ELFX86RelocationKind::Pointer64; - case ELF::R_X86_64_GOTPCREL: - return ELFX86RelocationKind::PCRel32GOTLoad; - case ELF::R_X86_64_GOTPCRELX: - return ELFX86RelocationKind::PCRel32GOTLoadRelaxable; - case ELF::R_X86_64_REX_GOTPCRELX: - return ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable; - case ELF::R_X86_64_GOTPCREL64: - return ELFX86RelocationKind::PCRel64GOT; - case ELF::R_X86_64_GOT64: - return ELFX86RelocationKind::GOT64; - case ELF::R_X86_64_GOTOFF64: - return ELFX86RelocationKind::GOTOFF64; - case ELF::R_X86_64_PLT32: - return ELFX86RelocationKind::Branch32; - case ELF::R_X86_64_TLSGD: - return ELFX86RelocationKind::PCRel32TLV; - } - return make_error<JITLinkError>( - "Unsupported x86-64 relocation type " + formatv("{0:d}: ", Type) + - object::getELFRelocationTypeName(ELF::EM_X86_64, Type)); - } - Error addRelocations() override { LLVM_DEBUG(dbgs() << "Processing relocations:\n"); @@ -179,6 +129,12 @@ private: Block &BlockToFix) { using Base = ELFLinkGraphBuilder<ELFT>; + auto ELFReloc = Rel.getType(false); + + // R_X86_64_NONE is a no-op. + if (LLVM_UNLIKELY(ELFReloc == ELF::R_X86_64_NONE)) + return Error::success(); + uint32_t SymbolIndex = Rel.getSymbol(false); auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); if (!ObjSymbol) @@ -194,62 +150,66 @@ private: inconvertibleErrorCode()); // Validate the relocation kind. - auto ELFRelocKind = getRelocationKind(Rel.getType(false)); - if (!ELFRelocKind) - return ELFRelocKind.takeError(); - int64_t Addend = Rel.r_addend; Edge::Kind Kind = Edge::Invalid; - switch (*ELFRelocKind) { - case PCRel32: + + switch (ELFReloc) { + case ELF::R_X86_64_PC32: + case ELF::R_X86_64_GOTPC32: Kind = x86_64::Delta32; break; - case Delta64: + case ELF::R_X86_64_PC64: + case ELF::R_X86_64_GOTPC64: Kind = x86_64::Delta64; break; - case Pointer32Signed: + case ELF::R_X86_64_32: + Kind = x86_64::Pointer32; + break; + case ELF::R_X86_64_16: + Kind = x86_64::Pointer16; + break; + case ELF::R_X86_64_8: + Kind = x86_64::Pointer8; + break; + case ELF::R_X86_64_32S: Kind = x86_64::Pointer32Signed; break; - case Pointer64: + case ELF::R_X86_64_64: Kind = x86_64::Pointer64; break; - case PCRel32GOTLoad: { + case ELF::R_X86_64_GOTPCREL: Kind = x86_64::RequestGOTAndTransformToDelta32; break; - } - case PCRel32REXGOTLoadRelaxable: { + case ELF::R_X86_64_REX_GOTPCRELX: Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadREXRelaxable; Addend = 0; break; - } - case PCRel32TLV: { + case ELF::R_X86_64_TLSGD: Kind = x86_64::RequestTLSDescInGOTAndTransformToDelta32; break; - } - case PCRel32GOTLoadRelaxable: { + case ELF::R_X86_64_GOTPCRELX: Kind = x86_64::RequestGOTAndTransformToPCRel32GOTLoadRelaxable; Addend = 0; break; - } - case PCRel64GOT: { + case ELF::R_X86_64_GOTPCREL64: Kind = x86_64::RequestGOTAndTransformToDelta64; break; - } - case GOT64: { + case ELF::R_X86_64_GOT64: Kind = x86_64::RequestGOTAndTransformToDelta64FromGOT; break; - } - case GOTOFF64: { + case ELF::R_X86_64_GOTOFF64: Kind = x86_64::Delta64FromGOT; break; - } - case Branch32: { + case ELF::R_X86_64_PLT32: Kind = x86_64::BranchPCRel32; // BranchPCRel32 implicitly handles the '-4' PC adjustment, so we have to // adjust the addend by '+4' to compensate. Addend += 4; break; - } + default: + return make_error<JITLinkError>( + "In " + G->getName() + ": Unsupported x86-64 relocation type " + + object::getELFRelocationTypeName(ELF::EM_X86_64, ELFReloc)); } auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset; @@ -267,8 +227,10 @@ private: public: ELFLinkGraphBuilder_x86_64(StringRef FileName, - const object::ELFFile<object::ELF64LE> &Obj) - : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), FileName, + const object::ELFFile<object::ELF64LE> &Obj, + SubtargetFeatures Features) + : ELFLinkGraphBuilder(Obj, Triple("x86_64-unknown-linux"), + std::move(Features), FileName, x86_64::getEdgeKindName) {} }; @@ -334,6 +296,22 @@ private: Linkage::Strong, Scope::Local, false, true); } + // If we still haven't found a GOT symbol then double check the externals. + // We may have a GOT-relative reference but no GOT section, in which case + // we just need to point the GOT symbol at some address in this graph. + if (!GOTSymbol) { + for (auto *Sym : G.external_symbols()) { + if (Sym->getName() == ELFGOTSymbolName) { + auto Blocks = G.blocks(); + if (!Blocks.empty()) { + G.makeAbsolute(*Sym, (*Blocks.begin())->getAddress()); + GOTSymbol = Sym; + break; + } + } + } + } + return Error::success(); } @@ -353,9 +331,14 @@ createLinkGraphFromELFObject_x86_64(MemoryBufferRef ObjectBuffer) { if (!ELFObj) return ELFObj.takeError(); + auto Features = (*ELFObj)->getFeatures(); + if (!Features) + return Features.takeError(); + auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); return ELFLinkGraphBuilder_x86_64((*ELFObj)->getFileName(), - ELFObjFile.getELFFile()) + ELFObjFile.getELFFile(), + std::move(*Features)) .buildGraph(); } diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index bd5b4d585550..4a2755d3696b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -8,6 +8,7 @@ #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/Magic.h" #include "llvm/ExecutionEngine/JITLink/COFF.h" #include "llvm/ExecutionEngine/JITLink/ELF.h" @@ -88,6 +89,21 @@ const char *getScopeName(Scope S) { llvm_unreachable("Unrecognized llvm.jitlink.Scope enum"); } +bool isCStringBlock(Block &B) { + if (B.getSize() == 0) // Empty blocks are not valid C-strings. + return false; + + // Zero-fill blocks of size one are valid empty strings. + if (B.isZeroFill()) + return B.getSize() == 1; + + for (size_t I = 0; I != B.getSize() - 1; ++I) + if (B.getContent()[I] == '\0') + return false; + + return B.getContent()[B.getSize() - 1] == '\0'; +} + raw_ostream &operator<<(raw_ostream &OS, const Block &B) { return OS << B.getAddress() << " -- " << (B.getAddress() + B.getSize()) << ": " diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 17de84fa6e11..feaa0fb6a58c 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -65,7 +65,7 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, if (AR) Alloc = std::move(*AR); else - return Ctx->notifyFailed(AR.takeError()); + return abandonAllocAndBailOut(std::move(Self), AR.takeError()); LLVM_DEBUG({ dbgs() << "Link graph \"" << G->getName() @@ -75,13 +75,13 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, // Run post-allocation passes. if (auto Err = runPasses(Passes.PostAllocationPasses)) - return Ctx->notifyFailed(std::move(Err)); + return abandonAllocAndBailOut(std::move(Self), std::move(Err)); // Notify client that the defined symbols have been assigned addresses. LLVM_DEBUG(dbgs() << "Resolving symbols defined in " << G->getName() << "\n"); if (auto Err = Ctx->notifyResolved(*G)) - return Ctx->notifyFailed(std::move(Err)); + return abandonAllocAndBailOut(std::move(Self), std::move(Err)); auto ExternalSymbols = getExternalSymbolNames(); @@ -218,8 +218,7 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { assert(!Sym->isDefined() && "Symbol being resolved is already defined"); auto ResultI = Result.find(Sym->getName()); if (ResultI != Result.end()) { - Sym->getAddressable().setAddress( - orc::ExecutorAddr(ResultI->second.getAddress())); + Sym->getAddressable().setAddress(ResultI->second.getAddress()); Sym->setLinkage(ResultI->second.getFlags().isWeak() ? Linkage::Weak : Linkage::Strong); Sym->setScope(ResultI->second.getFlags().isExported() ? Scope::Default diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index 2c9244526536..e69eddd6e119 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -123,26 +123,47 @@ private: Error fixUpBlocks(LinkGraph &G) const override { LLVM_DEBUG(dbgs() << "Fixing up blocks:\n"); - for (auto *B : G.blocks()) { - LLVM_DEBUG(dbgs() << " " << *B << ":\n"); + for (auto &Sec : G.sections()) { + bool NoAllocSection = + Sec.getMemLifetimePolicy() == orc::MemLifetimePolicy::NoAlloc; - // Copy Block data and apply fixups. - LLVM_DEBUG(dbgs() << " Applying fixups.\n"); - assert((!B->isZeroFill() || all_of(B->edges(), - [](const Edge &E) { - return E.getKind() == - Edge::KeepAlive; - })) && - "Non-KeepAlive edges in zero-fill block?"); - for (auto &E : B->edges()) { + for (auto *B : Sec.blocks()) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); - // Skip non-relocation edges. - if (!E.isRelocation()) - continue; + // Copy Block data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + assert((!B->isZeroFill() || all_of(B->edges(), + [](const Edge &E) { + return E.getKind() == + Edge::KeepAlive; + })) && + "Non-KeepAlive edges in zero-fill block?"); - // Dispatch to LinkerImpl for fixup. - if (auto Err = impl().applyFixup(G, *B, E)) - return Err; + // If this is a no-alloc section then copy the block content into + // memory allocated on the Graph's allocator (if it hasn't been + // already). + if (NoAllocSection) + (void)B->getMutableContent(G); + + for (auto &E : B->edges()) { + + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // If B is a block in a Standard or Finalize section then make sure + // that no edges point to symbols in NoAlloc sections. + assert( + (NoAllocSection || !E.getTarget().isDefined() || + E.getTarget().getBlock().getSection().getMemLifetimePolicy() != + orc::MemLifetimePolicy::NoAlloc) && + "Block in allocated section has edge pointing to no-alloc " + "section"); + + // Dispatch to LinkerImpl for fixup. + if (auto Err = impl().applyFixup(G, *B, E)) + return Err; + } } } diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index bd44b86f3081..f481504135a5 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -24,11 +24,12 @@ JITLinkMemoryManager::InFlightAlloc::~InFlightAlloc() = default; BasicLayout::BasicLayout(LinkGraph &G) : G(G) { for (auto &Sec : G.sections()) { - // Skip empty sections. - if (Sec.blocks().empty()) + // Skip empty sections, and sections with NoAlloc lifetime policies. + if (Sec.blocks().empty() || + Sec.getMemLifetimePolicy() == orc::MemLifetimePolicy::NoAlloc) continue; - auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemDeallocPolicy()}]; + auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemLifetimePolicy()}]; for (auto *B : Sec.blocks()) if (LLVM_LIKELY(!B->isZeroFill())) Seg.ContentBlocks.push_back(B); @@ -89,7 +90,7 @@ BasicLayout::getContiguousPageBasedLayoutSizes(uint64_t PageSize) { inconvertibleErrorCode()); uint64_t SegSize = alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize); - if (AG.getMemDeallocPolicy() == orc::MemDeallocPolicy::Standard) + if (AG.getMemLifetimePolicy() == orc::MemLifetimePolicy::Standard) SegsSizes.StandardSegs += SegSize; else SegsSizes.FinalizeSegs += SegSize; @@ -146,7 +147,7 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD, SegmentMap Segments, OnCreatedFunction OnCreated) { - static_assert(orc::AllocGroup::NumGroups == 16, + static_assert(orc::AllocGroup::NumGroups == 32, "AllocGroup has changed. Section names below must be updated"); StringRef AGSectionNames[] = { "__---.standard", "__R--.standard", "__-W-.standard", "__RW-.standard", @@ -163,12 +164,15 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, auto &AG = KV.first; auto &Seg = KV.second; + assert(AG.getMemLifetimePolicy() != orc::MemLifetimePolicy::NoAlloc && + "NoAlloc segments are not supported by SimpleSegmentAlloc"); + auto AGSectionName = AGSectionNames[static_cast<unsigned>(AG.getMemProt()) | - static_cast<bool>(AG.getMemDeallocPolicy()) << 3]; + static_cast<bool>(AG.getMemLifetimePolicy()) << 3]; auto &Sec = G->createSection(AGSectionName, AG.getMemProt()); - Sec.setMemDeallocPolicy(AG.getMemDeallocPolicy()); + Sec.setMemLifetimePolicy(AG.getMemLifetimePolicy()); if (Seg.ContentSize != 0) { NextAddr = @@ -236,10 +240,14 @@ public: IPInFlightAlloc(InProcessMemoryManager &MemMgr, LinkGraph &G, BasicLayout BL, sys::MemoryBlock StandardSegments, sys::MemoryBlock FinalizationSegments) - : MemMgr(MemMgr), G(G), BL(std::move(BL)), + : MemMgr(MemMgr), G(&G), BL(std::move(BL)), StandardSegments(std::move(StandardSegments)), FinalizationSegments(std::move(FinalizationSegments)) {} + ~IPInFlightAlloc() { + assert(!G && "InFlight alloc neither abandoned nor finalized"); + } + void finalize(OnFinalizedFunction OnFinalized) override { // Apply memory protections to all segments. @@ -249,7 +257,7 @@ public: } // Run finalization actions. - auto DeallocActions = runFinalizeActions(G.allocActions()); + auto DeallocActions = runFinalizeActions(G->allocActions()); if (!DeallocActions) { OnFinalized(DeallocActions.takeError()); return; @@ -261,6 +269,13 @@ public: return; } +#ifndef NDEBUG + // Set 'G' to null to flag that we've been successfully finalized. + // This allows us to assert at destruction time that a call has been made + // to either finalize or abandon. + G = nullptr; +#endif + // Continue with finalized allocation. OnFinalized(MemMgr.createFinalizedAlloc(std::move(StandardSegments), std::move(*DeallocActions))); @@ -272,6 +287,14 @@ public: Err = joinErrors(std::move(Err), errorCodeToError(EC)); if (auto EC = sys::Memory::releaseMappedMemory(StandardSegments)) Err = joinErrors(std::move(Err), errorCodeToError(EC)); + +#ifndef NDEBUG + // Set 'G' to null to flag that we've been successfully finalized. + // This allows us to assert at destruction time that a call has been made + // to either finalize or abandon. + G = nullptr; +#endif + OnAbandoned(std::move(Err)); } @@ -295,7 +318,7 @@ private: } InProcessMemoryManager &MemMgr; - LinkGraph &G; + LinkGraph *G; BasicLayout BL; sys::MemoryBlock StandardSegments; sys::MemoryBlock FinalizationSegments; @@ -397,7 +420,7 @@ void InProcessMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, auto &Seg = KV.second; auto &SegAddr = - (AG.getMemDeallocPolicy() == orc::MemDeallocPolicy::Standard) + (AG.getMemLifetimePolicy() == orc::MemLifetimePolicy::Standard) ? NextStandardSegAddr : NextFinalizeSegAddr; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO.cpp index e49480c78662..40086ccf2b66 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO.cpp @@ -54,7 +54,7 @@ createLinkGraphFromMachOObject(MemoryBufferRef ObjectBuffer) { uint32_t CPUType; memcpy(&CPUType, Data.data() + 4, sizeof(uint32_t)); if (Magic == MachO::MH_CIGAM_64) - CPUType = ByteSwap_32(CPUType); + CPUType = llvm::byteswap<uint32_t>(CPUType); LLVM_DEBUG({ dbgs() << "jitLink_MachO: cputype = " << format("0x%08" PRIx32, CPUType) diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp index 987689993397..c40e0f9ffc8d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -47,12 +47,13 @@ Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() { } MachOLinkGraphBuilder::MachOLinkGraphBuilder( - const object::MachOObjectFile &Obj, Triple TT, + const object::MachOObjectFile &Obj, Triple TT, SubtargetFeatures Features, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName) : Obj(Obj), - G(std::make_unique<LinkGraph>( - std::string(Obj.getFileName()), std::move(TT), getPointerSize(Obj), - getEndianness(Obj), std::move(GetEdgeKindName))) { + G(std::make_unique<LinkGraph>(std::string(Obj.getFileName()), + std::move(TT), std::move(Features), + getPointerSize(Obj), getEndianness(Obj), + std::move(GetEdgeKindName))) { auto &MachHeader = Obj.getHeader64(); SubsectionsViaSymbols = MachHeader.flags & MachO::MH_SUBSECTIONS_VIA_SYMBOLS; } @@ -185,10 +186,14 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { Prot = orc::MemProt::Read | orc::MemProt::Write; auto FullyQualifiedName = - G->allocateString(StringRef(NSec.SegName) + "," + NSec.SectName); + G->allocateContent(StringRef(NSec.SegName) + "," + NSec.SectName); NSec.GraphSection = &G->createSection( StringRef(FullyQualifiedName.data(), FullyQualifiedName.size()), Prot); + // TODO: Are there any other criteria for NoAlloc lifetime? + if (NSec.Flags & MachO::S_ATTR_DEBUG) + NSec.GraphSection->setMemLifetimePolicy(orc::MemLifetimePolicy::NoAlloc); + IndexToSection.insert(std::make_pair(SecIndex, std::move(NSec))); } @@ -267,7 +272,11 @@ Error MachOLinkGraphBuilder::createNormalizedSymbols() { Name = *NameOrErr; else return NameOrErr.takeError(); - } + } else if (Type & MachO::N_EXT) + return make_error<JITLinkError>("Symbol at index " + + formatv("{0}", SymbolIndex) + + " has no name (string table index 0), " + "but N_EXT bit is set"); LLVM_DEBUG({ dbgs() << " "; @@ -656,7 +665,7 @@ Error MachOLinkGraphBuilder::graphifyCStringSection( orc::ExecutorAddrDiff BlockStart = 0; // Scan section for null characters. - for (size_t I = 0; I != NSec.Size; ++I) + for (size_t I = 0; I != NSec.Size; ++I) { if (NSec.Data[I] == '\0') { size_t BlockSize = I + 1 - BlockStart; // Create a block for this null terminated string. @@ -723,6 +732,11 @@ Error MachOLinkGraphBuilder::graphifyCStringSection( BlockStart += BlockSize; } + } + + assert(llvm::all_of(NSec.GraphSection->blocks(), + [](Block *B) { return isCStringBlock(*B); }) && + "All blocks in section should hold single c-strings"); return Error::success(); } diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index ba6cfaf8aa94..2805c2960b9b 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -84,6 +84,7 @@ protected: using SectionParserFunction = std::function<Error(NormalizedSection &S)>; MachOLinkGraphBuilder(const object::MachOObjectFile &Obj, Triple TT, + SubtargetFeatures Features, LinkGraph::GetEdgeKindNameFunction GetEdgeKindName); LinkGraph &getGraph() const { return *G; } diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 3380bb563140..dd0b5d37d1b7 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -25,9 +25,10 @@ namespace { class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { public: - MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) + MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj, + SubtargetFeatures Features) : MachOLinkGraphBuilder(Obj, Triple("arm64-apple-darwin"), - aarch64::getEdgeKindName), + std::move(Features), aarch64::getEdgeKindName), NumSymbols(Obj.getSymtabLoadCommand().nsyms) {} private: @@ -541,7 +542,13 @@ createLinkGraphFromMachOObject_arm64(MemoryBufferRef ObjectBuffer) { auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjectBuffer); if (!MachOObj) return MachOObj.takeError(); - return MachOLinkGraphBuilder_arm64(**MachOObj).buildGraph(); + + auto Features = (*MachOObj)->getFeatures(); + if (!Features) + return Features.takeError(); + + return MachOLinkGraphBuilder_arm64(**MachOObj, std::move(*Features)) + .buildGraph(); } void link_MachO_arm64(std::unique_ptr<LinkGraph> G, diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index be40b740a5a7..4dba27bc61cb 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -25,9 +25,10 @@ namespace { class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { public: - MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj, + SubtargetFeatures Features) : MachOLinkGraphBuilder(Obj, Triple("x86_64-apple-darwin"), - x86_64::getEdgeKindName) {} + std::move(Features), x86_64::getEdgeKindName) {} private: enum MachONormalizedRelocationType : unsigned { @@ -466,7 +467,13 @@ createLinkGraphFromMachOObject_x86_64(MemoryBufferRef ObjectBuffer) { auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjectBuffer); if (!MachOObj) return MachOObj.takeError(); - return MachOLinkGraphBuilder_x86_64(**MachOObj).buildGraph(); + + auto Features = (*MachOObj)->getFeatures(); + if (!Features) + return Features.takeError(); + + return MachOLinkGraphBuilder_x86_64(**MachOObj, std::move(*Features)) + .buildGraph(); } void link_MachO_x86_64(std::unique_ptr<LinkGraph> G, diff --git a/llvm/lib/ExecutionEngine/JITLink/SEHFrameSupport.h b/llvm/lib/ExecutionEngine/JITLink/SEHFrameSupport.h index 0d95fbf439b5..21bfd36d44a2 100644 --- a/llvm/lib/ExecutionEngine/JITLink/SEHFrameSupport.h +++ b/llvm/lib/ExecutionEngine/JITLink/SEHFrameSupport.h @@ -13,10 +13,10 @@ #ifndef LLVM_EXECUTIONENGINE_JITLINK_SEHFRAMESUPPORT_H #define LLVM_EXECUTIONENGINE_JITLINK_SEHFRAMESUPPORT_H -#include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/Support/Error.h" +#include "llvm/TargetParser/Triple.h" namespace llvm { namespace jitlink { diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp new file mode 100644 index 000000000000..ffc3950cdec8 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp @@ -0,0 +1,519 @@ +//===--------- aarch32.cpp - Generic JITLink arm/thumb utilities ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing arm/thumb objects. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/aarch32.h" + +#include "llvm/ADT/StringExtras.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/MathExtras.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { +namespace aarch32 { + +/// Encode 22-bit immediate value for branch instructions without J1J2 range +/// extension (formats B T4, BL T1 and BLX T2). +/// +/// 00000:Imm11H:Imm11L:0 -> [ 00000:Imm11H, 00000:Imm11L ] +/// J1^ ^J2 will always be 1 +/// +HalfWords encodeImmBT4BlT1BlxT2(int64_t Value) { + constexpr uint32_t J1J2 = 0x2800; + uint32_t Imm11H = (Value >> 12) & 0x07ff; + uint32_t Imm11L = (Value >> 1) & 0x07ff; + return HalfWords{Imm11H, Imm11L | J1J2}; +} + +/// Decode 22-bit immediate value for branch instructions without J1J2 range +/// extension (formats B T4, BL T1 and BLX T2). +/// +/// [ 00000:Imm11H, 00000:Imm11L ] -> 00000:Imm11H:Imm11L:0 +/// J1^ ^J2 will always be 1 +/// +int64_t decodeImmBT4BlT1BlxT2(uint32_t Hi, uint32_t Lo) { + uint32_t Imm11H = Hi & 0x07ff; + uint32_t Imm11L = Lo & 0x07ff; + return SignExtend64<22>(Imm11H << 12 | Imm11L << 1); +} + +/// Encode 25-bit immediate value for branch instructions with J1J2 range +/// extension (formats B T4, BL T1 and BLX T2). +/// +/// S:I1:I2:Imm10:Imm11:0 -> [ 00000:S:Imm10, 00:J1:0:J2:Imm11 ] +/// +HalfWords encodeImmBT4BlT1BlxT2_J1J2(int64_t Value) { + uint32_t S = (Value >> 14) & 0x0400; + uint32_t J1 = (((~(Value >> 10)) ^ (Value >> 11)) & 0x2000); + uint32_t J2 = (((~(Value >> 11)) ^ (Value >> 13)) & 0x0800); + uint32_t Imm10 = (Value >> 12) & 0x03ff; + uint32_t Imm11 = (Value >> 1) & 0x07ff; + return HalfWords{S | Imm10, J1 | J2 | Imm11}; +} + +/// Decode 25-bit immediate value for branch instructions with J1J2 range +/// extension (formats B T4, BL T1 and BLX T2). +/// +/// [ 00000:S:Imm10, 00:J1:0:J2:Imm11] -> S:I1:I2:Imm10:Imm11:0 +/// +int64_t decodeImmBT4BlT1BlxT2_J1J2(uint32_t Hi, uint32_t Lo) { + uint32_t S = Hi & 0x0400; + uint32_t I1 = ~((Lo ^ (Hi << 3)) << 10) & 0x00800000; + uint32_t I2 = ~((Lo ^ (Hi << 1)) << 11) & 0x00400000; + uint32_t Imm10 = Hi & 0x03ff; + uint32_t Imm11 = Lo & 0x07ff; + return SignExtend64<25>(S << 14 | I1 | I2 | Imm10 << 12 | Imm11 << 1); +} + +/// Encode 16-bit immediate value for move instruction formats MOVT T1 and +/// MOVW T3. +/// +/// Imm4:Imm1:Imm3:Imm8 -> [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ] +/// +HalfWords encodeImmMovtT1MovwT3(uint16_t Value) { + uint32_t Imm4 = (Value >> 12) & 0x0f; + uint32_t Imm1 = (Value >> 11) & 0x01; + uint32_t Imm3 = (Value >> 8) & 0x07; + uint32_t Imm8 = Value & 0xff; + return HalfWords{Imm1 << 10 | Imm4, Imm3 << 12 | Imm8}; +} + +/// Decode 16-bit immediate value from move instruction formats MOVT T1 and +/// MOVW T3. +/// +/// [ 00000:i:000000:Imm4, 0:Imm3:0000:Imm8 ] -> Imm4:Imm1:Imm3:Imm8 +/// +uint16_t decodeImmMovtT1MovwT3(uint32_t Hi, uint32_t Lo) { + uint32_t Imm4 = Hi & 0x0f; + uint32_t Imm1 = (Hi >> 10) & 0x01; + uint32_t Imm3 = (Lo >> 12) & 0x07; + uint32_t Imm8 = Lo & 0xff; + uint32_t Imm16 = Imm4 << 12 | Imm1 << 11 | Imm3 << 8 | Imm8; + assert(Imm16 <= 0xffff && "Decoded value out-of-range"); + return Imm16; +} + +/// Encode register ID for instruction formats MOVT T1 and MOVW T3. +/// +/// Rd4 -> [0000000000000000, 0000:Rd4:00000000] +/// +HalfWords encodeRegMovtT1MovwT3(int64_t Value) { + uint32_t Rd4 = (Value & 0x0f) << 8; + return HalfWords{0, Rd4}; +} + +/// Decode register ID from instruction formats MOVT T1 and MOVW T3. +/// +/// [0000000000000000, 0000:Rd4:00000000] -> Rd4 +/// +int64_t decodeRegMovtT1MovwT3(uint32_t Hi, uint32_t Lo) { + uint32_t Rd4 = (Lo >> 8) & 0x0f; + return Rd4; +} + +/// 32-bit Thumb instructions are stored as two little-endian halfwords. +/// An instruction at address A encodes bytes A+1, A in the first halfword (Hi), +/// followed by bytes A+3, A+2 in the second halfword (Lo). +struct WritableThumbRelocation { + /// Create a writable reference to a Thumb32 fixup. + WritableThumbRelocation(char *FixupPtr) + : Hi{*reinterpret_cast<support::ulittle16_t *>(FixupPtr)}, + Lo{*reinterpret_cast<support::ulittle16_t *>(FixupPtr + 2)} {} + + support::ulittle16_t &Hi; // First halfword + support::ulittle16_t &Lo; // Second halfword +}; + +struct ThumbRelocation { + /// Create a read-only reference to a Thumb32 fixup. + ThumbRelocation(const char *FixupPtr) + : Hi{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr)}, + Lo{*reinterpret_cast<const support::ulittle16_t *>(FixupPtr + 2)} {} + + /// Create a read-only Thumb32 fixup from a writeable one. + ThumbRelocation(WritableThumbRelocation &Writable) + : Hi{Writable.Hi}, Lo(Writable.Lo) {} + + const support::ulittle16_t &Hi; // First halfword + const support::ulittle16_t &Lo; // Second halfword +}; + +Error makeUnexpectedOpcodeError(const LinkGraph &G, const ThumbRelocation &R, + Edge::Kind Kind) { + return make_error<JITLinkError>( + formatv("Invalid opcode [ 0x{0:x4}, 0x{1:x4} ] for relocation: {2}", + static_cast<uint16_t>(R.Hi), static_cast<uint16_t>(R.Lo), + G.getEdgeKindName(Kind))); +} + +template <EdgeKind_aarch32 Kind> bool checkOpcode(const ThumbRelocation &R) { + uint16_t Hi = R.Hi & FixupInfo<Kind>::OpcodeMask.Hi; + uint16_t Lo = R.Lo & FixupInfo<Kind>::OpcodeMask.Lo; + return Hi == FixupInfo<Kind>::Opcode.Hi && Lo == FixupInfo<Kind>::Opcode.Lo; +} + +template <EdgeKind_aarch32 Kind> +bool checkRegister(const ThumbRelocation &R, HalfWords Reg) { + uint16_t Hi = R.Hi & FixupInfo<Kind>::RegMask.Hi; + uint16_t Lo = R.Lo & FixupInfo<Kind>::RegMask.Lo; + return Hi == Reg.Hi && Lo == Reg.Lo; +} + +template <EdgeKind_aarch32 Kind> +bool writeRegister(WritableThumbRelocation &R, HalfWords Reg) { + static constexpr HalfWords Mask = FixupInfo<Kind>::RegMask; + assert((Mask.Hi & Reg.Hi) == Reg.Hi && (Mask.Hi & Reg.Hi) == Reg.Hi && + "Value bits exceed bit range of given mask"); + R.Hi = (R.Hi & ~Mask.Hi) | Reg.Hi; + R.Lo = (R.Lo & ~Mask.Lo) | Reg.Lo; +} + +template <EdgeKind_aarch32 Kind> +void writeImmediate(WritableThumbRelocation &R, HalfWords Imm) { + static constexpr HalfWords Mask = FixupInfo<Kind>::ImmMask; + assert((Mask.Hi & Imm.Hi) == Imm.Hi && (Mask.Hi & Imm.Hi) == Imm.Hi && + "Value bits exceed bit range of given mask"); + R.Hi = (R.Hi & ~Mask.Hi) | Imm.Hi; + R.Lo = (R.Lo & ~Mask.Lo) | Imm.Lo; +} + +Expected<int64_t> readAddendData(LinkGraph &G, Block &B, const Edge &E) { + support::endianness Endian = G.getEndianness(); + assert(Endian != support::native && "Declare as little or big explicitly"); + + Edge::Kind Kind = E.getKind(); + const char *BlockWorkingMem = B.getContent().data(); + const char *FixupPtr = BlockWorkingMem + E.getOffset(); + + switch (Kind) { + case Data_Delta32: + case Data_Pointer32: + return SignExtend64<32>(support::endian::read32(FixupPtr, Endian)); + default: + return make_error<JITLinkError>( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + " can not read implicit addend for aarch32 edge kind " + + G.getEdgeKindName(E.getKind())); + } +} + +Expected<int64_t> readAddendArm(LinkGraph &G, Block &B, const Edge &E) { + Edge::Kind Kind = E.getKind(); + + switch (Kind) { + case Arm_Call: + return make_error<JITLinkError>( + "Addend extraction for relocation type not yet implemented: " + + StringRef(G.getEdgeKindName(Kind))); + default: + return make_error<JITLinkError>( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + " can not read implicit addend for aarch32 edge kind " + + G.getEdgeKindName(E.getKind())); + } +} + +Expected<int64_t> readAddendThumb(LinkGraph &G, Block &B, const Edge &E, + const ArmConfig &ArmCfg) { + ThumbRelocation R(B.getContent().data() + E.getOffset()); + Edge::Kind Kind = E.getKind(); + + switch (Kind) { + case Thumb_Call: + if (!checkOpcode<Thumb_Call>(R)) + return makeUnexpectedOpcodeError(G, R, Kind); + return LLVM_LIKELY(ArmCfg.J1J2BranchEncoding) + ? decodeImmBT4BlT1BlxT2_J1J2(R.Hi, R.Lo) + : decodeImmBT4BlT1BlxT2(R.Hi, R.Lo); + + case Thumb_Jump24: + if (!checkOpcode<Thumb_Jump24>(R)) + return makeUnexpectedOpcodeError(G, R, Kind); + if (R.Lo & FixupInfo<Thumb_Jump24>::LoBitConditional) + return make_error<JITLinkError>("Relocation expects an unconditional " + "B.W branch instruction: " + + StringRef(G.getEdgeKindName(Kind))); + return LLVM_LIKELY(ArmCfg.J1J2BranchEncoding) + ? decodeImmBT4BlT1BlxT2_J1J2(R.Hi, R.Lo) + : decodeImmBT4BlT1BlxT2(R.Hi, R.Lo); + + case Thumb_MovwAbsNC: + if (!checkOpcode<Thumb_MovwAbsNC>(R)) + return makeUnexpectedOpcodeError(G, R, Kind); + // Initial addend is interpreted as a signed value + return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo)); + + case Thumb_MovtAbs: + if (!checkOpcode<Thumb_MovtAbs>(R)) + return makeUnexpectedOpcodeError(G, R, Kind); + // Initial addend is interpreted as a signed value + return SignExtend64<16>(decodeImmMovtT1MovwT3(R.Hi, R.Lo)); + + default: + return make_error<JITLinkError>( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + " can not read implicit addend for aarch32 edge kind " + + G.getEdgeKindName(E.getKind())); + } +} + +Error applyFixupData(LinkGraph &G, Block &B, const Edge &E) { + using namespace support; + + char *BlockWorkingMem = B.getAlreadyMutableContent().data(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + + auto Write32 = [FixupPtr, Endian = G.getEndianness()](int64_t Value) { + assert(Endian != native && "Must be explicit: little or big"); + assert(isInt<32>(Value) && "Must be in signed 32-bit range"); + uint32_t Imm = static_cast<int32_t>(Value); + if (LLVM_LIKELY(Endian == little)) + endian::write32<little>(FixupPtr, Imm); + else + endian::write32<big>(FixupPtr, Imm); + }; + + Edge::Kind Kind = E.getKind(); + uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue(); + int64_t Addend = E.getAddend(); + Symbol &TargetSymbol = E.getTarget(); + uint64_t TargetAddress = TargetSymbol.getAddress().getValue(); + assert(!TargetSymbol.hasTargetFlags(ThumbSymbol)); + + // Regular data relocations have size 4, alignment 1 and write the full 32-bit + // result to the place; no need for overflow checking. There are three + // exceptions: R_ARM_ABS8, R_ARM_ABS16, R_ARM_PREL31 + switch (Kind) { + case Data_Delta32: { + int64_t Value = TargetAddress - FixupAddress + Addend; + if (!isInt<32>(Value)) + return makeTargetOutOfRangeError(G, B, E); + Write32(Value); + return Error::success(); + } + case Data_Pointer32: { + int64_t Value = TargetAddress + Addend; + if (!isInt<32>(Value)) + return makeTargetOutOfRangeError(G, B, E); + Write32(Value); + return Error::success(); + } + default: + return make_error<JITLinkError>( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + " encountered unfixable aarch32 edge kind " + + G.getEdgeKindName(E.getKind())); + } +} + +Error applyFixupArm(LinkGraph &G, Block &B, const Edge &E) { + Edge::Kind Kind = E.getKind(); + + switch (Kind) { + case Arm_Call: + return make_error<JITLinkError>( + "Fix-up for relocation type not yet implemented: " + + StringRef(G.getEdgeKindName(Kind))); + default: + return make_error<JITLinkError>( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + " encountered unfixable aarch32 edge kind " + + G.getEdgeKindName(E.getKind())); + } +} + +Error applyFixupThumb(LinkGraph &G, Block &B, const Edge &E, + const ArmConfig &ArmCfg) { + WritableThumbRelocation R(B.getAlreadyMutableContent().data() + + E.getOffset()); + + Edge::Kind Kind = E.getKind(); + uint64_t FixupAddress = (B.getAddress() + E.getOffset()).getValue(); + int64_t Addend = E.getAddend(); + Symbol &TargetSymbol = E.getTarget(); + uint64_t TargetAddress = TargetSymbol.getAddress().getValue(); + if (TargetSymbol.hasTargetFlags(ThumbSymbol)) + TargetAddress |= 0x01; + + switch (Kind) { + case Thumb_Jump24: { + if (!checkOpcode<Thumb_Jump24>(R)) + return makeUnexpectedOpcodeError(G, R, Kind); + if (R.Lo & FixupInfo<Thumb_Jump24>::LoBitConditional) + return make_error<JITLinkError>("Relocation expects an unconditional " + "B.W branch instruction: " + + StringRef(G.getEdgeKindName(Kind))); + if (!(TargetSymbol.hasTargetFlags(ThumbSymbol))) + return make_error<JITLinkError>("Branch relocation needs interworking " + "stub when bridging to ARM: " + + StringRef(G.getEdgeKindName(Kind))); + + int64_t Value = TargetAddress - FixupAddress + Addend; + if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) { + if (!isInt<25>(Value)) + return makeTargetOutOfRangeError(G, B, E); + writeImmediate<Thumb_Jump24>(R, encodeImmBT4BlT1BlxT2_J1J2(Value)); + } else { + if (!isInt<22>(Value)) + return makeTargetOutOfRangeError(G, B, E); + writeImmediate<Thumb_Jump24>(R, encodeImmBT4BlT1BlxT2(Value)); + } + + return Error::success(); + } + + case Thumb_Call: { + if (!checkOpcode<Thumb_Call>(R)) + return makeUnexpectedOpcodeError(G, R, Kind); + + int64_t Value = TargetAddress - FixupAddress + Addend; + + // The call instruction itself is Thumb. The call destination can either be + // Thumb or Arm. We use BL to stay in Thumb and BLX to change to Arm. + bool TargetIsArm = !TargetSymbol.hasTargetFlags(ThumbSymbol); + bool InstrIsBlx = (R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) == 0; + if (TargetIsArm != InstrIsBlx) { + if (LLVM_LIKELY(TargetIsArm)) { + // Change opcode BL -> BLX and fix range value (account for 4-byte + // aligned destination while instruction may only be 2-byte aligned + // and clear Thumb bit). + R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx; + R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitH; + Value = alignTo(Value, 4); + } else { + // Change opcode BLX -> BL and set Thumb bit + R.Lo = R.Lo & ~FixupInfo<Thumb_Call>::LoBitNoBlx; + Value |= 0x01; + } + } + + if (LLVM_LIKELY(ArmCfg.J1J2BranchEncoding)) { + if (!isInt<25>(Value)) + return makeTargetOutOfRangeError(G, B, E); + writeImmediate<Thumb_Call>(R, encodeImmBT4BlT1BlxT2_J1J2(Value)); + } else { + if (!isInt<22>(Value)) + return makeTargetOutOfRangeError(G, B, E); + writeImmediate<Thumb_Call>(R, encodeImmBT4BlT1BlxT2(Value)); + } + + assert(((R.Lo & FixupInfo<Thumb_Call>::LoBitNoBlx) || + (R.Lo & FixupInfo<Thumb_Call>::LoBitH) == 0) && + "Opcode BLX implies H bit is clear (avoid UB in BLX T2)"); + return Error::success(); + } + + case Thumb_MovwAbsNC: { + if (!checkOpcode<Thumb_MovwAbsNC>(R)) + return makeUnexpectedOpcodeError(G, R, Kind); + uint16_t Value = (TargetAddress + Addend) & 0xffff; + writeImmediate<Thumb_MovwAbsNC>(R, encodeImmMovtT1MovwT3(Value)); + return Error::success(); + } + + case Thumb_MovtAbs: { + if (!checkOpcode<Thumb_MovtAbs>(R)) + return makeUnexpectedOpcodeError(G, R, Kind); + uint16_t Value = ((TargetAddress + Addend) >> 16) & 0xffff; + writeImmediate<Thumb_MovtAbs>(R, encodeImmMovtT1MovwT3(Value)); + return Error::success(); + } + + default: + return make_error<JITLinkError>( + "In graph " + G.getName() + ", section " + B.getSection().getName() + + " encountered unfixable aarch32 edge kind " + + G.getEdgeKindName(E.getKind())); + } +} + +const uint8_t Thumbv7ABS[] = { + 0x40, 0xf2, 0x00, 0x0c, // movw r12, #0x0000 ; lower 16-bit + 0xc0, 0xf2, 0x00, 0x0c, // movt r12, #0x0000 ; upper 16-bit + 0x60, 0x47 // bx r12 +}; + +template <> +Symbol &StubsManager<Thumbv7>::createEntry(LinkGraph &G, Symbol &Target) { + constexpr uint64_t Alignment = 4; + Block &B = addStub(G, Thumbv7ABS, Alignment); + LLVM_DEBUG({ + const char *StubPtr = B.getContent().data(); + HalfWords Reg12 = encodeRegMovtT1MovwT3(12); + assert(checkRegister<Thumb_MovwAbsNC>(StubPtr, Reg12) && + checkRegister<Thumb_MovtAbs>(StubPtr + 4, Reg12) && + "Linker generated stubs may only corrupt register r12 (IP)"); + }); + B.addEdge(Thumb_MovwAbsNC, 0, Target, 0); + B.addEdge(Thumb_MovtAbs, 4, Target, 0); + Symbol &Stub = G.addAnonymousSymbol(B, 0, B.getSize(), true, false); + Stub.setTargetFlags(ThumbSymbol); + return Stub; +} + +const char *getEdgeKindName(Edge::Kind K) { +#define KIND_NAME_CASE(K) \ + case K: \ + return #K; + + switch (K) { + KIND_NAME_CASE(Data_Delta32) + KIND_NAME_CASE(Arm_Call) + KIND_NAME_CASE(Thumb_Call) + KIND_NAME_CASE(Thumb_Jump24) + KIND_NAME_CASE(Thumb_MovwAbsNC) + KIND_NAME_CASE(Thumb_MovtAbs) + default: + return getGenericEdgeKindName(K); + } +#undef KIND_NAME_CASE +} + +const char *getCPUArchName(ARMBuildAttrs::CPUArch K) { +#define CPUARCH_NAME_CASE(K) \ + case K: \ + return #K; + + using namespace ARMBuildAttrs; + switch (K) { + CPUARCH_NAME_CASE(Pre_v4) + CPUARCH_NAME_CASE(v4) + CPUARCH_NAME_CASE(v4T) + CPUARCH_NAME_CASE(v5T) + CPUARCH_NAME_CASE(v5TE) + CPUARCH_NAME_CASE(v5TEJ) + CPUARCH_NAME_CASE(v6) + CPUARCH_NAME_CASE(v6KZ) + CPUARCH_NAME_CASE(v6T2) + CPUARCH_NAME_CASE(v6K) + CPUARCH_NAME_CASE(v7) + CPUARCH_NAME_CASE(v6_M) + CPUARCH_NAME_CASE(v6S_M) + CPUARCH_NAME_CASE(v7E_M) + CPUARCH_NAME_CASE(v8_A) + CPUARCH_NAME_CASE(v8_R) + CPUARCH_NAME_CASE(v8_M_Base) + CPUARCH_NAME_CASE(v8_M_Main) + CPUARCH_NAME_CASE(v8_1_M_Main) + CPUARCH_NAME_CASE(v9_A) + } + llvm_unreachable("Missing CPUArch in switch?"); +#undef CPUARCH_NAME_CASE +} + +} // namespace aarch32 +} // namespace jitlink +} // namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index 1011fa81f750..cc58255a338d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -47,6 +47,12 @@ const char *getEdgeKindName(Edge::Kind R) { return "MoveWide16"; case LDRLiteral19: return "LDRLiteral19"; + case TestAndBranch14PCRel: + return "TestAndBranch14PCRel"; + case CondBranch19PCRel: + return "CondBranch19PCRel"; + case ADRLiteral21: + return "ADRLiteral21"; case Page21: return "Page21"; case PageOffset12: diff --git a/llvm/lib/ExecutionEngine/JITLink/i386.cpp b/llvm/lib/ExecutionEngine/JITLink/i386.cpp index c2c5761cd272..e984bb10983d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/i386.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/i386.cpp @@ -34,10 +34,58 @@ const char *getEdgeKindName(Edge::Kind K) { return "Delta32FromGOT"; case RequestGOTAndTransformToDelta32FromGOT: return "RequestGOTAndTransformToDelta32FromGOT"; + case BranchPCRel32: + return "BranchPCRel32"; + case BranchPCRel32ToPtrJumpStub: + return "BranchPCRel32ToPtrJumpStub"; + case BranchPCRel32ToPtrJumpStubBypassable: + return "BranchPCRel32ToPtrJumpStubBypassable"; } return getGenericEdgeKindName(K); } const char NullPointerContent[PointerSize] = {0x00, 0x00, 0x00, 0x00}; + +const char PointerJumpStubContent[6] = { + static_cast<char>(0xFFu), 0x25, 0x00, 0x00, 0x00, 0x00}; + +Error optimizeGOTAndStubAccesses(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Optimizing GOT entries and stubs:\n"); + + for (auto *B : G.blocks()) + for (auto &E : B->edges()) { + if (E.getKind() == i386::BranchPCRel32ToPtrJumpStubBypassable) { + auto &StubBlock = E.getTarget().getBlock(); + assert(StubBlock.getSize() == sizeof(PointerJumpStubContent) && + "Stub block should be stub sized"); + assert(StubBlock.edges_size() == 1 && + "Stub block should only have one outgoing edge"); + + auto &GOTBlock = StubBlock.edges().begin()->getTarget().getBlock(); + assert(GOTBlock.getSize() == G.getPointerSize() && + "GOT block should be pointer sized"); + assert(GOTBlock.edges_size() == 1 && + "GOT block should only have one outgoing edge"); + + auto &GOTTarget = GOTBlock.edges().begin()->getTarget(); + orc::ExecutorAddr EdgeAddr = B->getAddress() + E.getOffset(); + orc::ExecutorAddr TargetAddr = GOTTarget.getAddress(); + + int64_t Displacement = TargetAddr - EdgeAddr + 4; + if (isInt<32>(Displacement)) { + E.setKind(i386::BranchPCRel32); + E.setTarget(GOTTarget); + LLVM_DEBUG({ + dbgs() << " Replaced stub branch with direct branch:\n "; + printEdge(dbgs(), *B, E, getEdgeKindName(E.getKind())); + dbgs() << "\n"; + }); + } + } + } + + return Error::success(); +} + } // namespace llvm::jitlink::i386 diff --git a/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp new file mode 100644 index 000000000000..4e21eace21d0 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/ppc64.cpp @@ -0,0 +1,102 @@ +//===----- ppc64.cpp - Generic JITLink ppc64 edge kinds, utilities ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing 64-bit PowerPC objects. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ppc64.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm::jitlink::ppc64 { + +const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}; + +const char PointerJumpStubContent_little[20] = { + 0x18, 0x00, 0x41, (char)0xf8, // std r2, 24(r1) + 0x00, 0x00, (char)0x82, 0x3d, // addis r12, r2, OffHa + 0x00, 0x00, (char)0x8c, (char)0xe9, // ld r12, OffLo(r12) + (char)0xa6, 0x03, (char)0x89, 0x7d, // mtctr r12 + 0x20, 0x04, (char)0x80, 0x4e, // bctr +}; + +const char PointerJumpStubContent_big[20] = { + (char)0xf8, 0x41, 0x00, 0x18, // std r2, 24(r1) + 0x3d, (char)0x82, 0x00, 0x00, // addis r12, r2, OffHa + (char)0xe9, (char)0x8c, 0x00, 0x00, // ld r12, OffLo(r12) + 0x7d, (char)0x89, 0x03, (char)0xa6, // mtctr r12 + 0x4e, (char)0x80, 0x04, 0x20, // bctr +}; + +// TODO: We can use prefixed instructions if LLJIT is running on power10. +const char PointerJumpStubNoTOCContent_little[32] = { + (char)0xa6, 0x02, (char)0x88, 0x7d, // mflr 12 + 0x05, (char)0x00, (char)0x9f, 0x42, // bcl 20,31,.+4 + (char)0xa6, 0x02, 0x68, 0x7d, // mflr 11 + (char)0xa6, 0x03, (char)0x88, 0x7d, // mtlr 12 + 0x00, 0x00, (char)0x8b, 0x3d, // addis 12,11,OffHa + 0x00, 0x00, (char)0x8c, (char)0xe9, // ld 12, OffLo(12) + (char)0xa6, 0x03, (char)0x89, 0x7d, // mtctr 12 + 0x20, 0x04, (char)0x80, 0x4e, // bctr +}; + +const char PointerJumpStubNoTOCContent_big[32] = { + 0x7d, (char)0x88, 0x02, (char)0xa6, // mflr 12 + 0x42, (char)0x9f, 0x00, 0x05, // bcl 20,31,.+4 + 0x7d, 0x68, 0x02, (char)0xa6, // mflr 11 + 0x7d, (char)0x88, 0x03, (char)0xa6, // mtlr 12 + 0x3d, (char)0x8b, 0x00, 0x00, // addis 12,11,OffHa + (char)0xe9, (char)0x8c, 0x00, 0x00, // ld 12, OffLo(12) + 0x7d, (char)0x89, 0x03, (char)0xa6, // mtctr 12 + 0x4e, (char)0x80, 0x04, 0x20, // bctr +}; + +const char *getEdgeKindName(Edge::Kind K) { + switch (K) { + case Pointer64: + return "Pointer64"; + case Pointer32: + return "Pointer32"; + case Delta64: + return "Delta64"; + case Delta32: + return "Delta32"; + case NegDelta32: + return "NegDelta32"; + case Delta16: + return "Delta16"; + case Delta16HA: + return "Delta16HA"; + case Delta16LO: + return "Delta16LO"; + case TOCDelta16HA: + return "TOCDelta16HA"; + case TOCDelta16LO: + return "TOCDelta16LO"; + case TOCDelta16DS: + return "TOCDelta16DS"; + case TOCDelta16LODS: + return "TOCDelta16LODS"; + case CallBranchDelta: + return "CallBranchDelta"; + case CallBranchDeltaRestoreTOC: + return "CallBranchDeltaRestoreTOC"; + case RequestPLTCallStub: + return "RequestPLTCallStub"; + case RequestPLTCallStubSaveTOC: + return "RequestPLTCallStubSaveTOC"; + case RequestPLTCallStubNoTOC: + return "RequestPLTCallStubNoTOC"; + default: + return getGenericEdgeKindName(static_cast<Edge::Kind>(K)); + } +} + +} // end namespace llvm::jitlink::ppc64 diff --git a/llvm/lib/ExecutionEngine/JITLink/riscv.cpp b/llvm/lib/ExecutionEngine/JITLink/riscv.cpp index 6ee92b065ca1..a78843b16147 100644 --- a/llvm/lib/ExecutionEngine/JITLink/riscv.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/riscv.cpp @@ -78,6 +78,10 @@ const char *getEdgeKindName(Edge::Kind K) { return "R_RISCV_SET32"; case R_RISCV_32_PCREL: return "R_RISCV_32_PCREL"; + case CallRelaxable: + return "CallRelaxable"; + case AlignRelaxable: + return "AlignRelaxable"; } return getGenericEdgeKindName(K); } diff --git a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp index 097e19e02530..273ac7b372a7 100644 --- a/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp @@ -28,6 +28,8 @@ const char *getEdgeKindName(Edge::Kind K) { return "Pointer32Signed"; case Pointer16: return "Pointer16"; + case Pointer8: + return "Pointer8"; case Delta64: return "Delta64"; case Delta32: @@ -102,8 +104,8 @@ Error optimizeGOTAndStubAccesses(LinkGraph &G) { orc::ExecutorAddr TargetAddr = GOTTarget.getAddress(); orc::ExecutorAddr EdgeAddr = B->getFixupAddress(E); int64_t Displacement = TargetAddr - EdgeAddr + 4; - bool TargetInRangeForImmU32 = isInRangeForImmU32(TargetAddr.getValue()); - bool DisplacementInRangeForImmS32 = isInRangeForImmS32(Displacement); + bool TargetInRangeForImmU32 = isUInt<32>(TargetAddr.getValue()); + bool DisplacementInRangeForImmS32 = isInt<32>(Displacement); // If both of the Target and displacement is out of range, then // there isn't optimization chance. @@ -173,7 +175,7 @@ Error optimizeGOTAndStubAccesses(LinkGraph &G) { orc::ExecutorAddr TargetAddr = GOTTarget.getAddress(); int64_t Displacement = TargetAddr - EdgeAddr + 4; - if (isInRangeForImmS32(Displacement)) { + if (isInt<32>(Displacement)) { E.setKind(x86_64::BranchPCRel32); E.setTarget(GOTTarget); LLVM_DEBUG({ diff --git a/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp index 40716a7f9b61..7c869bead0b0 100644 --- a/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp @@ -10,6 +10,7 @@ #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Object/COFF.h" @@ -54,8 +55,7 @@ public: void materialize(std::unique_ptr<MaterializationResponsibility> R) override { unsigned PointerSize; support::endianness Endianness; - const auto &TT = - CP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + const auto &TT = CP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::x86_64: @@ -125,8 +125,8 @@ private: llvm_unreachable("Unrecognized architecture"); } - auto HeaderContent = G.allocateString( - StringRef(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); + auto HeaderContent = G.allocateContent( + ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, 0); @@ -159,20 +159,36 @@ private: namespace llvm { namespace orc { -Expected<std::unique_ptr<COFFPlatform>> -COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - JITDylib &PlatformJD, const char *OrcRuntimePath, - LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, - const char *VCRuntimePath, - std::optional<SymbolAliasMap> RuntimeAliases) { - auto &EPC = ES.getExecutorProcessControl(); +Expected<std::unique_ptr<COFFPlatform>> COFFPlatform::Create( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer, + LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, + const char *VCRuntimePath, std::optional<SymbolAliasMap> RuntimeAliases) { // If the target is not supported then bail out immediately. - if (!supportedTarget(EPC.getTargetTriple())) + if (!supportedTarget(ES.getTargetTriple())) return make_error<StringError>("Unsupported COFFPlatform triple: " + - EPC.getTargetTriple().str(), + ES.getTargetTriple().str(), inconvertibleErrorCode()); + auto &EPC = ES.getExecutorProcessControl(); + + auto GeneratorArchive = + object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef()); + if (!GeneratorArchive) + return GeneratorArchive.takeError(); + + auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Create( + ObjLinkingLayer, nullptr, std::move(*GeneratorArchive)); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + // We need a second instance of the archive (for now) for the Platform. We + // can `cantFail` this call, since if it were going to fail it would have + // failed above. + auto RuntimeArchive = cantFail( + object::Archive::create(OrcRuntimeArchiveBuffer->getMemBufferRef())); + // Create default aliases if the caller didn't supply any. if (!RuntimeAliases) RuntimeAliases = standardPlatformAliases(ES); @@ -184,13 +200,13 @@ COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>"); // Add JIT-dispatch function support symbols. - if (auto Err = HostFuncJD.define(absoluteSymbols( - {{ES.intern("__orc_rt_jit_dispatch"), - {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), - JITSymbolFlags::Exported}}, - {ES.intern("__orc_rt_jit_dispatch_ctx"), - {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), - JITSymbolFlags::Exported}}}))) + if (auto Err = HostFuncJD.define( + absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunction, + JITSymbolFlags::Exported}}, + {ES.intern("__orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContext, + JITSymbolFlags::Exported}}}))) return std::move(Err); PlatformJD.addToLinkOrder(HostFuncJD); @@ -198,13 +214,30 @@ COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, // Create the instance. Error Err = Error::success(); auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform( - ES, ObjLinkingLayer, PlatformJD, OrcRuntimePath, + ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator), + std::move(OrcRuntimeArchiveBuffer), std::move(RuntimeArchive), std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err)); if (Err) return std::move(Err); return std::move(P); } +Expected<std::unique_ptr<COFFPlatform>> +COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, + const char *VCRuntimePath, + std::optional<SymbolAliasMap> RuntimeAliases) { + + auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath); + if (!ArchiveBuffer) + return createFileError(OrcRuntimePath, ArchiveBuffer.getError()); + + return Create(ES, ObjLinkingLayer, PlatformJD, std::move(*ArchiveBuffer), + std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, + std::move(RuntimeAliases)); +} + Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() { auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker"); if (!PerJDObj) @@ -348,37 +381,22 @@ bool COFFPlatform::supportedTarget(const Triple &TT) { } } -COFFPlatform::COFFPlatform(ExecutionSession &ES, - ObjectLinkingLayer &ObjLinkingLayer, - JITDylib &PlatformJD, const char *OrcRuntimePath, - LoadDynamicLibrary LoadDynLibrary, - bool StaticVCRuntime, const char *VCRuntimePath, - Error &Err) +COFFPlatform::COFFPlatform( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, + std::unique_ptr<StaticLibraryDefinitionGenerator> OrcRuntimeGenerator, + std::unique_ptr<MemoryBuffer> OrcRuntimeArchiveBuffer, + std::unique_ptr<object::Archive> OrcRuntimeArchive, + LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, + const char *VCRuntimePath, Error &Err) : ES(ES), ObjLinkingLayer(ObjLinkingLayer), LoadDynLibrary(std::move(LoadDynLibrary)), + OrcRuntimeArchiveBuffer(std::move(OrcRuntimeArchiveBuffer)), + OrcRuntimeArchive(std::move(OrcRuntimeArchive)), StaticVCRuntime(StaticVCRuntime), COFFHeaderStartSymbol(ES.intern("__ImageBase")) { ErrorAsOutParameter _(&Err); - // Create a generator for the ORC runtime archive. - auto OrcRuntimeArchiveGenerator = - StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath); - if (!OrcRuntimeArchiveGenerator) { - Err = OrcRuntimeArchiveGenerator.takeError(); - return; - } - - auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath); - if (!ArchiveBuffer) { - Err = createFileError(OrcRuntimePath, ArchiveBuffer.getError()); - return; - } - OrcRuntimeArchiveBuffer = std::move(*ArchiveBuffer); - OrcRuntimeArchive = - std::make_unique<object::Archive>(*OrcRuntimeArchiveBuffer, Err); - if (Err) - return; - Bootstrapping.store(true); ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this)); @@ -391,7 +409,7 @@ COFFPlatform::COFFPlatform(ExecutionSession &ES, } VCRuntimeBootstrap = std::move(*VCRT); - for (auto &Lib : (*OrcRuntimeArchiveGenerator)->getImportedDynamicLibraries()) + for (auto &Lib : OrcRuntimeGenerator->getImportedDynamicLibraries()) DylibsToPreload.insert(Lib); auto ImportedLibs = @@ -405,7 +423,7 @@ COFFPlatform::COFFPlatform(ExecutionSession &ES, for (auto &Lib : *ImportedLibs) DylibsToPreload.insert(Lib); - PlatformJD.addGenerator(std::move(*OrcRuntimeArchiveGenerator)); + PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); // PlatformJD hasn't been set up by the platform yet (since we're creating // the platform now), so set it up. @@ -415,10 +433,10 @@ COFFPlatform::COFFPlatform(ExecutionSession &ES, } for (auto& Lib : DylibsToPreload) - if (auto E2 = LoadDynLibrary(PlatformJD, Lib)) { - Err = std::move(E2); - return; - } + if (auto E2 = this->LoadDynLibrary(PlatformJD, Lib)) { + Err = std::move(E2); + return; + } if (StaticVCRuntime) if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) { @@ -561,10 +579,9 @@ void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, }); if (!JD) { - SendResult( - make_error<StringError>("No JITDylib with header addr " + - formatv("{0:x}", JDHeaderAddr.getValue()), - inconvertibleErrorCode())); + SendResult(make_error<StringError>("No JITDylib with header addr " + + formatv("{0:x}", JDHeaderAddr), + inconvertibleErrorCode())); return; } @@ -579,10 +596,7 @@ void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, StringRef SymbolName) { - LLVM_DEBUG({ - dbgs() << "COFFPlatform::rt_lookupSymbol(\"" - << formatv("{0:x}", Handle.getValue()) << "\")\n"; - }); + LLVM_DEBUG(dbgs() << "COFFPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"); JITDylib *JD = nullptr; @@ -594,12 +608,9 @@ void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, } if (!JD) { - LLVM_DEBUG({ - dbgs() << " No JITDylib for handle " - << formatv("{0:x}", Handle.getValue()) << "\n"; - }); + LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + - formatv("{0:x}", Handle.getValue()), + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } @@ -612,7 +623,7 @@ void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, void operator()(Expected<SymbolMap> Result) { if (Result) { assert(Result->size() == 1 && "Unexpected result map count"); - SendResult(ExecutorAddr(Result->begin()->second.getAddress())); + SendResult(Result->begin()->second.getAddress()); } else { SendResult(Result.takeError()); } @@ -850,7 +861,7 @@ Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections( jitlink::LinkGraph &G, MaterializationResponsibility &MR) { JITLinkSymbolSet InitSectionSymbols; for (auto &Sec : G.sections()) - if (COFFPlatform::isInitializerSection(Sec.getName())) + if (isCOFFInitializerSection(Sec.getName())) for (auto *B : Sec.blocks()) if (!B->edges_empty()) InitSectionSymbols.insert( @@ -885,14 +896,13 @@ Error COFFPlatform::COFFPlatformPlugin:: // Collect static initializers for (auto &S : G.sections()) - if (COFFPlatform::isInitializerSection(S.getName())) + if (isCOFFInitializerSection(S.getName())) for (auto *B : S.blocks()) { if (B->edges_empty()) continue; for (auto &E : B->edges()) BState.Initializers.push_back(std::make_pair( - S.getName().str(), - ExecutorAddr(E.getTarget().getAddress() + E.getAddend()))); + S.getName().str(), E.getTarget().getAddress() + E.getAddend())); } return Error::success(); diff --git a/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp b/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp index d9316fab2de3..94f696fa2086 100644 --- a/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp +++ b/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp @@ -160,7 +160,7 @@ COFFVCRuntimeBootstrapper::getMSVCToolchainPath() { if (!findVCToolChainViaCommandLine(*VFS, std::nullopt, std::nullopt, std::nullopt, VCToolChainPath, VSLayout) && !findVCToolChainViaEnvironment(*VFS, VCToolChainPath, VSLayout) && - !findVCToolChainViaSetupConfig(*VFS, VCToolChainPath, VSLayout) && + !findVCToolChainViaSetupConfig(*VFS, {}, VCToolChainPath, VSLayout) && !findVCToolChainViaRegistry(VCToolChainPath, VSLayout)) return make_error<StringError>("Couldn't find msvc toolchain.", inconvertibleErrorCode()); diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index 4a9d0d470a8e..0c23f2b25219 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -167,15 +167,16 @@ AsynchronousSymbolQuery::AsynchronousSymbolQuery( OutstandingSymbolsCount = Symbols.size(); for (auto &KV : Symbols) - ResolvedSymbols[KV.first] = nullptr; + ResolvedSymbols[KV.first] = ExecutorSymbolDef(); } void AsynchronousSymbolQuery::notifySymbolMetRequiredState( - const SymbolStringPtr &Name, JITEvaluatedSymbol Sym) { + const SymbolStringPtr &Name, ExecutorSymbolDef Sym) { auto I = ResolvedSymbols.find(Name); assert(I != ResolvedSymbols.end() && "Resolving symbol outside the requested set"); - assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); + assert(I->second == ExecutorSymbolDef() && + "Redundantly resolving symbol Name"); // If this is a materialization-side-effects-only symbol then drop it, // otherwise update its map entry with its resolved address. @@ -447,8 +448,8 @@ void ReExportsMaterializationUnit::materialize( if (KV.second.AliasFlags.hasMaterializationSideEffectsOnly()) continue; - ResolutionMap[KV.first] = JITEvaluatedSymbol( - (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); + ResolutionMap[KV.first] = {(*Result)[KV.second.Aliasee].getAddress(), + KV.second.AliasFlags}; } if (auto Err = QueryInfo->R->notifyResolved(ResolutionMap)) { ES.reportError(std::move(Err)); @@ -688,11 +689,15 @@ void JITDylib::removeGenerator(DefinitionGenerator &G) { } Expected<SymbolFlagsMap> -JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { +JITDylib::defineMaterializing(MaterializationResponsibility &FromMR, + SymbolFlagsMap SymbolFlags) { return ES.runSessionLocked([&]() -> Expected<SymbolFlagsMap> { - std::vector<SymbolTable::iterator> AddedSyms; - std::vector<SymbolFlagsMap::iterator> RejectedWeakDefs; + if (FromMR.RT->isDefunct()) + return make_error<ResourceTrackerDefunct>(FromMR.RT); + + std::vector<NonOwningSymbolStringPtr> AddedSyms; + std::vector<NonOwningSymbolStringPtr> RejectedWeakDefs; for (auto SFItr = SymbolFlags.begin(), SFEnd = SymbolFlags.end(); SFItr != SFEnd; ++SFItr) { @@ -708,27 +713,27 @@ JITDylib::defineMaterializing(SymbolFlagsMap SymbolFlags) { // If this is a strong definition then error out. if (!Flags.isWeak()) { // Remove any symbols already added. - for (auto &SI : AddedSyms) - Symbols.erase(SI); + for (auto &S : AddedSyms) + Symbols.erase(Symbols.find_as(S)); // FIXME: Return all duplicates. return make_error<DuplicateDefinition>(std::string(*Name)); } // Otherwise just make a note to discard this symbol after the loop. - RejectedWeakDefs.push_back(SFItr); + RejectedWeakDefs.push_back(NonOwningSymbolStringPtr(Name)); continue; } else EntryItr = Symbols.insert(std::make_pair(Name, SymbolTableEntry(Flags))).first; - AddedSyms.push_back(EntryItr); + AddedSyms.push_back(NonOwningSymbolStringPtr(Name)); EntryItr->second.setState(SymbolState::Materializing); } // Remove any rejected weak definitions from the SymbolFlags map. while (!RejectedWeakDefs.empty()) { - SymbolFlags.erase(RejectedWeakDefs.back()); + SymbolFlags.erase(SymbolFlags.find_as(RejectedWeakDefs.back())); RejectedWeakDefs.pop_back(); } @@ -944,7 +949,7 @@ Error JITDylib::resolve(MaterializationResponsibility &MR, struct WorklistEntry { SymbolTable::iterator SymI; - JITEvaluatedSymbol ResolvedSym; + ExecutorSymbolDef ResolvedSym; }; SymbolNameSet SymbolsInErrorState; @@ -964,7 +969,7 @@ Error JITDylib::resolve(MaterializationResponsibility &MR, "Resolving symbol with materializer attached?"); assert(SymI->second.getState() == SymbolState::Materializing && "Symbol should be materializing"); - assert(SymI->second.getAddress() == 0 && + assert(SymI->second.getAddress() == ExecutorAddr() && "Symbol has already been resolved"); if (SymI->second.getFlags().hasError()) @@ -976,8 +981,7 @@ Error JITDylib::resolve(MaterializationResponsibility &MR, (SymI->second.getFlags() & ~JITSymbolFlags::Common) && "Resolved flags should match the declared flags"); - Worklist.push_back( - {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)}); + Worklist.push_back({SymI, {KV.second.getAddress(), Flags}}); } } @@ -1328,6 +1332,18 @@ void JITDylib::setLinkOrder(JITDylibSearchOrder NewLinkOrder, }); } +void JITDylib::addToLinkOrder(const JITDylibSearchOrder &NewLinks) { + ES.runSessionLocked([&]() { + for (auto &KV : NewLinks) { + // Skip elements of NewLinks that are already in the link order. + if (llvm::find(LinkOrder, KV) != LinkOrder.end()) + continue; + + LinkOrder.push_back(std::move(KV)); + } + }); +} + void JITDylib::addToLinkOrder(JITDylib &JD, JITDylibLookupFlags JDLookupFlags) { ES.runSessionLocked([&]() { LinkOrder.push_back({&JD, JDLookupFlags}); }); } @@ -1437,16 +1453,23 @@ void JITDylib::dump(raw_ostream &OS) { OS << "Link order: " << LinkOrder << "\n" << "Symbol table:\n"; - for (auto &KV : Symbols) { + // Sort symbols so we get a deterministic order and can check them in tests. + std::vector<std::pair<SymbolStringPtr, SymbolTableEntry *>> SymbolsSorted; + for (auto &KV : Symbols) + SymbolsSorted.emplace_back(KV.first, &KV.second); + std::sort(SymbolsSorted.begin(), SymbolsSorted.end(), + [](const auto &L, const auto &R) { return *L.first < *R.first; }); + + for (auto &KV : SymbolsSorted) { OS << " \"" << *KV.first << "\": "; - if (auto Addr = KV.second.getAddress()) - OS << format("0x%016" PRIx64, Addr); + if (auto Addr = KV.second->getAddress()) + OS << Addr; else OS << "<not resolved> "; - OS << " " << KV.second.getFlags() << " " << KV.second.getState(); + OS << " " << KV.second->getFlags() << " " << KV.second->getState(); - if (KV.second.hasMaterializerAttached()) { + if (KV.second->hasMaterializerAttached()) { OS << " (Materializer "; auto I = UnmaterializedInfos.find(KV.first); assert(I != UnmaterializedInfos.end() && @@ -1940,6 +1963,7 @@ JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { JITDylib &ExecutionSession::createBareJITDylib(std::string Name) { assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { + assert(SessionOpen && "Cannot create JITDylib after session is closed"); JDs.push_back(new JITDylib(*this, std::move(Name))); return *JDs.back(); }); @@ -2156,7 +2180,7 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, #endif } -Expected<JITEvaluatedSymbol> +Expected<ExecutorSymbolDef> ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, SymbolStringPtr Name, SymbolState RequiredState) { SymbolLookupSet Names({Name}); @@ -2170,13 +2194,13 @@ ExecutionSession::lookup(const JITDylibSearchOrder &SearchOrder, return ResultMap.takeError(); } -Expected<JITEvaluatedSymbol> +Expected<ExecutorSymbolDef> ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, SymbolStringPtr Name, SymbolState RequiredState) { return lookup(makeJITDylibSearchOrder(SearchOrder), Name, RequiredState); } -Expected<JITEvaluatedSymbol> +Expected<ExecutorSymbolDef> ExecutionSession::lookup(ArrayRef<JITDylib *> SearchOrder, StringRef Name, SymbolState RequiredState) { return lookup(SearchOrder, intern(Name), RequiredState); @@ -2213,9 +2237,9 @@ Error ExecutionSession::registerJITDispatchHandlers( return Error::success(); } -void ExecutionSession::runJITDispatchHandler( - SendResultFunction SendResult, JITTargetAddress HandlerFnTagAddr, - ArrayRef<char> ArgBuffer) { +void ExecutionSession::runJITDispatchHandler(SendResultFunction SendResult, + ExecutorAddr HandlerFnTagAddr, + ArrayRef<char> ArgBuffer) { std::shared_ptr<JITDispatchHandlerFunction> F; { @@ -2666,7 +2690,7 @@ void ExecutionSession::OL_completeLookup( // whether it has a materializer attached, and if so prepare to run // it. if (SymI->second.hasMaterializerAttached()) { - assert(SymI->second.getAddress() == 0 && + assert(SymI->second.getAddress() == ExecutorAddr() && "Symbol not resolved but already has address?"); auto UMII = JD.UnmaterializedInfos.find(Name); assert(UMII != JD.UnmaterializedInfos.end() && @@ -2946,7 +2970,7 @@ Error ExecutionSession::OL_defineMaterializing( << NewSymbolFlags << "\n"; }); if (auto AcceptedDefs = - MR.JD.defineMaterializing(std::move(NewSymbolFlags))) { + MR.JD.defineMaterializing(MR, std::move(NewSymbolFlags))) { // Add all newly accepted symbols to this responsibility object. for (auto &KV : *AcceptedDefs) MR.SymbolFlags.insert(KV); diff --git a/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp index 02c3e617df68..acbf33888ade 100644 --- a/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp +++ b/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp @@ -60,26 +60,13 @@ public: private: typename ELFT::Shdr *Header; - - bool isTextOrDataSection() const; }; template <typename ELFT> void ELFDebugObjectSection<ELFT>::setTargetMemoryRange(SectionRange Range) { - // Only patch load-addresses for executable and data sections. - if (isTextOrDataSection()) - Header->sh_addr = - static_cast<typename ELFT::uint>(Range.getStart().getValue()); -} - -template <typename ELFT> -bool ELFDebugObjectSection<ELFT>::isTextOrDataSection() const { - switch (Header->sh_type) { - case ELF::SHT_PROGBITS: - case ELF::SHT_X86_64_UNWIND: - return Header->sh_flags & (ELF::SHF_EXECINSTR | ELF::SHF_ALLOC); - } - return false; + // All recorded sections are candidates for load-address patching. + Header->sh_addr = + static_cast<typename ELFT::uint>(Range.getStart().getValue()); } template <typename ELFT> @@ -106,16 +93,19 @@ Error ELFDebugObjectSection<ELFT>::validateInBounds(StringRef Buffer, template <typename ELFT> void ELFDebugObjectSection<ELFT>::dump(raw_ostream &OS, StringRef Name) { - if (auto Addr = static_cast<JITTargetAddress>(Header->sh_addr)) { + if (uint64_t Addr = Header->sh_addr) { OS << formatv(" {0:x16} {1}\n", Addr, Name); } else { OS << formatv(" {0}\n", Name); } } -enum class Requirement { +enum DebugObjectFlags : int { // Request final target memory load-addresses for all sections. - ReportFinalSectionLoadAddresses, + ReportFinalSectionLoadAddresses = 1 << 0, + + // We found sections with debug information when processing the input object. + HasDebugSections = 1 << 1, }; /// The plugin creates a debug object from when JITLink starts processing the @@ -127,10 +117,15 @@ class DebugObject { public: DebugObject(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD, ExecutionSession &ES) - : MemMgr(MemMgr), JD(JD), ES(ES) {} + : MemMgr(MemMgr), JD(JD), ES(ES), Flags(DebugObjectFlags{}) {} - void set(Requirement Req) { Reqs.insert(Req); } - bool has(Requirement Req) const { return Reqs.count(Req) > 0; } + bool hasFlags(DebugObjectFlags F) const { return Flags & F; } + void setFlags(DebugObjectFlags F) { + Flags = static_cast<DebugObjectFlags>(Flags | F); + } + void clearFlags(DebugObjectFlags F) { + Flags = static_cast<DebugObjectFlags>(Flags & ~F); + } using FinalizeContinuation = std::function<void(Expected<ExecutorAddrRange>)>; @@ -159,7 +154,7 @@ protected: private: ExecutionSession &ES; - std::set<Requirement> Reqs; + DebugObjectFlags Flags; FinalizedAlloc Alloc; }; @@ -171,8 +166,7 @@ void DebugObject::finalizeAsync(FinalizeContinuation OnFinalize) { if (auto SimpleSegAlloc = finalizeWorkingMemory()) { auto ROSeg = SimpleSegAlloc->getSegInfo(MemProt::Read); - ExecutorAddrRange DebugObjRange(ExecutorAddr(ROSeg.Addr), - ExecutorAddrDiff(ROSeg.WorkingMem.size())); + ExecutorAddrRange DebugObjRange(ROSeg.Addr, ROSeg.WorkingMem.size()); SimpleSegAlloc->finalize( [this, DebugObjRange, OnFinalize = std::move(OnFinalize)](Expected<FinalizedAlloc> FA) { @@ -222,7 +216,7 @@ private: JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD, ExecutionSession &ES) : DebugObject(MemMgr, JD, ES), Buffer(std::move(Buffer)) { - set(Requirement::ReportFinalSectionLoadAddresses); + setFlags(ReportFinalSectionLoadAddresses); } std::unique_ptr<WritableMemoryBuffer> Buffer; @@ -271,24 +265,23 @@ ELFDebugObject::CreateArchType(MemoryBufferRef Buffer, if (!ObjRef) return ObjRef.takeError(); - // TODO: Add support for other architectures. - uint16_t TargetMachineArch = ObjRef->getHeader().e_machine; - if (TargetMachineArch != ELF::EM_X86_64) - return nullptr; - Expected<ArrayRef<SectionHeader>> Sections = ObjRef->sections(); if (!Sections) return Sections.takeError(); - bool HasDwarfSection = false; for (const SectionHeader &Header : *Sections) { Expected<StringRef> Name = ObjRef->getSectionName(Header); if (!Name) return Name.takeError(); if (Name->empty()) continue; - HasDwarfSection |= isDwarfSection(*Name); + if (isDwarfSection(*Name)) + DebugObj->setFlags(HasDebugSections); + // Only record text and data sections (i.e. no bss, comments, rel, etc.) + if (Header.sh_type != ELF::SHT_PROGBITS && + Header.sh_type != ELF::SHT_X86_64_UNWIND) + continue; if (!(Header.sh_flags & ELF::SHF_ALLOC)) continue; @@ -297,13 +290,6 @@ ELFDebugObject::CreateArchType(MemoryBufferRef Buffer, return std::move(Err); } - if (!HasDwarfSection) { - LLVM_DEBUG(dbgs() << "Aborting debug registration for LinkGraph \"" - << DebugObj->Buffer->getBufferIdentifier() - << "\": input object contains no debug info\n"); - return nullptr; - } - return std::move(DebugObj); } @@ -371,12 +357,11 @@ Error ELFDebugObject::recordSection( StringRef Name, std::unique_ptr<ELFDebugObjectSection<ELFT>> Section) { if (Error Err = Section->validateInBounds(this->getBuffer(), Name.data())) return Err; - auto ItInserted = Sections.try_emplace(Name, std::move(Section)); - if (!ItInserted.second) - return make_error<StringError>("In " + Buffer->getBufferIdentifier() + - ", encountered duplicate section \"" + - Name + "\" while building debug object", - inconvertibleErrorCode()); + bool Inserted = Sections.try_emplace(Name, std::move(Section)).second; + if (!Inserted) + LLVM_DEBUG(dbgs() << "Skipping debug registration for section '" << Name + << "' in object " << Buffer->getBufferIdentifier() + << " (duplicate name)\n"); return Error::success(); } @@ -403,8 +388,15 @@ createDebugObjectFromBuffer(ExecutionSession &ES, LinkGraph &G, } DebugObjectManagerPlugin::DebugObjectManagerPlugin( + ExecutionSession &ES, std::unique_ptr<DebugObjectRegistrar> Target, + bool RequireDebugSections, bool AutoRegisterCode) + : ES(ES), Target(std::move(Target)), + RequireDebugSections(RequireDebugSections), + AutoRegisterCode(AutoRegisterCode) {} + +DebugObjectManagerPlugin::DebugObjectManagerPlugin( ExecutionSession &ES, std::unique_ptr<DebugObjectRegistrar> Target) - : ES(ES), Target(std::move(Target)) {} + : DebugObjectManagerPlugin(ES, std::move(Target), true, true) {} DebugObjectManagerPlugin::~DebugObjectManagerPlugin() = default; @@ -418,8 +410,14 @@ void DebugObjectManagerPlugin::notifyMaterializing( if (auto DebugObj = createDebugObjectFromBuffer(ES, G, Ctx, ObjBuffer)) { // Not all link artifacts allow debugging. - if (*DebugObj != nullptr) - PendingObjs[&MR] = std::move(*DebugObj); + if (*DebugObj == nullptr) + return; + if (RequireDebugSections && !(**DebugObj).hasFlags(HasDebugSections)) { + LLVM_DEBUG(dbgs() << "Skipping debug registration for LinkGraph '" + << G.getName() << "': no debug info\n"); + return; + } + PendingObjs[&MR] = std::move(*DebugObj); } else { ES.reportError(DebugObj.takeError()); } @@ -435,7 +433,7 @@ void DebugObjectManagerPlugin::modifyPassConfig( return; DebugObject &DebugObj = *It->second; - if (DebugObj.has(Requirement::ReportFinalSectionLoadAddresses)) { + if (DebugObj.hasFlags(ReportFinalSectionLoadAddresses)) { PassConfig.PostAllocationPasses.push_back( [&DebugObj](LinkGraph &Graph) -> Error { for (const Section &GraphSection : Graph.sections()) @@ -467,7 +465,8 @@ Error DebugObjectManagerPlugin::notifyEmitted( FinalizePromise.set_value(TargetMem.takeError()); return; } - if (Error Err = Target->registerDebugObject(*TargetMem)) { + if (Error Err = + Target->registerDebugObject(*TargetMem, AutoRegisterCode)) { FinalizePromise.set_value(std::move(Err)); return; } diff --git a/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp index 028bd245fb55..aca457642212 100644 --- a/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/DebugUtils.cpp @@ -172,9 +172,8 @@ raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) { return OS; } -raw_ostream &operator<<(raw_ostream &OS, const JITEvaluatedSymbol &Sym) { - return OS << format("0x%016" PRIx64, Sym.getAddress()) << " " - << Sym.getFlags(); +raw_ostream &operator<<(raw_ostream &OS, const ExecutorSymbolDef &Sym) { + return OS << Sym.getAddress() << " " << Sym.getFlags(); } raw_ostream &operator<<(raw_ostream &OS, const SymbolFlagsMap::value_type &KV) { @@ -299,8 +298,12 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPool &SSP) { std::lock_guard<std::mutex> Lock(SSP.PoolMutex); + SmallVector<std::pair<StringRef, int>, 0> Vec; for (auto &KV : SSP.Pool) - OS << KV.first() << ": " << KV.second << "\n"; + Vec.emplace_back(KV.first(), KV.second); + llvm::sort(Vec, less_first()); + for (auto &[K, V] : Vec) + OS << K << ": " << V << "\n"; return OS; } diff --git a/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp b/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp index 15e7ffb2f75a..830582bb3649 100644 --- a/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp +++ b/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp @@ -348,11 +348,12 @@ public: Writer.write(SecCmd); } + static constexpr bool AutoRegisterCode = true; SectionRange R(MachOContainerBlock->getSection()); G.allocActions().push_back( {cantFail(shared::WrapperFunctionCall::Create< - shared::SPSArgList<shared::SPSExecutorAddrRange>>( - RegisterActionAddr, R.getRange())), + shared::SPSArgList<shared::SPSExecutorAddrRange, bool>>( + RegisterActionAddr, R.getRange(), AutoRegisterCode)), {}}); return Error::success(); } @@ -377,11 +378,11 @@ GDBJITDebugInfoRegistrationPlugin::Create(ExecutionSession &ES, ? ES.intern("_llvm_orc_registerJITLoaderGDBAllocAction") : ES.intern("llvm_orc_registerJITLoaderGDBAllocAction"); - if (auto Addr = ES.lookup({&ProcessJD}, RegisterActionAddr)) + if (auto RegisterSym = ES.lookup({&ProcessJD}, RegisterActionAddr)) return std::make_unique<GDBJITDebugInfoRegistrationPlugin>( - ExecutorAddr(Addr->getAddress())); + RegisterSym->getAddress()); else - return Addr.takeError(); + return RegisterSym.takeError(); } Error GDBJITDebugInfoRegistrationPlugin::notifyFailed( diff --git a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp index 00032e4dca3f..1bb4ecdff299 100644 --- a/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp @@ -14,6 +14,7 @@ #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" #include <optional> @@ -40,8 +41,7 @@ public: unsigned PointerSize; support::endianness Endianness; jitlink::Edge::Kind EdgeKind; - const auto &TT = - ENP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + const auto &TT = ENP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::x86_64: @@ -96,31 +96,24 @@ private: ELFNixPlatform &ENP; }; -StringRef EHFrameSectionName = ".eh_frame"; -StringRef InitArrayFuncSectionName = ".init_array"; - -StringRef ThreadBSSSectionName = ".tbss"; -StringRef ThreadDataSectionName = ".tdata"; - } // end anonymous namespace namespace llvm { namespace orc { -Expected<std::unique_ptr<ELFNixPlatform>> -ELFNixPlatform::Create(ExecutionSession &ES, - ObjectLinkingLayer &ObjLinkingLayer, - JITDylib &PlatformJD, const char *OrcRuntimePath, - std::optional<SymbolAliasMap> RuntimeAliases) { - - auto &EPC = ES.getExecutorProcessControl(); +Expected<std::unique_ptr<ELFNixPlatform>> ELFNixPlatform::Create( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntime, + std::optional<SymbolAliasMap> RuntimeAliases) { // If the target is not supported then bail out immediately. - if (!supportedTarget(EPC.getTargetTriple())) + if (!supportedTarget(ES.getTargetTriple())) return make_error<StringError>("Unsupported ELFNixPlatform triple: " + - EPC.getTargetTriple().str(), + ES.getTargetTriple().str(), inconvertibleErrorCode()); + auto &EPC = ES.getExecutorProcessControl(); + // Create default aliases if the caller didn't supply any. if (!RuntimeAliases) { auto StandardRuntimeAliases = standardPlatformAliases(ES, PlatformJD); @@ -134,31 +127,41 @@ ELFNixPlatform::Create(ExecutionSession &ES, return std::move(Err); // Add JIT-dispatch function support symbols. - if (auto Err = PlatformJD.define(absoluteSymbols( - {{ES.intern("__orc_rt_jit_dispatch"), - {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), - JITSymbolFlags::Exported}}, - {ES.intern("__orc_rt_jit_dispatch_ctx"), - {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), - JITSymbolFlags::Exported}}}))) + if (auto Err = PlatformJD.define( + absoluteSymbols({{ES.intern("__orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunction, + JITSymbolFlags::Exported}}, + {ES.intern("__orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContext, + JITSymbolFlags::Exported}}}))) return std::move(Err); - // Create a generator for the ORC runtime archive. - auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load( - ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple()); - if (!OrcRuntimeArchiveGenerator) - return OrcRuntimeArchiveGenerator.takeError(); - // Create the instance. Error Err = Error::success(); - auto P = std::unique_ptr<ELFNixPlatform>( - new ELFNixPlatform(ES, ObjLinkingLayer, PlatformJD, - std::move(*OrcRuntimeArchiveGenerator), Err)); + auto P = std::unique_ptr<ELFNixPlatform>(new ELFNixPlatform( + ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err)); if (Err) return std::move(Err); return std::move(P); } +Expected<std::unique_ptr<ELFNixPlatform>> +ELFNixPlatform::Create(ExecutionSession &ES, + ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + std::optional<SymbolAliasMap> RuntimeAliases) { + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = + StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + return Create(ES, ObjLinkingLayer, PlatformJD, + std::move(*OrcRuntimeArchiveGenerator), + std::move(RuntimeAliases)); +} + Error ELFNixPlatform::setupJITDylib(JITDylib &JD) { return JD.define( std::make_unique<DSOHandleMaterializationUnit>(*this, DSOHandleSymbol)); @@ -204,47 +207,6 @@ ELFNixPlatform::standardPlatformAliases(ExecutionSession &ES, SymbolAliasMap Aliases; addAliases(ES, Aliases, requiredCXXAliases()); addAliases(ES, Aliases, standardRuntimeUtilityAliases()); - - // Determine whether or not the libunwind extended-API function for - // dynamically registering an entire .eh_frame section is available. - // If it is not, we assume that libgcc_s is being used, and alias to - // its __register_frame with the same functionality. - auto RTRegisterFrame = ES.intern("__orc_rt_register_eh_frame_section"); - auto LibUnwindRegisterFrame = ES.intern("__unw_add_dynamic_eh_frame_section"); - auto RTDeregisterFrame = ES.intern("__orc_rt_deregister_eh_frame_section"); - auto LibUnwindDeregisterFrame = - ES.intern("__unw_remove_dynamic_eh_frame_section"); - auto SM = ES.lookup(makeJITDylibSearchOrder(&PlatformJD), - SymbolLookupSet() - .add(LibUnwindRegisterFrame, - SymbolLookupFlags::WeaklyReferencedSymbol) - .add(LibUnwindDeregisterFrame, - SymbolLookupFlags::WeaklyReferencedSymbol)); - if (!SM) { // Weak-ref means no "missing symbol" errors, so this must be - // something more serious that we should report. - return SM.takeError(); - } else if (SM->size() == 2) { - LLVM_DEBUG({ - dbgs() << "Using libunwind " << LibUnwindRegisterFrame - << " for unwind info registration\n"; - }); - Aliases[std::move(RTRegisterFrame)] = {LibUnwindRegisterFrame, - JITSymbolFlags::Exported}; - Aliases[std::move(RTDeregisterFrame)] = {LibUnwindDeregisterFrame, - JITSymbolFlags::Exported}; - } else { - // Since LLVM libunwind is not present, we assume that unwinding - // is provided by libgcc - LLVM_DEBUG({ - dbgs() << "Using libgcc __register_frame" - << " for unwind info registration\n"; - }); - Aliases[std::move(RTRegisterFrame)] = {ES.intern("__register_frame"), - JITSymbolFlags::Exported}; - Aliases[std::move(RTDeregisterFrame)] = {ES.intern("__deregister_frame"), - JITSymbolFlags::Exported}; - } - return Aliases; } @@ -272,13 +234,6 @@ ELFNixPlatform::standardRuntimeUtilityAliases() { StandardRuntimeUtilityAliases); } -bool ELFNixPlatform::isInitializerSection(StringRef SecName) { - if (SecName.consume_front(InitArrayFuncSectionName) && - (SecName.empty() || SecName[0] == '.')) - return true; - return false; -} - bool ELFNixPlatform::supportedTarget(const Triple &TT) { switch (TT.getArch()) { case Triple::x86_64: @@ -433,8 +388,7 @@ void ELFNixPlatform::rt_getInitializers(SendInitializerSequenceFn SendResult, void ELFNixPlatform::rt_getDeinitializers( SendDeinitializerSequenceFn SendResult, ExecutorAddr Handle) { LLVM_DEBUG({ - dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" - << formatv("{0:x}", Handle.getValue()) << "\")\n"; + dbgs() << "ELFNixPlatform::rt_getDeinitializers(\"" << Handle << "\")\n"; }); JITDylib *JD = nullptr; @@ -447,12 +401,9 @@ void ELFNixPlatform::rt_getDeinitializers( } if (!JD) { - LLVM_DEBUG({ - dbgs() << " No JITDylib for handle " - << formatv("{0:x}", Handle.getValue()) << "\n"; - }); + LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + - formatv("{0:x}", Handle.getValue()), + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } @@ -464,8 +415,7 @@ void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, StringRef SymbolName) { LLVM_DEBUG({ - dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" - << formatv("{0:x}", Handle.getValue()) << "\")\n"; + dbgs() << "ELFNixPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"; }); JITDylib *JD = nullptr; @@ -478,12 +428,9 @@ void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, } if (!JD) { - LLVM_DEBUG({ - dbgs() << " No JITDylib for handle " - << formatv("{0:x}", Handle.getValue()) << "\n"; - }); + LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + - formatv("{0:x}", Handle.getValue()), + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } @@ -496,7 +443,7 @@ void ELFNixPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, void operator()(Expected<SymbolMap> Result) { if (Result) { assert(Result->size() == 1 && "Unexpected result map count"); - SendResult(ExecutorAddr(Result->begin()->second.getAddress())); + SendResult(Result->begin()->second.getAddress()); } else { SendResult(Result.takeError()); } @@ -538,7 +485,7 @@ Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) { for (const auto &KV : AddrsToRecord) { auto &Name = KV.first; assert(RuntimeSymbolAddrs->count(Name) && "Missing runtime symbol?"); - KV.second->setValue((*RuntimeSymbolAddrs)[Name].getAddress()); + *KV.second = (*RuntimeSymbolAddrs)[Name].getAddress(); } auto PJDDSOHandle = ES.lookup( @@ -547,7 +494,8 @@ Error ELFNixPlatform::bootstrapELFNixRuntime(JITDylib &PlatformJD) { return PJDDSOHandle.takeError(); if (auto Err = ES.callSPSWrapper<void(uint64_t)>( - orc_rt_elfnix_platform_bootstrap, PJDDSOHandle->getAddress())) + orc_rt_elfnix_platform_bootstrap, + PJDDSOHandle->getAddress().getValue())) return Err; // FIXME: Ordering is fuzzy here. We're probably best off saying @@ -596,8 +544,7 @@ Error ELFNixPlatform::registerInitInfo( for (auto *Sec : InitSections) { // FIXME: Avoid copy here. jitlink::SectionRange R(*Sec); - InitSeq->InitSections[Sec->getName()].push_back( - {ExecutorAddr(R.getStart()), ExecutorAddr(R.getEnd())}); + InitSeq->InitSections[Sec->getName()].push_back(R.getRange()); } return Error::success(); @@ -724,20 +671,19 @@ void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses( Config.PostFixupPasses.push_back([this](jitlink::LinkGraph &G) -> Error { ELFPerObjectSectionsToRegister POSR; - if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { + if (auto *EHFrameSection = G.findSectionByName(ELFEHFrameSectionName)) { jitlink::SectionRange R(*EHFrameSection); if (!R.empty()) - POSR.EHFrameSection = {ExecutorAddr(R.getStart()), - ExecutorAddr(R.getEnd())}; + POSR.EHFrameSection = R.getRange(); } // Get a pointer to the thread data section if there is one. It will be used // below. jitlink::Section *ThreadDataSection = - G.findSectionByName(ThreadDataSectionName); + G.findSectionByName(ELFThreadDataSectionName); // Handle thread BSS section if there is one. - if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) { + if (auto *ThreadBSSSection = G.findSectionByName(ELFThreadBSSSectionName)) { // If there's already a thread data section in this graph then merge the // thread BSS section content into it, otherwise just treat the thread // BSS section as the thread data section. @@ -752,8 +698,7 @@ void ELFNixPlatform::ELFNixPlatformPlugin::addEHAndTLVSupportPasses( if (ThreadDataSection) { jitlink::SectionRange R(*ThreadDataSection); if (!R.empty()) - POSR.ThreadDataSection = {ExecutorAddr(R.getStart()), - ExecutorAddr(R.getEnd())}; + POSR.ThreadDataSection = R.getRange(); } if (POSR.EHFrameSection.Start || POSR.ThreadDataSection.Start) { @@ -781,7 +726,7 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::preserveInitSections( JITLinkSymbolSet InitSectionSymbols; for (auto &InitSection : G.sections()) { // Skip non-init sections. - if (!isInitializerSection(InitSection.getName())) + if (!isELFInitializerSection(InitSection.getName())) continue; // Make a pass over live symbols in the section: those blocks are already @@ -816,10 +761,10 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections( SmallVector<jitlink::Section *> InitSections; - LLVM_DEBUG({ dbgs() << "ELFNixPlatform::registerInitSections\n"; }); + LLVM_DEBUG(dbgs() << "ELFNixPlatform::registerInitSections\n"); for (auto &Sec : G.sections()) { - if (isInitializerSection(Sec.getName())) { + if (isELFInitializerSection(Sec.getName())) { InitSections.push_back(&Sec); } } @@ -829,8 +774,7 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::registerInitSections( dbgs() << "ELFNixPlatform: Scraped " << G.getName() << " init sections:\n"; for (auto *Sec : InitSections) { jitlink::SectionRange R(*Sec); - dbgs() << " " << Sec->getName() << ": " - << formatv("[ {0:x} -- {1:x} ]", R.getStart(), R.getEnd()) << "\n"; + dbgs() << " " << Sec->getName() << ": " << R.getRange() << "\n"; } }); diff --git a/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp b/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp index 30d641ee00cf..b8969de54936 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp @@ -45,14 +45,13 @@ Expected<std::unique_ptr<EPCDebugObjectRegistrar>> createJITLoaderGDBRegistrar( assert((*Result)[0].size() == 1 && "Unexpected number of addresses in result"); - return std::make_unique<EPCDebugObjectRegistrar>( - ES, ExecutorAddr((*Result)[0][0])); + return std::make_unique<EPCDebugObjectRegistrar>(ES, (*Result)[0][0]); } -Error EPCDebugObjectRegistrar::registerDebugObject( - ExecutorAddrRange TargetMem) { - return ES.callSPSWrapper<void(shared::SPSExecutorAddrRange)>(RegisterFn, - TargetMem); +Error EPCDebugObjectRegistrar::registerDebugObject(ExecutorAddrRange TargetMem, + bool AutoRegisterCode) { + return ES.callSPSWrapper<void(shared::SPSExecutorAddrRange, bool)>( + RegisterFn, TargetMem, AutoRegisterCode); } } // namespace orc diff --git a/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp b/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp index 1adcc9156957..46e16a55c7e1 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp @@ -53,8 +53,7 @@ Error EPCDynamicLibrarySearchGenerator::tryToGenerate( auto ResultI = Result->front().begin(); for (auto &KV : LookupSymbols) { if (*ResultI) - NewSymbols[KV.first] = - JITEvaluatedSymbol(ResultI->getValue(), JITSymbolFlags::Exported); + NewSymbols[KV.first] = {*ResultI, JITSymbolFlags::Exported}; ++ResultI; } diff --git a/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp b/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp index 3aa94a7f43e2..56cd982cd5e1 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp @@ -57,9 +57,8 @@ Expected<std::unique_ptr<EPCEHFrameRegistrar>> EPCEHFrameRegistrar::Create( auto RegisterEHFrameWrapperFnAddr = (*Result)[0][0]; auto DeregisterEHFrameWrapperFnAddr = (*Result)[0][1]; - return std::make_unique<EPCEHFrameRegistrar>( - ES, ExecutorAddr(RegisterEHFrameWrapperFnAddr), - ExecutorAddr(DeregisterEHFrameWrapperFnAddr)); + return std::make_unique<EPCEHFrameRegistrar>(ES, RegisterEHFrameWrapperFnAddr, + DeregisterEHFrameWrapperFnAddr); } Error EPCEHFrameRegistrar::registerEHFrames(ExecutorAddrRange EHFrameSection) { diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp index a3d857c3bfc4..b05f08fd7cdf 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp @@ -158,7 +158,7 @@ void EPCGenericJITLinkMemoryManager::completeAllocation( auto &SegInfo = SegInfos[AG]; SegInfo.ContentSize = Seg.ContentSize; SegInfo.ZeroFillSize = Seg.ZeroFillSize; - SegInfo.Addr = ExecutorAddr(Seg.Addr); + SegInfo.Addr = Seg.Addr; SegInfo.WorkingMem = Seg.WorkingMem; } diff --git a/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp index ec82081937e2..fbe25d70c38a 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp @@ -235,7 +235,7 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { for (unsigned I = 0; I != 3; ++I) { FR.Segments.push_back({}); auto &Seg = FR.Segments.back(); - Seg.AG = SegMemProts[I]; + Seg.RAG = SegMemProts[I]; Seg.Addr = RemoteAddrs[I]->Start; for (auto &SecAlloc : *SegSections[I]) { Seg.Size = alignTo(Seg.Size, SecAlloc.Align); diff --git a/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp index ddfb30500c7b..833be826f8ae 100644 --- a/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp @@ -58,16 +58,16 @@ public: Error deallocateStubs(); - Error createStub(StringRef StubName, JITTargetAddress StubAddr, + Error createStub(StringRef StubName, ExecutorAddr StubAddr, JITSymbolFlags StubFlags) override; Error createStubs(const StubInitsMap &StubInits) override; - JITEvaluatedSymbol findStub(StringRef Name, bool ExportedStubsOnly) override; + ExecutorSymbolDef findStub(StringRef Name, bool ExportedStubsOnly) override; - JITEvaluatedSymbol findPointer(StringRef Name) override; + ExecutorSymbolDef findPointer(StringRef Name) override; - Error updatePointer(StringRef Name, JITTargetAddress NewAddr) override; + Error updatePointer(StringRef Name, ExecutorAddr NewAddr) override; private: using StubInfo = std::pair<IndirectStubInfo, JITSymbolFlags>; @@ -118,12 +118,10 @@ Error EPCTrampolinePool::grow() { unsigned NumTrampolines = TrampolinesPerPage; auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); - EPCIU.getABISupport().writeTrampolines(SegInfo.WorkingMem.data(), - SegInfo.Addr.getValue(), - ResolverAddress, NumTrampolines); + EPCIU.getABISupport().writeTrampolines( + SegInfo.WorkingMem.data(), SegInfo.Addr, ResolverAddress, NumTrampolines); for (unsigned I = 0; I < NumTrampolines; ++I) - AvailableTrampolines.push_back(SegInfo.Addr.getValue() + - (I * TrampolineSize)); + AvailableTrampolines.push_back(SegInfo.Addr + (I * TrampolineSize)); auto FA = Alloc->finalize(); if (!FA) @@ -135,7 +133,7 @@ Error EPCTrampolinePool::grow() { } Error EPCIndirectStubsManager::createStub(StringRef StubName, - JITTargetAddress StubAddr, + ExecutorAddr StubAddr, JITSymbolFlags StubFlags) { StubInitsMap SIM; SIM[StubName] = std::make_pair(StubAddr, StubFlags); @@ -162,18 +160,16 @@ Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { unsigned ASIdx = 0; std::vector<tpctypes::UInt32Write> PtrUpdates; for (auto &SI : StubInits) - PtrUpdates.push_back( - {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), - static_cast<uint32_t>(SI.second.first)}); + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast<uint32_t>(SI.second.first.getValue())}); return MemAccess.writeUInt32s(PtrUpdates); } case 8: { unsigned ASIdx = 0; std::vector<tpctypes::UInt64Write> PtrUpdates; for (auto &SI : StubInits) - PtrUpdates.push_back( - {ExecutorAddr((*AvailableStubInfos)[ASIdx++].PointerAddress), - static_cast<uint64_t>(SI.second.first)}); + PtrUpdates.push_back({(*AvailableStubInfos)[ASIdx++].PointerAddress, + static_cast<uint64_t>(SI.second.first.getValue())}); return MemAccess.writeUInt64s(PtrUpdates); } default: @@ -182,27 +178,27 @@ Error EPCIndirectStubsManager::createStubs(const StubInitsMap &StubInits) { } } -JITEvaluatedSymbol EPCIndirectStubsManager::findStub(StringRef Name, - bool ExportedStubsOnly) { +ExecutorSymbolDef EPCIndirectStubsManager::findStub(StringRef Name, + bool ExportedStubsOnly) { std::lock_guard<std::mutex> Lock(ISMMutex); auto I = StubInfos.find(Name); if (I == StubInfos.end()) - return nullptr; + return ExecutorSymbolDef(); return {I->second.first.StubAddress, I->second.second}; } -JITEvaluatedSymbol EPCIndirectStubsManager::findPointer(StringRef Name) { +ExecutorSymbolDef EPCIndirectStubsManager::findPointer(StringRef Name) { std::lock_guard<std::mutex> Lock(ISMMutex); auto I = StubInfos.find(Name); if (I == StubInfos.end()) - return nullptr; + return ExecutorSymbolDef(); return {I->second.first.PointerAddress, I->second.second}; } Error EPCIndirectStubsManager::updatePointer(StringRef Name, - JITTargetAddress NewAddr) { + ExecutorAddr NewAddr) { - JITTargetAddress PtrAddr = 0; + ExecutorAddr PtrAddr; { std::lock_guard<std::mutex> Lock(ISMMutex); auto I = StubInfos.find(Name); @@ -215,11 +211,11 @@ Error EPCIndirectStubsManager::updatePointer(StringRef Name, auto &MemAccess = EPCIU.getExecutorProcessControl().getMemoryAccess(); switch (EPCIU.getABISupport().getPointerSize()) { case 4: { - tpctypes::UInt32Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); + tpctypes::UInt32Write PUpdate(PtrAddr, NewAddr.getValue()); return MemAccess.writeUInt32s(PUpdate); } case 8: { - tpctypes::UInt64Write PUpdate(ExecutorAddr(PtrAddr), NewAddr); + tpctypes::UInt64Write PUpdate(PtrAddr, NewAddr.getValue()); return MemAccess.writeUInt64s(PUpdate); } default: @@ -290,9 +286,9 @@ Error EPCIndirectionUtils::cleanup() { return Err; } -Expected<JITTargetAddress> -EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { +Expected<ExecutorAddr> +EPCIndirectionUtils::writeResolverBlock(ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { using namespace jitlink; assert(ABI && "ABI can not be null"); @@ -307,7 +303,7 @@ EPCIndirectionUtils::writeResolverBlock(JITTargetAddress ReentryFnAddr, return Alloc.takeError(); auto SegInfo = Alloc->getSegInfo(MemProt::Read | MemProt::Exec); - ResolverBlockAddr = SegInfo.Addr.getValue(); + ResolverBlockAddr = SegInfo.Addr; ABI->writeResolverCode(SegInfo.WorkingMem.data(), ResolverBlockAddr, ReentryFnAddr, ReentryCtxAddr); @@ -331,7 +327,7 @@ TrampolinePool &EPCIndirectionUtils::getTrampolinePool() { } LazyCallThroughManager &EPCIndirectionUtils::createLazyCallThroughManager( - ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr) { + ExecutionSession &ES, ExecutorAddr ErrorHandlerAddr) { assert(!LCTM && "createLazyCallThroughManager can not have been called before"); LCTM = std::make_unique<LazyCallThroughManager>(ES, ErrorHandlerAddr, @@ -377,9 +373,8 @@ EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { auto StubSeg = Alloc->getSegInfo(StubProt); auto PtrSeg = Alloc->getSegInfo(PtrProt); - ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), - StubSeg.Addr.getValue(), - PtrSeg.Addr.getValue(), NumStubsToAllocate); + ABI->writeIndirectStubsBlock(StubSeg.WorkingMem.data(), StubSeg.Addr, + PtrSeg.Addr, NumStubsToAllocate); auto FA = Alloc->finalize(); if (!FA) @@ -390,8 +385,8 @@ EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { auto StubExecutorAddr = StubSeg.Addr; auto PtrExecutorAddr = PtrSeg.Addr; for (unsigned I = 0; I != NumStubsToAllocate; ++I) { - AvailableIndirectStubs.push_back(IndirectStubInfo( - StubExecutorAddr.getValue(), PtrExecutorAddr.getValue())); + AvailableIndirectStubs.push_back( + IndirectStubInfo(StubExecutorAddr, PtrExecutorAddr)); StubExecutorAddr += ABI->getStubSize(); PtrExecutorAddr += ABI->getPointerSize(); } @@ -412,19 +407,19 @@ EPCIndirectionUtils::getIndirectStubs(unsigned NumStubs) { static JITTargetAddress reentry(JITTargetAddress LCTMAddr, JITTargetAddress TrampolineAddr) { auto &LCTM = *jitTargetAddressToPointer<LazyCallThroughManager *>(LCTMAddr); - std::promise<JITTargetAddress> LandingAddrP; + std::promise<ExecutorAddr> LandingAddrP; auto LandingAddrF = LandingAddrP.get_future(); LCTM.resolveTrampolineLandingAddress( - TrampolineAddr, - [&](JITTargetAddress Addr) { LandingAddrP.set_value(Addr); }); - return LandingAddrF.get(); + ExecutorAddr(TrampolineAddr), + [&](ExecutorAddr Addr) { LandingAddrP.set_value(Addr); }); + return LandingAddrF.get().getValue(); } Error setUpInProcessLCTMReentryViaEPCIU(EPCIndirectionUtils &EPCIU) { auto &LCTM = EPCIU.getLazyCallThroughManager(); return EPCIU - .writeResolverBlock(pointerToJITTargetAddress(&reentry), - pointerToJITTargetAddress(&LCTM)) + .writeResolverBlock(ExecutorAddr::fromPtr(&reentry), + ExecutorAddr::fromPtr(&LCTM)) .takeError(); } diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 377a59993eb0..fb685e6c3727 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -156,8 +156,7 @@ Error CtorDtorRunner::run() { for (auto &KV : CtorDtorsByPriority) { for (auto &Name : KV.second) { assert(CtorDtorMap->count(Name) && "No entry for Name"); - auto CtorDtor = reinterpret_cast<CtorDtorTy>( - static_cast<uintptr_t>((*CtorDtorMap)[Name].getAddress())); + auto CtorDtor = (*CtorDtorMap)[Name].getAddress().toPtr<CtorDtorTy>(); CtorDtor(); } } @@ -186,12 +185,10 @@ int LocalCXXRuntimeOverridesBase::CXAAtExitOverride(DestructorPtr Destructor, Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, MangleAndInterner &Mangle) { SymbolMap RuntimeInterposes; - RuntimeInterposes[Mangle("__dso_handle")] = - JITEvaluatedSymbol(toTargetAddress(&DSOHandleOverride), - JITSymbolFlags::Exported); - RuntimeInterposes[Mangle("__cxa_atexit")] = - JITEvaluatedSymbol(toTargetAddress(&CXAAtExitOverride), - JITSymbolFlags::Exported); + RuntimeInterposes[Mangle("__dso_handle")] = { + ExecutorAddr::fromPtr(&DSOHandleOverride), JITSymbolFlags::Exported}; + RuntimeInterposes[Mangle("__cxa_atexit")] = { + ExecutorAddr::fromPtr(&CXAAtExitOverride), JITSymbolFlags::Exported}; return JD.define(absoluteSymbols(std::move(RuntimeInterposes))); } @@ -257,11 +254,8 @@ Error DynamicLibrarySearchGenerator::tryToGenerate( std::string Tmp((*Name).data() + HasGlobalPrefix, (*Name).size() - HasGlobalPrefix); - if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { - NewSymbols[Name] = JITEvaluatedSymbol( - static_cast<JITTargetAddress>(reinterpret_cast<uintptr_t>(Addr)), - JITSymbolFlags::Exported); - } + if (void *P = Dylib.getAddressOfSymbol(Tmp.c_str())) + NewSymbols[Name] = {ExecutorAddr::fromPtr(P), JITSymbolFlags::Exported}; } if (NewSymbols.empty()) @@ -274,57 +268,41 @@ Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> StaticLibraryDefinitionGenerator::Load( ObjectLayer &L, const char *FileName, GetObjectFileInterface GetObjFileInterface) { - auto ArchiveBuffer = MemoryBuffer::getFile(FileName); - - if (!ArchiveBuffer) - return createFileError(FileName, ArchiveBuffer.getError()); - - return Create(L, std::move(*ArchiveBuffer), std::move(GetObjFileInterface)); -} - -Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> -StaticLibraryDefinitionGenerator::Load( - ObjectLayer &L, const char *FileName, const Triple &TT, - GetObjectFileInterface GetObjFileInterface) { auto B = object::createBinary(FileName); if (!B) return createFileError(FileName, B.takeError()); // If this is a regular archive then create an instance from it. - if (isa<object::Archive>(B->getBinary())) - return Create(L, std::move(B->takeBinary().second), + if (isa<object::Archive>(B->getBinary())) { + auto [Archive, ArchiveBuffer] = B->takeBinary(); + return Create(L, std::move(ArchiveBuffer), + std::unique_ptr<object::Archive>( + static_cast<object::Archive *>(Archive.release())), std::move(GetObjFileInterface)); + } // If this is a universal binary then search for a slice matching the given // Triple. if (auto *UB = cast<object::MachOUniversalBinary>(B->getBinary())) { - for (const auto &Obj : UB->objects()) { - auto ObjTT = Obj.getTriple(); - if (ObjTT.getArch() == TT.getArch() && - ObjTT.getSubArch() == TT.getSubArch() && - (TT.getVendor() == Triple::UnknownVendor || - ObjTT.getVendor() == TT.getVendor())) { - // We found a match. Create an instance from a buffer covering this - // slice. - auto SliceBuffer = MemoryBuffer::getFileSlice(FileName, Obj.getSize(), - Obj.getOffset()); - if (!SliceBuffer) - return make_error<StringError>( - Twine("Could not create buffer for ") + TT.str() + " slice of " + - FileName + ": [ " + formatv("{0:x}", Obj.getOffset()) + - " .. " + formatv("{0:x}", Obj.getOffset() + Obj.getSize()) + - ": " + SliceBuffer.getError().message(), - SliceBuffer.getError()); - return Create(L, std::move(*SliceBuffer), - std::move(GetObjFileInterface)); - } - } - return make_error<StringError>(Twine("Universal binary ") + FileName + - " does not contain a slice for " + - TT.str(), - inconvertibleErrorCode()); + const auto &TT = L.getExecutionSession().getTargetTriple(); + + auto SliceRange = getSliceRangeForArch(*UB, TT); + if (!SliceRange) + return SliceRange.takeError(); + + auto SliceBuffer = MemoryBuffer::getFileSlice(FileName, SliceRange->second, + SliceRange->first); + if (!SliceBuffer) + return make_error<StringError>( + Twine("Could not create buffer for ") + TT.str() + " slice of " + + FileName + ": [ " + formatv("{0:x}", SliceRange->first) + " .. " + + formatv("{0:x}", SliceRange->first + SliceRange->second) + ": " + + SliceBuffer.getError().message(), + SliceBuffer.getError()); + + return Create(L, std::move(*SliceBuffer), std::move(GetObjFileInterface)); } return make_error<StringError>(Twine("Unrecognized file type for ") + @@ -335,12 +313,15 @@ StaticLibraryDefinitionGenerator::Load( Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> StaticLibraryDefinitionGenerator::Create( ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, + std::unique_ptr<object::Archive> Archive, GetObjectFileInterface GetObjFileInterface) { + Error Err = Error::success(); std::unique_ptr<StaticLibraryDefinitionGenerator> ADG( new StaticLibraryDefinitionGenerator( - L, std::move(ArchiveBuffer), std::move(GetObjFileInterface), Err)); + L, std::move(ArchiveBuffer), std::move(Archive), + std::move(GetObjFileInterface), Err)); if (Err) return std::move(Err); @@ -348,6 +329,50 @@ StaticLibraryDefinitionGenerator::Create( return std::move(ADG); } +Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> +StaticLibraryDefinitionGenerator::Create( + ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, + GetObjectFileInterface GetObjFileInterface) { + + auto B = object::createBinary(ArchiveBuffer->getMemBufferRef()); + if (!B) + return B.takeError(); + + // If this is a regular archive then create an instance from it. + if (isa<object::Archive>(*B)) + return Create(L, std::move(ArchiveBuffer), + std::unique_ptr<object::Archive>( + static_cast<object::Archive *>(B->release())), + std::move(GetObjFileInterface)); + + // If this is a universal binary then search for a slice matching the given + // Triple. + if (auto *UB = cast<object::MachOUniversalBinary>(B->get())) { + + const auto &TT = L.getExecutionSession().getTargetTriple(); + + auto SliceRange = getSliceRangeForArch(*UB, TT); + if (!SliceRange) + return SliceRange.takeError(); + + MemoryBufferRef SliceRef( + StringRef(ArchiveBuffer->getBufferStart() + SliceRange->first, + SliceRange->second), + ArchiveBuffer->getBufferIdentifier()); + + auto Archive = object::Archive::create(SliceRef); + if (!Archive) + return Archive.takeError(); + + return Create(L, std::move(ArchiveBuffer), std::move(*Archive), + std::move(GetObjFileInterface)); + } + + return make_error<StringError>(Twine("Unrecognized file type for ") + + ArchiveBuffer->getBufferIdentifier(), + inconvertibleErrorCode()); +} + Error StaticLibraryDefinitionGenerator::tryToGenerate( LookupState &LS, LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { @@ -417,12 +442,33 @@ Error StaticLibraryDefinitionGenerator::buildObjectFilesMap() { return Error::success(); } +Expected<std::pair<size_t, size_t>> +StaticLibraryDefinitionGenerator::getSliceRangeForArch( + object::MachOUniversalBinary &UB, const Triple &TT) { + + for (const auto &Obj : UB.objects()) { + auto ObjTT = Obj.getTriple(); + if (ObjTT.getArch() == TT.getArch() && + ObjTT.getSubArch() == TT.getSubArch() && + (TT.getVendor() == Triple::UnknownVendor || + ObjTT.getVendor() == TT.getVendor())) { + // We found a match. Return the range for the slice. + return std::make_pair(Obj.getOffset(), Obj.getSize()); + } + } + + return make_error<StringError>(Twine("Universal binary ") + UB.getFileName() + + " does not contain a slice for " + + TT.str(), + inconvertibleErrorCode()); +} + StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator( ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, + std::unique_ptr<object::Archive> Archive, GetObjectFileInterface GetObjFileInterface, Error &Err) : L(L), GetObjFileInterface(std::move(GetObjFileInterface)), - ArchiveBuffer(std::move(ArchiveBuffer)), - Archive(std::make_unique<object::Archive>(*this->ArchiveBuffer, Err)) { + ArchiveBuffer(std::move(ArchiveBuffer)), Archive(std::move(Archive)) { ErrorAsOutParameter _(&Err); if (!this->GetObjFileInterface) this->GetObjFileInterface = getObjectFileInterface; @@ -506,7 +552,7 @@ DLLImportDefinitionGenerator::getTargetEndianness(const Triple &TT) { Expected<std::unique_ptr<jitlink::LinkGraph>> DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) { - Triple TT = ES.getExecutorProcessControl().getTargetTriple(); + Triple TT = ES.getTargetTriple(); auto PointerSize = getTargetEndianness(TT); if (!PointerSize) return PointerSize.takeError(); @@ -522,13 +568,13 @@ DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) { for (auto &KV : Resolved) { jitlink::Symbol &Target = G->addAbsoluteSymbol( - *KV.first, ExecutorAddr(KV.second.getAddress()), *PointerSize, + *KV.first, KV.second.getAddress(), *PointerSize, jitlink::Linkage::Strong, jitlink::Scope::Local, false); // Create __imp_ symbol jitlink::Symbol &Ptr = jitlink::x86_64::createAnonymousPointer(*G, Sec, &Target); - auto NameCopy = G->allocateString(Twine(getImpPrefix()) + *KV.first); + auto NameCopy = G->allocateContent(Twine(getImpPrefix()) + *KV.first); StringRef NameCopyRef = StringRef(NameCopy.data(), NameCopy.size()); Ptr.setName(NameCopyRef); Ptr.setLinkage(jitlink::Linkage::Strong); diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index 361fcd4a2e9c..b8b013f8a7a9 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -11,8 +11,8 @@ #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/Host.h" #include "llvm/Support/Process.h" +#include "llvm/TargetParser/Host.h" #define DEBUG_TYPE "orc" @@ -192,7 +192,7 @@ SelfExecutorProcessControl::jitDispatchViaWrapperFunctionManager( shared::WrapperFunctionResult Result) mutable { ResultP.set_value(std::move(Result)); }, - pointerToJITTargetAddress(FnTag), {Data, Size}); + ExecutorAddr::fromPtr(FnTag), {Data, Size}); return ResultF.get().release(); } diff --git a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 989bb094cc25..a0d81cdf2086 100644 --- a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -8,13 +8,13 @@ #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" #include "llvm/IR/IRBuilder.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" #include "llvm/MC/MCInstrAnalysis.h" #include "llvm/Support/Format.h" +#include "llvm/TargetParser/Triple.h" #include "llvm/Transforms/Utils/Cloning.h" #include <sstream> @@ -40,7 +40,7 @@ public: private: void materialize(std::unique_ptr<MaterializationResponsibility> R) override { SymbolMap Result; - Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); + Result[Name] = {Compile(), JITSymbolFlags::Exported}; // No dependencies, so these calls cannot fail. cantFail(R->notifyResolved(Result)); cantFail(R->notifyEmitted()); @@ -62,7 +62,7 @@ namespace orc { TrampolinePool::~TrampolinePool() = default; void IndirectStubsManager::anchor() {} -Expected<JITTargetAddress> +Expected<ExecutorAddr> JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { if (auto TrampolineAddr = TP->getTrampoline()) { auto CallbackName = @@ -78,8 +78,8 @@ JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { return TrampolineAddr.takeError(); } -JITTargetAddress JITCompileCallbackManager::executeCompileCallback( - JITTargetAddress TrampolineAddr) { +ExecutorAddr +JITCompileCallbackManager::executeCompileCallback(ExecutorAddr TrampolineAddr) { SymbolStringPtr Name; { @@ -91,14 +91,10 @@ JITTargetAddress JITCompileCallbackManager::executeCompileCallback( // callee. if (I == AddrToSymbol.end()) { Lock.unlock(); - std::string ErrMsg; - { - raw_string_ostream ErrMsgStream(ErrMsg); - ErrMsgStream << "No compile callback for trampoline at " - << format("0x%016" PRIx64, TrampolineAddr); - } ES.reportError( - make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); + make_error<StringError>("No compile callback for trampoline at " + + formatv("{0:x}", TrampolineAddr), + inconvertibleErrorCode())); return ErrorHandlerAddress; } else Name = I->second; @@ -120,7 +116,7 @@ JITTargetAddress JITCompileCallbackManager::executeCompileCallback( Expected<std::unique_ptr<JITCompileCallbackManager>> createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, - JITTargetAddress ErrorHandlerAddress) { + ExecutorAddr ErrorHandlerAddress) { switch (T.getArch()) { default: return make_error<StringError>( @@ -244,9 +240,9 @@ createLocalIndirectStubsManagerBuilder(const Triple &T) { } } -Constant* createIRTypedAddress(FunctionType &FT, JITTargetAddress Addr) { +Constant* createIRTypedAddress(FunctionType &FT, ExecutorAddr Addr) { Constant *AddrIntVal = - ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr); + ConstantInt::get(Type::getInt64Ty(FT.getContext()), Addr.getValue()); Constant *AddrPtrVal = ConstantExpr::getCast(Instruction::IntToPtr, AddrIntVal, PointerType::get(&FT, 0)); @@ -329,26 +325,6 @@ Function* cloneFunctionDecl(Module &Dst, const Function &F, return NewF; } -void moveFunctionBody(Function &OrigF, ValueToValueMapTy &VMap, - ValueMaterializer *Materializer, - Function *NewF) { - assert(!OrigF.isDeclaration() && "Nothing to move"); - if (!NewF) - NewF = cast<Function>(VMap[&OrigF]); - else - assert(VMap[&OrigF] == NewF && "Incorrect function mapping in VMap."); - assert(NewF && "Function mapping missing from VMap."); - assert(NewF->getParent() != OrigF.getParent() && - "moveFunctionBody should only be used to move bodies between " - "modules."); - - SmallVector<ReturnInst *, 8> Returns; // Ignore returns cloned. - CloneFunctionInto(NewF, &OrigF, VMap, - CloneFunctionChangeType::DifferentModule, Returns, "", - nullptr, nullptr, Materializer); - OrigF.deleteBody(); -} - GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, ValueToValueMapTy *VMap) { GlobalVariable *NewGV = new GlobalVariable( @@ -361,24 +337,6 @@ GlobalVariable* cloneGlobalVariableDecl(Module &Dst, const GlobalVariable &GV, return NewGV; } -void moveGlobalVariableInitializer(GlobalVariable &OrigGV, - ValueToValueMapTy &VMap, - ValueMaterializer *Materializer, - GlobalVariable *NewGV) { - assert(OrigGV.hasInitializer() && "Nothing to move"); - if (!NewGV) - NewGV = cast<GlobalVariable>(VMap[&OrigGV]); - else - assert(VMap[&OrigGV] == NewGV && - "Incorrect global variable mapping in VMap."); - assert(NewGV->getParent() != OrigGV.getParent() && - "moveGlobalVariableInitializer should only be used to move " - "initializers between modules"); - - NewGV->setInitializer(MapValue(OrigGV.getInitializer(), VMap, RF_None, - nullptr, Materializer)); -} - GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, ValueToValueMapTy &VMap) { assert(OrigA.getAliasee() && "Original alias doesn't have an aliasee?"); @@ -390,15 +348,6 @@ GlobalAlias* cloneGlobalAliasDecl(Module &Dst, const GlobalAlias &OrigA, return NewA; } -void cloneModuleFlagsMetadata(Module &Dst, const Module &Src, - ValueToValueMapTy &VMap) { - auto *MFs = Src.getModuleFlagsMetadata(); - if (!MFs) - return; - for (auto *MF : MFs->operands()) - Dst.addModuleFlag(MapMetadata(MF, VMap)); -} - Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, jitlink::LinkGraph &G, MCDisassembler &Disassembler, diff --git a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index 70a3c404d836..b66f52f1ec5d 100644 --- a/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/llvm/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -9,8 +9,8 @@ #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/MC/TargetRegistry.h" -#include "llvm/Support/Host.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/Host.h" namespace llvm { namespace orc { @@ -18,13 +18,10 @@ namespace orc { JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT) : TT(std::move(TT)) { Options.EmulatedTLS = true; - Options.ExplicitEmulatedTLS = true; Options.UseInitArray = true; } Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() { - // FIXME: getProcessTriple is bogus. It returns the host LLVM was compiled on, - // rather than a valid triple for the current process. JITTargetMachineBuilder TMBuilder((Triple(sys::getProcessTriple()))); // Retrieve host CPU name and sub-target features and add them to builder. diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index bc84988e3254..7c7c2f000368 100644 --- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -9,6 +9,11 @@ #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/COFFPlatform.h" +#include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" +#include "llvm/ExecutionEngine/Orc/DebuggerSupportPlugin.h" +#include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" +#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" @@ -98,10 +103,16 @@ public: ORC_RT_RTLD_GLOBAL = 0x8 }; - if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlopen_wrapper")) { - return J.getExecutionSession().callSPSWrapper<SPSDLOpenSig>( - *WrapperAddr, DSOHandles[&JD], JD.getName(), - int32_t(ORC_RT_RTLD_LAZY)); + auto &ES = J.getExecutionSession(); + auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo( + [](const JITDylibSearchOrder &SO) { return SO; }); + + if (auto WrapperAddr = + ES.lookup(MainSearchOrder, + J.mangleAndIntern("__orc_rt_jit_dlopen_wrapper"))) { + return ES.callSPSWrapper<SPSDLOpenSig>(WrapperAddr->getAddress(), + DSOHandles[&JD], JD.getName(), + int32_t(ORC_RT_RTLD_LAZY)); } else return WrapperAddr.takeError(); } @@ -110,10 +121,16 @@ public: using llvm::orc::shared::SPSExecutorAddr; using SPSDLCloseSig = int32_t(SPSExecutorAddr); - if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlclose_wrapper")) { + auto &ES = J.getExecutionSession(); + auto MainSearchOrder = J.getMainJITDylib().withLinkOrderDo( + [](const JITDylibSearchOrder &SO) { return SO; }); + + if (auto WrapperAddr = + ES.lookup(MainSearchOrder, + J.mangleAndIntern("__orc_rt_jit_dlclose_wrapper"))) { int32_t result; auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>( - *WrapperAddr, result, DSOHandles[&JD]); + WrapperAddr->getAddress(), result, DSOHandles[&JD]); if (E) return E; else if (result) @@ -176,7 +193,7 @@ private: /// some runtime API, including __cxa_atexit, dlopen, and dlclose. class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { public: - GenericLLVMIRPlatformSupport(LLJIT &J) + GenericLLVMIRPlatformSupport(LLJIT &J, JITDylib &PlatformJD) : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")), DeInitFunctionPrefix(J.mangle("__orc_deinit_func.")) { @@ -188,17 +205,14 @@ public: SymbolMap StdInterposes; - StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(this), - JITSymbolFlags::Exported); - StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(registerCxaAtExitHelper), - JITSymbolFlags()); + StdInterposes[J.mangleAndIntern("__lljit.platform_support_instance")] = { + ExecutorAddr::fromPtr(this), JITSymbolFlags::Exported}; + StdInterposes[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = { + ExecutorAddr::fromPtr(registerCxaAtExitHelper), JITSymbolFlags()}; - cantFail( - J.getMainJITDylib().define(absoluteSymbols(std::move(StdInterposes)))); - cantFail(setupJITDylib(J.getMainJITDylib())); - cantFail(J.addIRModule(J.getMainJITDylib(), createPlatformRuntimeModule())); + cantFail(PlatformJD.define(absoluteSymbols(std::move(StdInterposes)))); + cantFail(setupJITDylib(PlatformJD)); + cantFail(J.addIRModule(PlatformJD, createPlatformRuntimeModule())); } ExecutionSession &getExecutionSession() { return J.getExecutionSession(); } @@ -208,12 +222,10 @@ public: // Add per-jitdylib standard interposes. SymbolMap PerJDInterposes; - PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper), - JITSymbolFlags()); - PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper), - JITSymbolFlags()); + PerJDInterposes[J.mangleAndIntern("__lljit.run_atexits_helper")] = { + ExecutorAddr::fromPtr(runAtExitsHelper), JITSymbolFlags()}; + PerJDInterposes[J.mangleAndIntern("__lljit.atexit_helper")] = { + ExecutorAddr::fromPtr(registerAtExitHelper), JITSymbolFlags()}; cantFail(JD.define(absoluteSymbols(std::move(PerJDInterposes)))); auto Ctx = std::make_unique<LLVMContext>(); @@ -227,7 +239,7 @@ public: "__dso_handle"); DSOHandle->setVisibility(GlobalValue::DefaultVisibility); DSOHandle->setInitializer( - ConstantInt::get(Int64Ty, pointerToJITTargetAddress(&JD))); + ConstantInt::get(Int64Ty, ExecutorAddr::fromPtr(&JD).getValue())); auto *GenericIRPlatformSupportTy = StructType::create(*Ctx, "lljit.GenericLLJITIRPlatformSupport"); @@ -287,7 +299,7 @@ public: dbgs() << " Running init " << formatv("{0:x16}", InitFnAddr) << "...\n"; }); - auto *InitFn = jitTargetAddressToFunction<void (*)()>(InitFnAddr); + auto *InitFn = InitFnAddr.toPtr<void (*)()>(); InitFn(); } } else @@ -308,7 +320,7 @@ public: dbgs() << " Running deinit " << formatv("{0:x16}", DeinitFnAddr) << "...\n"; }); - auto *DeinitFn = jitTargetAddressToFunction<void (*)()>(DeinitFnAddr); + auto *DeinitFn = DeinitFnAddr.toPtr<void (*)()>(); DeinitFn(); } } else @@ -329,8 +341,7 @@ public: } private: - - Expected<std::vector<JITTargetAddress>> getInitializers(JITDylib &JD) { + Expected<std::vector<ExecutorAddr>> getInitializers(JITDylib &JD) { if (auto Err = issueInitLookups(JD)) return std::move(Err); @@ -370,7 +381,7 @@ private: if (!LookupResult) return LookupResult.takeError(); - std::vector<JITTargetAddress> Initializers; + std::vector<ExecutorAddr> Initializers; while (!DFSLinkOrder.empty()) { auto &NextJD = *DFSLinkOrder.back(); DFSLinkOrder.pop_back(); @@ -384,7 +395,7 @@ private: return Initializers; } - Expected<std::vector<JITTargetAddress>> getDeinitializers(JITDylib &JD) { + Expected<std::vector<ExecutorAddr>> getDeinitializers(JITDylib &JD) { auto &ES = getExecutionSession(); auto LLJITRunAtExits = J.mangleAndIntern("__lljit_run_atexits"); @@ -427,7 +438,7 @@ private: if (!LookupResult) return LookupResult.takeError(); - std::vector<JITTargetAddress> DeInitializers; + std::vector<ExecutorAddr> DeInitializers; for (auto &NextJD : DFSLinkOrder) { auto DeInitsItr = LookupResult->find(NextJD.get()); assert(DeInitsItr != LookupResult->end() && @@ -695,6 +706,14 @@ Error LLJITBuilderState::prepareForConstruction() { dbgs() << "\n"; }); + // Create DL if not specified. + if (!DL) { + if (auto DLOrErr = JTMB->getDefaultDataLayoutForTarget()) + DL = std::move(*DLOrErr); + else + return DLOrErr.takeError(); + } + // If neither ES nor EPC has been set then create an EPC instance. if (!ES && !EPC) { LLVM_DEBUG({ @@ -705,21 +724,38 @@ Error LLJITBuilderState::prepareForConstruction() { EPC = std::move(*EPCOrErr); else return EPCOrErr.takeError(); - } else + } else if (EPC) { LLVM_DEBUG({ dbgs() << "Using explicitly specified ExecutorProcessControl instance " << EPC.get() << "\n"; }); + } else { + LLVM_DEBUG({ + dbgs() << "Using explicitly specified ExecutionSession instance " + << ES.get() << "\n"; + }); + } // If the client didn't configure any linker options then auto-configure the // JIT linker. if (!CreateObjectLinkingLayer) { auto &TT = JTMB->getTargetTriple(); - if (TT.getArch() == Triple::riscv64 || - TT.getArch() == Triple::loongarch64 || - (TT.isOSBinFormatMachO() && - (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64))) { - + bool UseJITLink = false; + switch (TT.getArch()) { + case Triple::riscv64: + case Triple::loongarch64: + UseJITLink = true; + break; + case Triple::aarch64: + UseJITLink = !TT.isOSBinFormatCOFF(); + break; + case Triple::x86_64: + UseJITLink = !TT.isOSBinFormatCOFF(); + break; + default: + break; + } + if (UseJITLink) { JTMB->setRelocationModel(Reloc::PIC_); JTMB->setCodeModel(CodeModel::Small); CreateObjectLinkingLayer = @@ -737,6 +773,30 @@ Error LLJITBuilderState::prepareForConstruction() { } } + // If we need a process JITDylib but no setup function has been given then + // create a default one. + if (!SetupProcessSymbolsJITDylib && + (LinkProcessSymbolsByDefault || EnableDebuggerSupport)) { + + LLVM_DEBUG({ + dbgs() << "Creating default Process JD setup function (neeeded for"; + if (LinkProcessSymbolsByDefault) + dbgs() << " <link-process-syms-by-default>"; + if (EnableDebuggerSupport) + dbgs() << " <debugger-support>"; + dbgs() << ")\n"; + }); + + SetupProcessSymbolsJITDylib = [this](JITDylib &JD) -> Error { + auto G = orc::DynamicLibrarySearchGenerator::GetForCurrentProcess( + DL->getGlobalPrefix()); + if (!G) + return G.takeError(); + JD.addGenerator(std::move(*G)); + return Error::success(); + }; + } + return Error::success(); } @@ -747,6 +807,54 @@ LLJIT::~LLJIT() { ES->reportError(std::move(Err)); } +JITDylibSP LLJIT::getProcessSymbolsJITDylib() { return ProcessSymbols; } + +JITDylibSP LLJIT::getPlatformJITDylib() { return Platform; } + +Expected<JITDylib &> LLJIT::createJITDylib(std::string Name) { + auto JD = ES->createJITDylib(std::move(Name)); + if (!JD) + return JD.takeError(); + + JD->addToLinkOrder(DefaultLinks); + return JD; +} + +Expected<JITDylib &> LLJIT::loadPlatformDynamicLibrary(const char *Path) { + auto G = EPCDynamicLibrarySearchGenerator::Load(*ES, Path); + if (!G) + return G.takeError(); + + if (auto *ExistingJD = ES->getJITDylibByName(Path)) + return *ExistingJD; + + auto &JD = ES->createBareJITDylib(Path); + JD.addGenerator(std::move(*G)); + return JD; +} + +Error LLJIT::linkStaticLibraryInto(JITDylib &JD, + std::unique_ptr<MemoryBuffer> LibBuffer) { + auto G = StaticLibraryDefinitionGenerator::Create(*ObjLinkingLayer, + std::move(LibBuffer)); + if (!G) + return G.takeError(); + + JD.addGenerator(std::move(*G)); + + return Error::success(); +} + +Error LLJIT::linkStaticLibraryInto(JITDylib &JD, const char *Path) { + auto G = StaticLibraryDefinitionGenerator::Load(*ObjLinkingLayer, Path); + if (!G) + return G.takeError(); + + JD.addGenerator(std::move(*G)); + + return Error::success(); +} + Error LLJIT::addIRModule(ResourceTrackerSP RT, ThreadSafeModule TSM) { assert(TSM && "Can not add null module"); @@ -777,7 +885,7 @@ Expected<ExecutorAddr> LLJIT::lookupLinkerMangled(JITDylib &JD, if (auto Sym = ES->lookup( makeJITDylibSearchOrder(&JD, JITDylibLookupFlags::MatchAllSymbols), Name)) - return ExecutorAddr(Sym->getAddress()); + return Sym->getAddress(); else return Sym.takeError(); } @@ -832,7 +940,7 @@ LLJIT::createCompileFunction(LLJITBuilderState &S, } LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) - : DL(""), TT(S.JTMB->getTargetTriple()) { + : DL(std::move(*S.DL)), TT(S.JTMB->getTargetTriple()) { ErrorAsOutParameter _(&Err); @@ -851,22 +959,6 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) } } - if (auto MainOrErr = this->ES->createJITDylib("main")) - Main = &*MainOrErr; - else { - Err = MainOrErr.takeError(); - return; - } - - if (S.DL) - DL = std::move(*S.DL); - else if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) - DL = std::move(*DLOrErr); - else { - Err = DLOrErr.takeError(); - return; - } - auto ObjLayer = createObjectLinkingLayer(S, *ES); if (!ObjLayer) { Err = ObjLayer.takeError(); @@ -905,10 +997,77 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) }); } - if (S.SetUpPlatform) - Err = S.SetUpPlatform(*this); - else - setUpGenericLLVMIRPlatform(*this); + if (S.SetupProcessSymbolsJITDylib) { + ProcessSymbols = &ES->createBareJITDylib("<Process Symbols>"); + if (auto Err2 = S.SetupProcessSymbolsJITDylib(*ProcessSymbols)) { + Err = std::move(Err2); + return; + } + } + + if (S.EnableDebuggerSupport) { + if (auto *OLL = dyn_cast<ObjectLinkingLayer>(ObjLinkingLayer.get())) { + switch (TT.getObjectFormat()) { + case Triple::ELF: { + auto Registrar = createJITLoaderGDBRegistrar(*ES); + if (!Registrar) { + Err = Registrar.takeError(); + return; + } + OLL->addPlugin(std::make_unique<DebugObjectManagerPlugin>( + *ES, std::move(*Registrar), true, true)); + break; + } + case Triple::MachO: { + assert(ProcessSymbols && "ProcessSymbols JD should be available when " + "EnableDebuggerSupport is set"); + auto DS = + GDBJITDebugInfoRegistrationPlugin::Create(*ES, *ProcessSymbols, TT); + if (!DS) { + Err = DS.takeError(); + return; + } + OLL->addPlugin(std::move(*DS)); + break; + } + default: + LLVM_DEBUG({ + dbgs() << "Cannot enable LLJIT debugger support: " + << Triple::getObjectFormatTypeName(TT.getObjectFormat()) + << " not supported.\n"; + }); + } + } else { + LLVM_DEBUG({ + dbgs() << "Cannot enable LLJIT debugger support: " + " debugger support is only available when using JITLink.\n"; + }); + } + } + + if (!S.SetUpPlatform) + S.SetUpPlatform = setUpGenericLLVMIRPlatform; + + if (auto PlatformJDOrErr = S.SetUpPlatform(*this)) { + Platform = PlatformJDOrErr->get(); + if (Platform) + DefaultLinks.push_back( + {Platform, JITDylibLookupFlags::MatchExportedSymbolsOnly}); + } else { + Err = PlatformJDOrErr.takeError(); + return; + } + + if (S.LinkProcessSymbolsByDefault) + DefaultLinks.push_back( + {ProcessSymbols, JITDylibLookupFlags::MatchExportedSymbolsOnly}); + + if (auto MainOrErr = createJITDylib("main")) + Main = &*MainOrErr; + else { + Err = MainOrErr.takeError(); + return; + } } std::string LLJIT::mangle(StringRef UnmangledName) const { @@ -934,24 +1093,136 @@ Error LLJIT::applyDataLayout(Module &M) { return Error::success(); } -Error setUpOrcPlatform(LLJIT& J) { - LLVM_DEBUG( - { dbgs() << "Setting up orc platform support for LLJIT\n"; }); - J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J)); +Error setUpOrcPlatformManually(LLJIT &J) { + LLVM_DEBUG({ dbgs() << "Setting up orc platform support for LLJIT\n"; }); + J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J)); + return Error::success(); +} + +class LoadAndLinkDynLibrary { +public: + LoadAndLinkDynLibrary(LLJIT &J) : J(J) {} + Error operator()(JITDylib &JD, StringRef DLLName) { + if (!DLLName.ends_with_insensitive(".dll")) + return make_error<StringError>("DLLName not ending with .dll", + inconvertibleErrorCode()); + auto DLLNameStr = DLLName.str(); // Guarantees null-termination. + auto DLLJD = J.loadPlatformDynamicLibrary(DLLNameStr.c_str()); + if (!DLLJD) + return DLLJD.takeError(); + JD.addToLinkOrder(*DLLJD); return Error::success(); + } + +private: + LLJIT &J; +}; + +Expected<JITDylibSP> ExecutorNativePlatform::operator()(LLJIT &J) { + auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib(); + if (!ProcessSymbolsJD) + return make_error<StringError>( + "Native platforms require a process symbols JITDylib", + inconvertibleErrorCode()); + + const Triple &TT = J.getTargetTriple(); + ObjectLinkingLayer *ObjLinkingLayer = + dyn_cast<ObjectLinkingLayer>(&J.getObjLinkingLayer()); + + if (!ObjLinkingLayer) + return make_error<StringError>( + "SetUpTargetPlatform requires ObjectLinkingLayer", + inconvertibleErrorCode()); + + std::unique_ptr<MemoryBuffer> RuntimeArchiveBuffer; + if (OrcRuntime.index() == 0) { + auto A = errorOrToExpected(MemoryBuffer::getFile(std::get<0>(OrcRuntime))); + if (!A) + return A.takeError(); + RuntimeArchiveBuffer = std::move(*A); + } else + RuntimeArchiveBuffer = std::move(std::get<1>(OrcRuntime)); + + auto &ES = J.getExecutionSession(); + auto &PlatformJD = ES.createBareJITDylib("<Platform>"); + PlatformJD.addToLinkOrder(*ProcessSymbolsJD); + + J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J)); + + switch (TT.getObjectFormat()) { + case Triple::COFF: { + const char *VCRuntimePath = nullptr; + bool StaticVCRuntime = false; + if (VCRuntime) { + VCRuntimePath = VCRuntime->first.c_str(); + StaticVCRuntime = VCRuntime->second; + } + if (auto P = COFFPlatform::Create( + ES, *ObjLinkingLayer, PlatformJD, std::move(RuntimeArchiveBuffer), + LoadAndLinkDynLibrary(J), StaticVCRuntime, VCRuntimePath)) + J.getExecutionSession().setPlatform(std::move(*P)); + else + return P.takeError(); + break; + } + case Triple::ELF: { + auto G = StaticLibraryDefinitionGenerator::Create( + *ObjLinkingLayer, std::move(RuntimeArchiveBuffer)); + if (!G) + return G.takeError(); + + if (auto P = ELFNixPlatform::Create(ES, *ObjLinkingLayer, PlatformJD, + std::move(*G))) + J.getExecutionSession().setPlatform(std::move(*P)); + else + return P.takeError(); + break; + } + case Triple::MachO: { + auto G = StaticLibraryDefinitionGenerator::Create( + *ObjLinkingLayer, std::move(RuntimeArchiveBuffer)); + if (!G) + return G.takeError(); + + if (auto P = MachOPlatform::Create(ES, *ObjLinkingLayer, PlatformJD, + std::move(*G))) + ES.setPlatform(std::move(*P)); + else + return P.takeError(); + break; + } + default: + return make_error<StringError>("Unsupported object format in triple " + + TT.str(), + inconvertibleErrorCode()); + } + + return &PlatformJD; } -void setUpGenericLLVMIRPlatform(LLJIT &J) { +Expected<JITDylibSP> setUpGenericLLVMIRPlatform(LLJIT &J) { LLVM_DEBUG( { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; }); - J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J)); + auto ProcessSymbolsJD = J.getProcessSymbolsJITDylib(); + if (!ProcessSymbolsJD) + return make_error<StringError>( + "Native platforms require a process symbols JITDylib", + inconvertibleErrorCode()); + + auto &PlatformJD = J.getExecutionSession().createBareJITDylib("<Platform>"); + PlatformJD.addToLinkOrder(*ProcessSymbolsJD); + + J.setPlatformSupport( + std::make_unique<GenericLLVMIRPlatformSupport>(J, PlatformJD)); + + return &PlatformJD; } -Error setUpInactivePlatform(LLJIT &J) { +Expected<JITDylibSP> setUpInactivePlatform(LLJIT &J) { LLVM_DEBUG( { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; }); J.setPlatformSupport(std::make_unique<InactivePlatformSupport>()); - return Error::success(); + return nullptr; } Error LLLazyJITBuilderState::prepareForConstruction() { @@ -984,7 +1255,7 @@ LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { LCTMgr = std::move(S.LCTMgr); else { if (auto LCTMgrOrErr = createLocalLazyCallThroughManager( - S.TT, *ES, S.LazyCompileFailureAddr.getValue())) + S.TT, *ES, S.LazyCompileFailureAddr)) LCTMgr = std::move(*LCTMgrOrErr); else { Err = LCTMgrOrErr.takeError(); diff --git a/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/llvm/lib/ExecutionEngine/Orc/Layer.cpp index 95380d912392..3368d3276cb3 100644 --- a/llvm/lib/ExecutionEngine/Orc/Layer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -125,6 +125,10 @@ void IRMaterializationUnit::discard(const JITDylib &JD, assert(!I->second->isDeclaration() && "Discard should only apply to definitions"); I->second->setLinkage(GlobalValue::AvailableExternallyLinkage); + // According to the IR verifier, "Declaration[s] may not be in a Comdat!" + // Remove it, if this is a GlobalObject. + if (auto *GO = dyn_cast<GlobalObject>(I->second)) + GO->setComdat(nullptr); SymbolToDefinition.erase(I); } diff --git a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp index c0a740d42dbd..d95a642934f1 100644 --- a/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -8,19 +8,20 @@ #include "llvm/ExecutionEngine/Orc/LazyReexports.h" -#include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" +#include "llvm/TargetParser/Triple.h" #define DEBUG_TYPE "orc" namespace llvm { namespace orc { -LazyCallThroughManager::LazyCallThroughManager( - ExecutionSession &ES, JITTargetAddress ErrorHandlerAddr, TrampolinePool *TP) +LazyCallThroughManager::LazyCallThroughManager(ExecutionSession &ES, + ExecutorAddr ErrorHandlerAddr, + TrampolinePool *TP) : ES(ES), ErrorHandlerAddr(ErrorHandlerAddr), TP(TP) {} -Expected<JITTargetAddress> LazyCallThroughManager::getCallThroughTrampoline( +Expected<ExecutorAddr> LazyCallThroughManager::getCallThroughTrampoline( JITDylib &SourceJD, SymbolStringPtr SymbolName, NotifyResolvedFunction NotifyResolved) { assert(TP && "TrampolinePool not set"); @@ -36,24 +37,24 @@ Expected<JITTargetAddress> LazyCallThroughManager::getCallThroughTrampoline( return *Trampoline; } -JITTargetAddress LazyCallThroughManager::reportCallThroughError(Error Err) { +ExecutorAddr LazyCallThroughManager::reportCallThroughError(Error Err) { ES.reportError(std::move(Err)); return ErrorHandlerAddr; } Expected<LazyCallThroughManager::ReexportsEntry> -LazyCallThroughManager::findReexport(JITTargetAddress TrampolineAddr) { +LazyCallThroughManager::findReexport(ExecutorAddr TrampolineAddr) { std::lock_guard<std::mutex> Lock(LCTMMutex); auto I = Reexports.find(TrampolineAddr); if (I == Reexports.end()) return createStringError(inconvertibleErrorCode(), - "Missing reexport for trampoline address %p", - TrampolineAddr); + "Missing reexport for trampoline address %p" + + formatv("{0:x}", TrampolineAddr)); return I->second; } -Error LazyCallThroughManager::notifyResolved(JITTargetAddress TrampolineAddr, - JITTargetAddress ResolvedAddr) { +Error LazyCallThroughManager::notifyResolved(ExecutorAddr TrampolineAddr, + ExecutorAddr ResolvedAddr) { NotifyResolvedFunction NotifyResolved; { std::lock_guard<std::mutex> Lock(LCTMMutex); @@ -68,7 +69,7 @@ Error LazyCallThroughManager::notifyResolved(JITTargetAddress TrampolineAddr, } void LazyCallThroughManager::resolveTrampolineLandingAddress( - JITTargetAddress TrampolineAddr, + ExecutorAddr TrampolineAddr, NotifyLandingResolvedFunction NotifyLandingResolved) { auto Entry = findReexport(TrampolineAddr); @@ -84,7 +85,7 @@ void LazyCallThroughManager::resolveTrampolineLandingAddress( if (Result) { assert(Result->size() == 1 && "Unexpected result size"); assert(Result->count(SymbolName) && "Unexpected result value"); - JITTargetAddress LandingAddr = (*Result)[SymbolName].getAddress(); + ExecutorAddr LandingAddr = (*Result)[SymbolName].getAddress(); if (auto Err = notifyResolved(TrampolineAddr, LandingAddr)) NotifyLandingResolved(reportCallThroughError(std::move(Err))); @@ -104,7 +105,7 @@ void LazyCallThroughManager::resolveTrampolineLandingAddress( Expected<std::unique_ptr<LazyCallThroughManager>> createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, - JITTargetAddress ErrorHandlerAddr) { + ExecutorAddr ErrorHandlerAddr) { switch (T.getArch()) { default: return make_error<StringError>( @@ -187,7 +188,7 @@ void LazyReexportsMaterializationUnit::materialize( auto CallThroughTrampoline = LCTManager.getCallThroughTrampoline( SourceJD, Alias.second.Aliasee, [&ISManager = this->ISManager, - StubSym = Alias.first](JITTargetAddress ResolvedAddr) -> Error { + StubSym = Alias.first](ExecutorAddr ResolvedAddr) -> Error { return ISManager.updatePointer(*StubSym, ResolvedAddr); }); diff --git a/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp b/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp index 59c63d38458b..75075c5c2a22 100644 --- a/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp +++ b/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp @@ -31,8 +31,8 @@ void lookupAndRecordAddrs( return OnRec(Result.takeError()); for (auto &KV : Pairs) { auto I = Result->find(KV.first); - KV.second->setValue((I != Result->end()) ? I->second.getAddress() - : 0); + *KV.second = + I != Result->end() ? I->second.getAddress() : orc::ExecutorAddr(); } OnRec(Error::success()); }, diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index 914a1b5afc71..a3a766d602c1 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -9,10 +9,13 @@ #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/MachO.h" +#include "llvm/ExecutionEngine/JITLink/aarch64.h" #include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" #include <optional> @@ -62,8 +65,7 @@ std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP, std::string Name) { unsigned PointerSize; support::endianness Endianness; - const auto &TT = - MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + const auto &TT = MOP.getExecutionSession().getTargetTriple(); switch (TT.getArch()) { case Triple::aarch64: @@ -147,8 +149,8 @@ private: if (G.getEndianness() != support::endian::system_endianness()) MachO::swapStruct(Hdr); - auto HeaderContent = G.allocateString( - StringRef(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); + auto HeaderContent = G.allocateContent( + ArrayRef<char>(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, 0); @@ -246,24 +248,23 @@ private: ExecutorAddr MachOHeaderAddr; }; -StringRef DataCommonSectionName = "__DATA,__common"; -StringRef DataDataSectionName = "__DATA,__data"; -StringRef EHFrameSectionName = "__TEXT,__eh_frame"; -StringRef CompactUnwindInfoSectionName = "__TEXT,__unwind_info"; -StringRef ModInitFuncSectionName = "__DATA,__mod_init_func"; -StringRef ObjCClassListSectionName = "__DATA,__objc_classlist"; -StringRef ObjCImageInfoSectionName = "__DATA,__objc_image_info"; -StringRef ObjCSelRefsSectionName = "__DATA,__objc_selrefs"; -StringRef Swift5ProtoSectionName = "__TEXT,__swift5_proto"; -StringRef Swift5ProtosSectionName = "__TEXT,__swift5_protos"; -StringRef Swift5TypesSectionName = "__TEXT,__swift5_types"; -StringRef ThreadBSSSectionName = "__DATA,__thread_bss"; -StringRef ThreadDataSectionName = "__DATA,__thread_data"; -StringRef ThreadVarsSectionName = "__DATA,__thread_vars"; +static StringRef ObjCRuntimeObjectSectionsData[] = { + MachOObjCCatListSectionName, MachOObjCClassListSectionName, + MachOObjCClassRefsSectionName, MachOObjCConstSectionName, + MachOObjCDataSectionName, MachOObjCSelRefsSectionName}; -StringRef InitSectionNames[] = { - ModInitFuncSectionName, ObjCSelRefsSectionName, ObjCClassListSectionName, - Swift5ProtosSectionName, Swift5ProtoSectionName, Swift5TypesSectionName}; +static StringRef ObjCRuntimeObjectSectionsText[] = { + MachOObjCClassNameSectionName, MachOObjCMethNameSectionName, + MachOObjCMethTypeSectionName, MachOSwift5TypesSectionName, + MachOSwift5TypeRefSectionName, MachOSwift5FieldMetadataSectionName, + MachOSwift5EntrySectionName, MachOSwift5ProtoSectionName, + MachOSwift5ProtosSectionName}; + +static StringRef ObjCRuntimeObjectSectionName = + "__llvm_jitlink_ObjCRuntimeRegistrationObject"; + +static StringRef ObjCImageInfoSymbolName = + "__llvm_jitlink_macho_objc_imageinfo"; } // end anonymous namespace @@ -272,17 +273,18 @@ namespace orc { Expected<std::unique_ptr<MachOPlatform>> MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, - JITDylib &PlatformJD, const char *OrcRuntimePath, + JITDylib &PlatformJD, + std::unique_ptr<DefinitionGenerator> OrcRuntime, std::optional<SymbolAliasMap> RuntimeAliases) { - auto &EPC = ES.getExecutorProcessControl(); - // If the target is not supported then bail out immediately. - if (!supportedTarget(EPC.getTargetTriple())) + if (!supportedTarget(ES.getTargetTriple())) return make_error<StringError>("Unsupported MachOPlatform triple: " + - EPC.getTargetTriple().str(), + ES.getTargetTriple().str(), inconvertibleErrorCode()); + auto &EPC = ES.getExecutorProcessControl(); + // Create default aliases if the caller didn't supply any. if (!RuntimeAliases) RuntimeAliases = standardPlatformAliases(ES); @@ -292,31 +294,40 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, return std::move(Err); // Add JIT-dispatch function support symbols. - if (auto Err = PlatformJD.define(absoluteSymbols( - {{ES.intern("___orc_rt_jit_dispatch"), - {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), - JITSymbolFlags::Exported}}, - {ES.intern("___orc_rt_jit_dispatch_ctx"), - {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), - JITSymbolFlags::Exported}}}))) + if (auto Err = PlatformJD.define( + absoluteSymbols({{ES.intern("___orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunction, + JITSymbolFlags::Exported}}, + {ES.intern("___orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContext, + JITSymbolFlags::Exported}}}))) return std::move(Err); - // Create a generator for the ORC runtime archive. - auto OrcRuntimeArchiveGenerator = StaticLibraryDefinitionGenerator::Load( - ObjLinkingLayer, OrcRuntimePath, EPC.getTargetTriple()); - if (!OrcRuntimeArchiveGenerator) - return OrcRuntimeArchiveGenerator.takeError(); - // Create the instance. Error Err = Error::success(); - auto P = std::unique_ptr<MachOPlatform>( - new MachOPlatform(ES, ObjLinkingLayer, PlatformJD, - std::move(*OrcRuntimeArchiveGenerator), Err)); + auto P = std::unique_ptr<MachOPlatform>(new MachOPlatform( + ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err)); if (Err) return std::move(Err); return std::move(P); } +Expected<std::unique_ptr<MachOPlatform>> +MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + std::optional<SymbolAliasMap> RuntimeAliases) { + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = + StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath); + if (!OrcRuntimeArchiveGenerator) + return OrcRuntimeArchiveGenerator.takeError(); + + return Create(ES, ObjLinkingLayer, PlatformJD, + std::move(*OrcRuntimeArchiveGenerator), + std::move(RuntimeAliases)); +} + Error MachOPlatform::setupJITDylib(JITDylib &JD) { if (auto Err = JD.define(std::make_unique<MachOHeaderMaterializationUnit>( *this, MachOHeaderStartSymbol))) @@ -398,15 +409,6 @@ MachOPlatform::standardRuntimeUtilityAliases() { StandardRuntimeUtilityAliases); } -bool MachOPlatform::isInitializerSection(StringRef SegName, - StringRef SectName) { - for (auto &Name : InitSectionNames) { - if (Name.startswith(SegName) && Name.substr(7) == SectName) - return true; - } - return false; -} - bool MachOPlatform::supportedTarget(const Triple &TT) { switch (TT.getArch()) { case Triple::aarch64: @@ -654,10 +656,9 @@ void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, }); if (!JD) { - SendResult( - make_error<StringError>("No JITDylib with header addr " + - formatv("{0:x}", JDHeaderAddr.getValue()), - inconvertibleErrorCode())); + SendResult(make_error<StringError>("No JITDylib with header addr " + + formatv("{0:x}", JDHeaderAddr), + inconvertibleErrorCode())); return; } @@ -667,8 +668,7 @@ void MachOPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, ExecutorAddr Handle, StringRef SymbolName) { LLVM_DEBUG({ - dbgs() << "MachOPlatform::rt_lookupSymbol(\"" - << formatv("{0:x}", Handle.getValue()) << "\")\n"; + dbgs() << "MachOPlatform::rt_lookupSymbol(\"" << Handle << "\")\n"; }); JITDylib *JD = nullptr; @@ -681,12 +681,9 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, } if (!JD) { - LLVM_DEBUG({ - dbgs() << " No JITDylib for handle " - << formatv("{0:x}", Handle.getValue()) << "\n"; - }); + LLVM_DEBUG(dbgs() << " No JITDylib for handle " << Handle << "\n"); SendResult(make_error<StringError>("No JITDylib associated with handle " + - formatv("{0:x}", Handle.getValue()), + formatv("{0:x}", Handle), inconvertibleErrorCode())); return; } @@ -699,7 +696,7 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, void operator()(Expected<SymbolMap> Result) { if (Result) { assert(Result->size() == 1 && "Unexpected result map count"); - SendResult(ExecutorAddr(Result->begin()->second.getAddress())); + SendResult(Result->begin()->second.getAddress()); } else { SendResult(Result.takeError()); } @@ -766,10 +763,14 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( // then add passes to preserve, process and register the init // sections/symbols. Config.PrePrunePasses.push_back([this, &MR](LinkGraph &G) { - if (auto Err = preserveInitSections(G, MR)) + if (auto Err = preserveImportantSections(G, MR)) return Err; return processObjCImageInfo(G, MR); }); + Config.PostPrunePasses.push_back( + [this](LinkGraph &G) { return createObjCRuntimeObject(G); }); + Config.PostAllocationPasses.push_back( + [this, &MR](LinkGraph &G) { return populateObjCRuntimeObject(G, MR); }); } // Insert TLV lowering at the start of the PostPrunePasses, since we want @@ -829,7 +830,10 @@ Error MachOPlatform::MachOPlatformPlugin:: &MP.RegisterObjectPlatformSections.Addr}, {*MP.DeregisterObjectPlatformSections.Name, &MP.DeregisterObjectPlatformSections.Addr}, - {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}}; + {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}, + {*MP.RegisterObjCRuntimeObject.Name, &MP.RegisterObjCRuntimeObject.Addr}, + {*MP.DeregisterObjCRuntimeObject.Name, + &MP.DeregisterObjCRuntimeObject.Addr}}; bool RegisterMachOHeader = false; @@ -898,11 +902,40 @@ Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol( return Error::success(); } -Error MachOPlatform::MachOPlatformPlugin::preserveInitSections( +Error MachOPlatform::MachOPlatformPlugin::preserveImportantSections( jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + // __objc_imageinfo is "important": we want to preserve it and record its + // address in the first graph that it appears in, then verify and discard it + // in all subsequent graphs. In this pass we preserve unconditionally -- we'll + // manually throw it away in the processObjCImageInfo pass. + if (auto *ObjCImageInfoSec = + G.findSectionByName(MachOObjCImageInfoSectionName)) { + if (ObjCImageInfoSec->blocks_size() != 1) + return make_error<StringError>( + "In " + G.getName() + + "__DATA,__objc_imageinfo contains multiple blocks", + inconvertibleErrorCode()); + G.addAnonymousSymbol(**ObjCImageInfoSec->blocks().begin(), 0, 0, false, + true); + for (auto *B : ObjCImageInfoSec->blocks()) + if (!B->edges_empty()) + return make_error<StringError>("In " + G.getName() + ", " + + MachOObjCImageInfoSectionName + + " contains references to symbols", + inconvertibleErrorCode()); + } + + // Init sections are important: We need to preserve them and so that their + // addresses can be captured and reported to the ORC runtime in + // registerObjectPlatformSections. JITLinkSymbolSet InitSectionSymbols; - for (auto &InitSectionName : InitSectionNames) { + for (auto &InitSectionName : MachOInitSectionNames) { + // Skip ObjCImageInfo -- this shouldn't have any dependencies, and we may + // remove it later. + if (InitSectionName == MachOObjCImageInfoSectionName) + continue; + // Skip non-init sections. auto *InitSection = G.findSectionByName(InitSectionName); if (!InitSection) @@ -944,7 +977,7 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( // OR // (2) We already have a recorded __objc_imageinfo for this JITDylib, // in which case we just verify it. - auto *ObjCImageInfo = G.findSectionByName(ObjCImageInfoSectionName); + auto *ObjCImageInfo = G.findSectionByName(MachOObjCImageInfoSectionName); if (!ObjCImageInfo) return Error::success(); @@ -952,14 +985,14 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( // Check that the section is not empty if present. if (ObjCImageInfoBlocks.empty()) - return make_error<StringError>("Empty " + ObjCImageInfoSectionName + + return make_error<StringError>("Empty " + MachOObjCImageInfoSectionName + " section in " + G.getName(), inconvertibleErrorCode()); // Check that there's only one block in the section. if (std::next(ObjCImageInfoBlocks.begin()) != ObjCImageInfoBlocks.end()) return make_error<StringError>("Multiple blocks in " + - ObjCImageInfoSectionName + + MachOObjCImageInfoSectionName + " section in " + G.getName(), inconvertibleErrorCode()); @@ -971,7 +1004,7 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( for (auto &E : B->edges()) if (E.getTarget().isDefined() && &E.getTarget().getBlock().getSection() == ObjCImageInfo) - return make_error<StringError>(ObjCImageInfoSectionName + + return make_error<StringError>(MachOObjCImageInfoSectionName + " is referenced within file " + G.getName(), inconvertibleErrorCode()); @@ -990,12 +1023,12 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( if (ObjCImageInfoItr != ObjCImageInfos.end()) { // We've already registered an __objc_imageinfo section. Verify the // content of this new section matches, then delete it. - if (ObjCImageInfoItr->second.first != Version) + if (ObjCImageInfoItr->second.Version != Version) return make_error<StringError>( "ObjC version in " + G.getName() + " does not match first registered version", inconvertibleErrorCode()); - if (ObjCImageInfoItr->second.second != Flags) + if (ObjCImageInfoItr->second.Flags != Flags) return make_error<StringError>("ObjC flags in " + G.getName() + " do not match first registered flags", inconvertibleErrorCode()); @@ -1007,7 +1040,14 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( } else { // We haven't registered an __objc_imageinfo section yet. Register and // move on. The section should already be marked no-dead-strip. - ObjCImageInfos[&MR.getTargetJITDylib()] = std::make_pair(Version, Flags); + G.addDefinedSymbol(ObjCImageInfoBlock, 0, ObjCImageInfoSymbolName, + ObjCImageInfoBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Hidden, false, true); + if (auto Err = MR.defineMaterializing( + {{MR.getExecutionSession().intern(ObjCImageInfoSymbolName), + JITSymbolFlags()}})) + return Err; + ObjCImageInfos[&MR.getTargetJITDylib()] = {Version, Flags}; } return Error::success(); @@ -1024,7 +1064,7 @@ Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( } // Store key in __thread_vars struct fields. - if (auto *ThreadDataSec = G.findSectionByName(ThreadVarsSectionName)) { + if (auto *ThreadDataSec = G.findSectionByName(MachOThreadVarsSectionName)) { std::optional<uint64_t> Key; { std::lock_guard<std::mutex> Lock(MP.PlatformMutex); @@ -1098,10 +1138,11 @@ MachOPlatform::MachOPlatformPlugin::findUnwindSectionInfo( } }; - if (Section *EHFrameSec = G.findSectionByName(EHFrameSectionName)) + if (Section *EHFrameSec = G.findSectionByName(MachOEHFrameSectionName)) ScanUnwindInfoSection(*EHFrameSec, US.DwarfSection); - if (Section *CUInfoSec = G.findSectionByName(CompactUnwindInfoSectionName)) + if (Section *CUInfoSec = + G.findSectionByName(MachOCompactUnwindInfoSectionName)) ScanUnwindInfoSection(*CUInfoSec, US.CompactUnwindSection); // If we didn't find any pointed-to code-blocks then there's no need to @@ -1150,10 +1191,10 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( // Get a pointer to the thread data section if there is one. It will be used // below. jitlink::Section *ThreadDataSection = - G.findSectionByName(ThreadDataSectionName); + G.findSectionByName(MachOThreadDataSectionName); // Handle thread BSS section if there is one. - if (auto *ThreadBSSSection = G.findSectionByName(ThreadBSSSectionName)) { + if (auto *ThreadBSSSection = G.findSectionByName(MachOThreadBSSSectionName)) { // If there's already a thread data section in this graph then merge the // thread BSS section content into it, otherwise just treat the thread // BSS section as the thread data section. @@ -1166,8 +1207,9 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( SmallVector<std::pair<StringRef, ExecutorAddrRange>, 8> MachOPlatformSecs; // Collect data sections to register. - StringRef DataSections[] = {DataDataSectionName, DataCommonSectionName, - EHFrameSectionName}; + StringRef DataSections[] = {MachODataDataSectionName, + MachODataCommonSectionName, + MachOEHFrameSectionName}; for (auto &SecName : DataSections) { if (auto *Sec = G.findSectionByName(SecName)) { jitlink::SectionRange R(*Sec); @@ -1181,17 +1223,13 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( if (ThreadDataSection) { jitlink::SectionRange R(*ThreadDataSection); if (!R.empty()) - MachOPlatformSecs.push_back({ThreadDataSectionName, R.getRange()}); + MachOPlatformSecs.push_back({MachOThreadDataSectionName, R.getRange()}); } // If any platform sections were found then add an allocation action to call // the registration function. - StringRef PlatformSections[] = { - ModInitFuncSectionName, ObjCClassListSectionName, - ObjCImageInfoSectionName, ObjCSelRefsSectionName, - Swift5ProtoSectionName, Swift5ProtosSectionName, - Swift5TypesSectionName, - }; + StringRef PlatformSections[] = {MachOModInitFuncSectionName, + ObjCRuntimeObjectSectionName}; for (auto &SecName : PlatformSections) { auto *Sec = G.findSectionByName(SecName); @@ -1252,5 +1290,207 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( return Error::success(); } +Error MachOPlatform::MachOPlatformPlugin::createObjCRuntimeObject( + jitlink::LinkGraph &G) { + + bool NeedTextSegment = false; + size_t NumRuntimeSections = 0; + + for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData) + if (G.findSectionByName(ObjCRuntimeSectionName)) + ++NumRuntimeSections; + + for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) { + if (G.findSectionByName(ObjCRuntimeSectionName)) { + ++NumRuntimeSections; + NeedTextSegment = true; + } + } + + // Early out for no runtime sections. + if (NumRuntimeSections == 0) + return Error::success(); + + // If there were any runtime sections then we need to add an __objc_imageinfo + // section. + ++NumRuntimeSections; + + size_t MachOSize = sizeof(MachO::mach_header_64) + + (NeedTextSegment + 1) * sizeof(MachO::segment_command_64) + + NumRuntimeSections * sizeof(MachO::section_64); + + auto &Sec = G.createSection(ObjCRuntimeObjectSectionName, + MemProt::Read | MemProt::Write); + G.createMutableContentBlock(Sec, MachOSize, ExecutorAddr(), 16, 0, true); + + return Error::success(); +} + +Error MachOPlatform::MachOPlatformPlugin::populateObjCRuntimeObject( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + + auto *ObjCRuntimeObjectSec = + G.findSectionByName(ObjCRuntimeObjectSectionName); + + if (!ObjCRuntimeObjectSec) + return Error::success(); + + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + case Triple::x86_64: + // Supported. + break; + default: + return make_error<StringError>("Unrecognized MachO arch in triple " + + G.getTargetTriple().str(), + inconvertibleErrorCode()); + } + + auto &SecBlock = **ObjCRuntimeObjectSec->blocks().begin(); + + struct SecDesc { + MachO::section_64 Sec; + unique_function<void(size_t RecordOffset)> AddFixups; + }; + + std::vector<SecDesc> TextSections, DataSections; + auto AddSection = [&](SecDesc &SD, jitlink::Section &GraphSec) { + jitlink::SectionRange SR(GraphSec); + StringRef FQName = GraphSec.getName(); + memset(&SD.Sec, 0, sizeof(MachO::section_64)); + memcpy(SD.Sec.sectname, FQName.drop_front(7).data(), FQName.size() - 7); + memcpy(SD.Sec.segname, FQName.data(), 6); + SD.Sec.addr = SR.getStart() - SecBlock.getAddress(); + SD.Sec.size = SR.getSize(); + SD.Sec.flags = MachO::S_REGULAR; + }; + + // Add the __objc_imageinfo section. + { + DataSections.push_back({}); + auto &SD = DataSections.back(); + memset(&SD.Sec, 0, sizeof(SD.Sec)); + memcpy(SD.Sec.sectname, "__objc_imageinfo", 16); + strcpy(SD.Sec.segname, "__DATA"); + SD.Sec.size = 8; + SD.AddFixups = [&](size_t RecordOffset) { + jitlink::Edge::Kind PointerEdge = jitlink::Edge::Invalid; + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + PointerEdge = jitlink::aarch64::Pointer64; + break; + case Triple::x86_64: + PointerEdge = jitlink::x86_64::Pointer64; + break; + default: + llvm_unreachable("Unsupported architecture"); + } + + // Look for an existing __objc_imageinfo symbol. + jitlink::Symbol *ObjCImageInfoSym = nullptr; + for (auto *Sym : G.external_symbols()) + if (Sym->getName() == ObjCImageInfoSymbolName) { + ObjCImageInfoSym = Sym; + break; + } + if (!ObjCImageInfoSym) + for (auto *Sym : G.absolute_symbols()) + if (Sym->getName() == ObjCImageInfoSymbolName) { + ObjCImageInfoSym = Sym; + break; + } + if (!ObjCImageInfoSym) + for (auto *Sym : G.defined_symbols()) + if (Sym->hasName() && Sym->getName() == ObjCImageInfoSymbolName) { + ObjCImageInfoSym = Sym; + break; + } + if (!ObjCImageInfoSym) + ObjCImageInfoSym = + &G.addExternalSymbol(ObjCImageInfoSymbolName, 8, false); + + SecBlock.addEdge(PointerEdge, + RecordOffset + ((char *)&SD.Sec.addr - (char *)&SD.Sec), + *ObjCImageInfoSym, -SecBlock.getAddress().getValue()); + }; + } + + for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsData) { + if (auto *GraphSec = G.findSectionByName(ObjCRuntimeSectionName)) { + DataSections.push_back({}); + AddSection(DataSections.back(), *GraphSec); + } + } + + for (auto ObjCRuntimeSectionName : ObjCRuntimeObjectSectionsText) { + if (auto *GraphSec = G.findSectionByName(ObjCRuntimeSectionName)) { + TextSections.push_back({}); + AddSection(TextSections.back(), *GraphSec); + } + } + + assert(ObjCRuntimeObjectSec->blocks_size() == 1 && + "Unexpected number of blocks in runtime sections object"); + + // Build the header struct up-front. This also gives us a chance to check + // that the triple is supported, which we'll assume below. + MachO::mach_header_64 Hdr; + Hdr.magic = MachO::MH_MAGIC_64; + switch (G.getTargetTriple().getArch()) { + case Triple::aarch64: + Hdr.cputype = MachO::CPU_TYPE_ARM64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; + break; + case Triple::x86_64: + Hdr.cputype = MachO::CPU_TYPE_X86_64; + Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; + break; + default: + llvm_unreachable("Unsupported architecture"); + } + + Hdr.filetype = MachO::MH_DYLIB; + Hdr.ncmds = 1 + !TextSections.empty(); + Hdr.sizeofcmds = + Hdr.ncmds * sizeof(MachO::segment_command_64) + + (TextSections.size() + DataSections.size()) * sizeof(MachO::section_64); + Hdr.flags = 0; + Hdr.reserved = 0; + + auto SecContent = SecBlock.getAlreadyMutableContent(); + char *P = SecContent.data(); + auto WriteMachOStruct = [&](auto S) { + if (G.getEndianness() != support::endian::system_endianness()) + MachO::swapStruct(S); + memcpy(P, &S, sizeof(S)); + P += sizeof(S); + }; + + auto WriteSegment = [&](StringRef Name, std::vector<SecDesc> &Secs) { + MachO::segment_command_64 SegLC; + memset(&SegLC, 0, sizeof(SegLC)); + memcpy(SegLC.segname, Name.data(), Name.size()); + SegLC.cmd = MachO::LC_SEGMENT_64; + SegLC.cmdsize = sizeof(MachO::segment_command_64) + + Secs.size() * sizeof(MachO::section_64); + SegLC.nsects = Secs.size(); + WriteMachOStruct(SegLC); + for (auto &SD : Secs) { + if (SD.AddFixups) + SD.AddFixups(P - SecContent.data()); + WriteMachOStruct(SD.Sec); + } + }; + + WriteMachOStruct(Hdr); + if (!TextSections.empty()) + WriteSegment("__TEXT", TextSections); + if (!DataSections.empty()) + WriteSegment("__DATA", DataSections); + + assert(P == SecContent.end() && "Underflow writing ObjC runtime object"); + return Error::success(); +} + } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp index b457c7297bed..ca4950077ffe 100644 --- a/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -322,7 +322,8 @@ void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); tpctypes::SharedMemorySegFinalizeRequest SegReq; - SegReq.AG = Segment.AG; + SegReq.RAG = {Segment.AG.getMemProt(), Segment.AG.getMemLifetimePolicy() == + MemLifetimePolicy::Finalize}; SegReq.Addr = AI.MappingBase + Segment.Offset; SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp index 0c3beba43a35..7c8fa63477d0 100644 --- a/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp @@ -10,6 +10,7 @@ #include "llvm/ExecutionEngine/Orc/COFFPlatform.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Object/COFF.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/MachO.h" @@ -85,7 +86,7 @@ getMachOObjectFileSymbolInfo(ExecutionSession &ES, } auto SegName = Obj.getSectionFinalSegmentName(Sec.getRawDataRefImpl()); auto SecName = cantFail(Obj.getSectionName(Sec.getRawDataRefImpl())); - if (MachOPlatform::isInitializerSection(SegName, SecName)) { + if (isMachOInitializerSection(SegName, SecName)) { addInitSymbol(I, ES, Obj.getFileName()); break; } @@ -138,7 +139,7 @@ getELFObjectFileSymbolInfo(ExecutionSession &ES, SymbolStringPtr InitSymbol; for (auto &Sec : Obj.sections()) { if (auto SecName = Sec.getName()) { - if (ELFNixPlatform::isInitializerSection(*SecName)) { + if (isELFInitializerSection(*SecName)) { addInitSymbol(I, ES, Obj.getFileName()); break; } @@ -219,7 +220,7 @@ getCOFFObjectFileSymbolInfo(ExecutionSession &ES, SymbolStringPtr InitSymbol; for (auto &Sec : Obj.sections()) { if (auto SecName = Sec.getName()) { - if (COFFPlatform::isInitializerSection(*SecName)) { + if (isCOFFInitializerSection(*SecName)) { addInitSymbol(I, ES, Obj.getFileName()); break; } @@ -287,22 +288,5 @@ getObjectFileInterface(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { return getGenericObjectFileSymbolInfo(ES, **Obj); } -bool hasInitializerSection(jitlink::LinkGraph &G) { - bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); - bool IsElf = G.getTargetTriple().isOSBinFormatELF(); - if (!IsMachO && !IsElf) - return false; - - for (auto &Sec : G.sections()) { - if (IsMachO && std::apply(MachOPlatform::isInitializerSection, - Sec.getName().split(","))) - return true; - if (IsElf && ELFNixPlatform::isInitializerSection(Sec.getName())) - return true; - } - - return false; -} - } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 2b11c472e812..a29f3d1c3aec 100644 --- a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -8,8 +8,10 @@ #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" +#include "llvm/ExecutionEngine/JITLink/aarch32.h" #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Support/MemoryBuffer.h" #include <string> #include <vector> @@ -22,6 +24,55 @@ using namespace llvm::orc; namespace { +bool hasInitializerSection(jitlink::LinkGraph &G) { + bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); + bool IsElf = G.getTargetTriple().isOSBinFormatELF(); + if (!IsMachO && !IsElf) + return false; + + for (auto &Sec : G.sections()) { + if (IsMachO && isMachOInitializerSection(Sec.getName())) + return true; + if (IsElf && isELFInitializerSection(Sec.getName())) + return true; + } + + return false; +} + +ExecutorAddr getJITSymbolPtrForSymbol(Symbol &Sym, const Triple &TT) { + switch (TT.getArch()) { + case Triple::arm: + case Triple::armeb: + case Triple::thumb: + case Triple::thumbeb: + if (Sym.hasTargetFlags(aarch32::ThumbSymbol)) { + // Set LSB to indicate thumb target + assert(Sym.isCallable() && "Only callable symbols can have thumb flag"); + assert((Sym.getAddress().getValue() & 0x01) == 0 && "LSB is clear"); + return Sym.getAddress() + 0x01; + } + return Sym.getAddress(); + default: + return Sym.getAddress(); + } +} + +JITSymbolFlags getJITSymbolFlagsForSymbol(Symbol &Sym) { + JITSymbolFlags Flags; + + if (Sym.getLinkage() == Linkage::Weak) + Flags |= JITSymbolFlags::Weak; + + if (Sym.getScope() == Scope::Default) + Flags |= JITSymbolFlags::Exported; + + if (Sym.isCallable()) + Flags |= JITSymbolFlags::Callable; + + return Flags; +} + class LinkGraphMaterializationUnit : public MaterializationUnit { public: static std::unique_ptr<LinkGraphMaterializationUnit> @@ -48,14 +99,8 @@ private: continue; assert(Sym->hasName() && "Anonymous non-local symbol?"); - JITSymbolFlags Flags; - if (Sym->getScope() == Scope::Default) - Flags |= JITSymbolFlags::Exported; - - if (Sym->isCallable()) - Flags |= JITSymbolFlags::Callable; - - LGI.SymbolFlags[ES.intern(Sym->getName())] = Flags; + LGI.SymbolFlags[ES.intern(Sym->getName())] = + getJITSymbolFlagsForSymbol(*Sym); } if (hasInitializerSection(G)) @@ -189,17 +234,9 @@ public: for (auto *Sym : G.defined_symbols()) if (Sym->hasName() && Sym->getScope() != Scope::Local) { auto InternedName = ES.intern(Sym->getName()); - JITSymbolFlags Flags; - - if (Sym->isCallable()) - Flags |= JITSymbolFlags::Callable; - if (Sym->getScope() == Scope::Default) - Flags |= JITSymbolFlags::Exported; - if (Sym->getLinkage() == Linkage::Weak) - Flags |= JITSymbolFlags::Weak; - - InternedResult[InternedName] = - JITEvaluatedSymbol(Sym->getAddress().getValue(), Flags); + auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); + auto Flags = getJITSymbolFlagsForSymbol(*Sym); + InternedResult[InternedName] = {Ptr, Flags}; if (AutoClaim && !MR->getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); @@ -210,15 +247,9 @@ public: for (auto *Sym : G.absolute_symbols()) if (Sym->hasName() && Sym->getScope() != Scope::Local) { auto InternedName = ES.intern(Sym->getName()); - JITSymbolFlags Flags; - if (Sym->isCallable()) - Flags |= JITSymbolFlags::Callable; - if (Sym->getScope() == Scope::Default) - Flags |= JITSymbolFlags::Exported; - if (Sym->getLinkage() == Linkage::Weak) - Flags |= JITSymbolFlags::Weak; - InternedResult[InternedName] = - JITEvaluatedSymbol(Sym->getAddress().getValue(), Flags); + auto Ptr = getJITSymbolPtrForSymbol(*Sym, G.getTargetTriple()); + auto Flags = getJITSymbolFlagsForSymbol(*Sym); + InternedResult[InternedName] = {Ptr, Flags}; if (AutoClaim && !MR->getSymbols().count(InternedName)) { assert(!ExtraSymbolsToClaim.count(InternedName) && "Duplicate symbol to claim?"); @@ -407,10 +438,8 @@ private: Sym->getScope() != Scope::Local) { auto Name = ES.intern(Sym->getName()); if (!MR->getSymbols().count(ES.intern(Sym->getName()))) { - JITSymbolFlags SF = JITSymbolFlags::Weak; - if (Sym->getScope() == Scope::Default) - SF |= JITSymbolFlags::Exported; - NewSymbolsToClaim[Name] = SF; + NewSymbolsToClaim[Name] = + getJITSymbolFlagsForSymbol(*Sym) | JITSymbolFlags::Weak; NameToSym.push_back(std::make_pair(std::move(Name), Sym)); } } diff --git a/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp index 48dd0df80415..6d568199378a 100644 --- a/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp +++ b/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -14,17 +14,17 @@ #define DEBUG_TYPE "orc" using namespace llvm; +using namespace llvm::orc; template <typename ORCABI> -bool stubAndPointerRangesOk(JITTargetAddress StubBlockAddr, - JITTargetAddress PointerBlockAddr, - unsigned NumStubs) { +static bool stubAndPointerRangesOk(ExecutorAddr StubBlockAddr, + ExecutorAddr PointerBlockAddr, + unsigned NumStubs) { constexpr unsigned MaxDisp = ORCABI::StubToPointerMaxDisplacement; - JITTargetAddress FirstStub = StubBlockAddr; - JITTargetAddress LastStub = FirstStub + ((NumStubs - 1) * ORCABI::StubSize); - JITTargetAddress FirstPointer = PointerBlockAddr; - JITTargetAddress LastPointer = - FirstPointer + ((NumStubs - 1) * ORCABI::StubSize); + ExecutorAddr FirstStub = StubBlockAddr; + ExecutorAddr LastStub = FirstStub + ((NumStubs - 1) * ORCABI::StubSize); + ExecutorAddr FirstPointer = PointerBlockAddr; + ExecutorAddr LastPointer = FirstPointer + ((NumStubs - 1) * ORCABI::StubSize); if (FirstStub < FirstPointer) { if (LastStub >= FirstPointer) @@ -44,9 +44,9 @@ namespace llvm { namespace orc { void OrcAArch64::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { const uint32_t ResolverCode[] = { // resolver_entry: @@ -135,8 +135,8 @@ void OrcAArch64::writeResolverCode(char *ResolverWorkingMem, } void OrcAArch64::writeTrampolines(char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, unsigned NumTrampolines) { unsigned OffsetToPtr = alignTo(NumTrampolines * TrampolineSize, 8); @@ -159,17 +159,17 @@ void OrcAArch64::writeTrampolines(char *TrampolineBlockWorkingMem, } void OrcAArch64::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs // stub1: - // ldr x0, ptr1 ; PC-rel load of ptr1 - // br x0 ; Jump to resolver + // ldr x16, ptr1 ; PC-rel load of ptr1 + // br x16 ; Jump to resolver // stub2: - // ldr x0, ptr2 ; PC-rel load of ptr2 - // br x0 ; Jump to resolver + // ldr x16, ptr2 ; PC-rel load of ptr2 + // br x16 ; Jump to resolver // // ... // @@ -188,17 +188,19 @@ void OrcAArch64::writeIndirectStubsBlock( "PointersBlock is out of range"); uint64_t PtrDisplacement = PointersBlockTargetAddress - StubsBlockTargetAddress; + assert((PtrDisplacement % 8 == 0) && + "Displacement to pointer is not a multiple of 8"); uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); - uint64_t PtrOffsetField = PtrDisplacement << 3; + uint64_t PtrOffsetField = ((PtrDisplacement >> 2) & 0x7ffff) << 5; for (unsigned I = 0; I < NumStubs; ++I) Stub[I] = 0xd61f020058000010 | PtrOffsetField; } -void OrcX86_64_Base::writeTrampolines( - char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, unsigned NumTrampolines) { +void OrcX86_64_Base::writeTrampolines(char *TrampolineBlockWorkingMem, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, + unsigned NumTrampolines) { unsigned OffsetToPtr = NumTrampolines * TrampolineSize; @@ -214,8 +216,8 @@ void OrcX86_64_Base::writeTrampolines( } void OrcX86_64_Base::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs @@ -250,9 +252,9 @@ void OrcX86_64_Base::writeIndirectStubsBlock( } void OrcX86_64_SysV::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { LLVM_DEBUG({ dbgs() << "Writing resolver code to " @@ -324,9 +326,9 @@ void OrcX86_64_SysV::writeResolverCode(char *ResolverWorkingMem, } void OrcX86_64_Win32::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { // resolverCode is similar to OrcX86_64 with differences specific to windows // x64 calling convention: arguments go into rcx, rdx and come in reverse @@ -402,12 +404,13 @@ void OrcX86_64_Win32::writeResolverCode(char *ResolverWorkingMem, } void OrcI386::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { - assert((ReentryFnAddr >> 32) == 0 && "ReentryFnAddr out of range"); - assert((ReentryCtxAddr >> 32) == 0 && "ReentryCtxAddr out of range"); + assert((ReentryFnAddr.getValue() >> 32) == 0 && "ReentryFnAddr out of range"); + assert((ReentryCtxAddr.getValue() >> 32) == 0 && + "ReentryCtxAddr out of range"); const uint8_t ResolverCode[] = { // resolver_entry: @@ -455,10 +458,10 @@ void OrcI386::writeResolverCode(char *ResolverWorkingMem, } void OrcI386::writeTrampolines(char *TrampolineWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, unsigned NumTrampolines) { - assert((ResolverAddr >> 32) == 0 && "ResolverAddr out of range"); + assert((ResolverAddr.getValue() >> 32) == 0 && "ResolverAddr out of range"); uint64_t CallRelImm = 0xF1C4C400000000e8; uint64_t ResolverRel = ResolverAddr - TrampolineBlockTargetAddress - 5; @@ -468,12 +471,13 @@ void OrcI386::writeTrampolines(char *TrampolineWorkingMem, Trampolines[I] = CallRelImm | (ResolverRel << 8); } -void OrcI386::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { - assert((StubsBlockTargetAddress >> 32) == 0 && +void OrcI386::writeIndirectStubsBlock(char *StubsBlockWorkingMem, + ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, + unsigned NumStubs) { + assert((StubsBlockTargetAddress.getValue() >> 32) == 0 && "StubsBlockTargetAddress is out of range"); - assert((PointersBlockTargetAddress >> 32) == 0 && + assert((PointersBlockTargetAddress.getValue() >> 32) == 0 && "PointersBlockTargetAddress is out of range"); // Stub format is: @@ -501,15 +505,15 @@ void OrcI386::writeIndirectStubsBlock( "PointersBlock is out of range"); uint64_t *Stub = reinterpret_cast<uint64_t *>(StubsBlockWorkingMem); - uint64_t PtrAddr = PointersBlockTargetAddress; + uint64_t PtrAddr = PointersBlockTargetAddress.getValue(); for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 4) Stub[I] = 0xF1C40000000025ff | (PtrAddr << 16); } void OrcMips32_Base::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr, + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr, bool isBigEndian) { const uint32_t ResolverCode[] = { @@ -596,32 +600,32 @@ void OrcMips32_Base::writeResolverCode(char *ResolverWorkingMem, memcpy(ResolverWorkingMem + Offsett, &MoveVxT9, sizeof(MoveVxT9)); uint32_t ReentryCtxLUi = - 0x3c040000 | (((ReentryCtxAddr + 0x8000) >> 16) & 0xFFFF); - uint32_t ReentryCtxADDiu = 0x24840000 | ((ReentryCtxAddr)&0xFFFF); + 0x3c040000 | (((ReentryCtxAddr.getValue() + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryCtxADDiu = 0x24840000 | (ReentryCtxAddr.getValue() & 0xFFFF); memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxLUi, sizeof(ReentryCtxLUi)); memcpy(ResolverWorkingMem + ReentryCtxAddrOffset + 4, &ReentryCtxADDiu, sizeof(ReentryCtxADDiu)); uint32_t ReentryFnLUi = - 0x3c190000 | (((ReentryFnAddr + 0x8000) >> 16) & 0xFFFF); - uint32_t ReentryFnADDiu = 0x27390000 | ((ReentryFnAddr)&0xFFFF); + 0x3c190000 | (((ReentryFnAddr.getValue() + 0x8000) >> 16) & 0xFFFF); + uint32_t ReentryFnADDiu = 0x27390000 | (ReentryFnAddr.getValue() & 0xFFFF); memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnLUi, sizeof(ReentryFnLUi)); memcpy(ResolverWorkingMem + ReentryFnAddrOffset + 4, &ReentryFnADDiu, sizeof(ReentryFnADDiu)); } -void OrcMips32_Base::writeTrampolines( - char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, unsigned NumTrampolines) { +void OrcMips32_Base::writeTrampolines(char *TrampolineBlockWorkingMem, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, + unsigned NumTrampolines) { - assert((ResolverAddr >> 32) == 0 && "ResolverAddr out of range"); + assert((ResolverAddr.getValue() >> 32) == 0 && "ResolverAddr out of range"); uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); - uint32_t RHiAddr = ((ResolverAddr + 0x8000) >> 16); + uint32_t RHiAddr = ((ResolverAddr.getValue() + 0x8000) >> 16); for (unsigned I = 0; I < NumTrampolines; ++I) { // move $t8,$ra @@ -631,16 +635,16 @@ void OrcMips32_Base::writeTrampolines( // nop Trampolines[5 * I + 0] = 0x03e0c025; Trampolines[5 * I + 1] = 0x3c190000 | (RHiAddr & 0xFFFF); - Trampolines[5 * I + 2] = 0x27390000 | (ResolverAddr & 0xFFFF); + Trampolines[5 * I + 2] = 0x27390000 | (ResolverAddr.getValue() & 0xFFFF); Trampolines[5 * I + 3] = 0x0320f809; Trampolines[5 * I + 4] = 0x00000000; } } void OrcMips32_Base::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { - assert((StubsBlockTargetAddress >> 32) == 0 && + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { + assert((StubsBlockTargetAddress.getValue() >> 32) == 0 && "InitialPtrVal is out of range"); // Stub format is: @@ -671,7 +675,7 @@ void OrcMips32_Base::writeIndirectStubsBlock( // Populate the stubs page stubs and mark it executable. uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlockWorkingMem); - uint64_t PtrAddr = PointersBlockTargetAddress; + uint64_t PtrAddr = PointersBlockTargetAddress.getValue(); for (unsigned I = 0; I < NumStubs; ++I) { uint32_t HiAddr = ((PtrAddr + 0x8000) >> 16); @@ -684,9 +688,9 @@ void OrcMips32_Base::writeIndirectStubsBlock( } void OrcMips64::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { const uint32_t ResolverCode[] = { //resolver_entry: @@ -775,14 +779,16 @@ void OrcMips64::writeResolverCode(char *ResolverWorkingMem, memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); uint32_t ReentryCtxLUi = - 0x3c040000 | (((ReentryCtxAddr + 0x800080008000) >> 48) & 0xFFFF); + 0x3c040000 | + (((ReentryCtxAddr.getValue() + 0x800080008000) >> 48) & 0xFFFF); uint32_t ReentryCtxDADDiu = - 0x64840000 | (((ReentryCtxAddr + 0x80008000) >> 32) & 0xFFFF); + 0x64840000 | (((ReentryCtxAddr.getValue() + 0x80008000) >> 32) & 0xFFFF); uint32_t ReentryCtxDSLL = 0x00042438; uint32_t ReentryCtxDADDiu2 = - 0x64840000 | ((((ReentryCtxAddr + 0x8000) >> 16) & 0xFFFF)); + 0x64840000 | ((((ReentryCtxAddr.getValue() + 0x8000) >> 16) & 0xFFFF)); uint32_t ReentryCtxDSLL2 = 0x00042438; - uint32_t ReentryCtxDADDiu3 = 0x64840000 | ((ReentryCtxAddr)&0xFFFF); + uint32_t ReentryCtxDADDiu3 = + 0x64840000 | (ReentryCtxAddr.getValue() & 0xFFFF); memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxLUi, sizeof(ReentryCtxLUi)); @@ -798,19 +804,20 @@ void OrcMips64::writeResolverCode(char *ResolverWorkingMem, sizeof(ReentryCtxDADDiu3)); uint32_t ReentryFnLUi = - 0x3c190000 | (((ReentryFnAddr + 0x800080008000) >> 48) & 0xFFFF); + 0x3c190000 | + (((ReentryFnAddr.getValue() + 0x800080008000) >> 48) & 0xFFFF); uint32_t ReentryFnDADDiu = - 0x67390000 | (((ReentryFnAddr + 0x80008000) >> 32) & 0xFFFF); + 0x67390000 | (((ReentryFnAddr.getValue() + 0x80008000) >> 32) & 0xFFFF); uint32_t ReentryFnDSLL = 0x0019cc38; uint32_t ReentryFnDADDiu2 = - 0x67390000 | (((ReentryFnAddr + 0x8000) >> 16) & 0xFFFF); + 0x67390000 | (((ReentryFnAddr.getValue() + 0x8000) >> 16) & 0xFFFF); uint32_t ReentryFnDSLL2 = 0x0019cc38; - uint32_t ReentryFnDADDiu3 = 0x67390000 | ((ReentryFnAddr)&0xFFFF); + uint32_t ReentryFnDADDiu3 = 0x67390000 | (ReentryFnAddr.getValue() & 0xFFFF); memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnLUi, sizeof(ReentryFnLUi)); @@ -827,16 +834,16 @@ void OrcMips64::writeResolverCode(char *ResolverWorkingMem, } void OrcMips64::writeTrampolines(char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, unsigned NumTrampolines) { uint32_t *Trampolines = reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); - uint64_t HeighestAddr = ((ResolverAddr + 0x800080008000) >> 48); - uint64_t HeigherAddr = ((ResolverAddr + 0x80008000) >> 32); - uint64_t HiAddr = ((ResolverAddr + 0x8000) >> 16); + uint64_t HeighestAddr = ((ResolverAddr.getValue() + 0x800080008000) >> 48); + uint64_t HeigherAddr = ((ResolverAddr.getValue() + 0x80008000) >> 32); + uint64_t HiAddr = ((ResolverAddr.getValue() + 0x8000) >> 16); for (unsigned I = 0; I < NumTrampolines; ++I) { Trampolines[10 * I + 0] = 0x03e0c025; // move $t8,$ra @@ -845,17 +852,18 @@ void OrcMips64::writeTrampolines(char *TrampolineBlockWorkingMem, Trampolines[10 * I + 3] = 0x0019cc38; // dsll $t9,$t9,16 Trampolines[10 * I + 4] = 0x67390000 | (HiAddr & 0xFFFF); // daddiu $t9,$t9,%hi(ptr) Trampolines[10 * I + 5] = 0x0019cc38; // dsll $t9,$t9,16 - Trampolines[10 * I + 6] = - 0x67390000 | (ResolverAddr & 0xFFFF); // daddiu $t9,$t9,%lo(ptr) + Trampolines[10 * I + 6] = 0x67390000 | (ResolverAddr.getValue() & + 0xFFFF); // daddiu $t9,$t9,%lo(ptr) Trampolines[10 * I + 7] = 0x0320f809; // jalr $t9 Trampolines[10 * I + 8] = 0x00000000; // nop Trampolines[10 * I + 9] = 0x00000000; // nop } } -void OrcMips64::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { +void OrcMips64::writeIndirectStubsBlock(char *StubsBlockWorkingMem, + ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, + unsigned NumStubs) { // Stub format is: // // .section __orc_stubs @@ -890,7 +898,7 @@ void OrcMips64::writeIndirectStubsBlock( // Populate the stubs page stubs and mark it executable. uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlockWorkingMem); - uint64_t PtrAddr = PointersBlockTargetAddress; + uint64_t PtrAddr = PointersBlockTargetAddress.getValue(); for (unsigned I = 0; I < NumStubs; ++I, PtrAddr += 8) { uint64_t HeighestAddr = ((PtrAddr + 0x800080008000) >> 48); @@ -908,9 +916,9 @@ void OrcMips64::writeIndirectStubsBlock( } void OrcRiscv64::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { const uint32_t ResolverCode[] = { 0xef810113, // 0x00: addi sp,sp,-264 @@ -1008,8 +1016,8 @@ void OrcRiscv64::writeResolverCode(char *ResolverWorkingMem, } void OrcRiscv64::writeTrampolines(char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, unsigned NumTrampolines) { unsigned OffsetToPtr = alignTo(NumTrampolines * TrampolineSize, 8); @@ -1031,8 +1039,8 @@ void OrcRiscv64::writeTrampolines(char *TrampolineBlockWorkingMem, } void OrcRiscv64::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs @@ -1078,9 +1086,9 @@ void OrcRiscv64::writeIndirectStubsBlock( } void OrcLoongArch64::writeResolverCode(char *ResolverWorkingMem, - JITTargetAddress ResolverTargetAddress, - JITTargetAddress ReentryFnAddr, - JITTargetAddress ReentryCtxAddr) { + ExecutorAddr ResolverTargetAddress, + ExecutorAddr ReentryFnAddr, + ExecutorAddr ReentryCtxAddr) { LLVM_DEBUG({ dbgs() << "Writing resolver code to " @@ -1150,10 +1158,10 @@ void OrcLoongArch64::writeResolverCode(char *ResolverWorkingMem, sizeof(uint64_t)); } -void OrcLoongArch64::writeTrampolines( - char *TrampolineBlockWorkingMem, - JITTargetAddress TrampolineBlockTargetAddress, - JITTargetAddress ResolverAddr, unsigned NumTrampolines) { +void OrcLoongArch64::writeTrampolines(char *TrampolineBlockWorkingMem, + ExecutorAddr TrampolineBlockTargetAddress, + ExecutorAddr ResolverAddr, + unsigned NumTrampolines) { LLVM_DEBUG({ dbgs() << "Writing trampoline code to " @@ -1181,8 +1189,8 @@ void OrcLoongArch64::writeTrampolines( } void OrcLoongArch64::writeIndirectStubsBlock( - char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, - JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + char *StubsBlockWorkingMem, ExecutorAddr StubsBlockTargetAddress, + ExecutorAddr PointersBlockTargetAddress, unsigned NumStubs) { // Stub format is: // // .section __orc_stubs diff --git a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp index b823197b404f..a73aec6d98c6 100644 --- a/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp +++ b/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -184,8 +184,8 @@ static SymbolMap toSymbolMap(LLVMOrcCSymbolMapPairs Syms, size_t NumPairs) { SymbolMap SM; for (size_t I = 0; I != NumPairs; ++I) { JITSymbolFlags Flags = toJITSymbolFlags(Syms[I].Sym.Flags); - SM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = - JITEvaluatedSymbol(Syms[I].Sym.Address, Flags); + SM[OrcV2CAPIHelper::moveToSymbolStringPtr(unwrap(Syms[I].Name))] = { + ExecutorAddr(Syms[I].Sym.Address), Flags}; } return SM; } @@ -269,8 +269,8 @@ static LLVMOrcSymbolLookupFlags fromSymbolLookupFlags(SymbolLookupFlags SLF) { } static LLVMJITEvaluatedSymbol -fromJITEvaluatedSymbol(const JITEvaluatedSymbol &S) { - return {S.getAddress(), fromJITSymbolFlags(S.getFlags())}; +fromExecutorSymbolDef(const ExecutorSymbolDef &S) { + return {S.getAddress().getValue(), fromJITSymbolFlags(S.getFlags())}; } } // end anonymous namespace @@ -385,7 +385,7 @@ void LLVMOrcExecutionSessionLookup( for (auto &KV : *Result) CResult.push_back(LLVMOrcCSymbolMapPair{ wrap(OrcV2CAPIHelper::getRawPoolEntryPtr(KV.first)), - fromJITEvaluatedSymbol(KV.second)}); + fromExecutorSymbolDef(KV.second)}); HandleResult(LLVMErrorSuccess, CResult.data(), CResult.size(), Ctx); } else HandleResult(wrap(Result.takeError()), nullptr, 0, Ctx); @@ -741,31 +741,19 @@ LLVMErrorRef LLVMOrcCreateDynamicLibrarySearchGeneratorForPath( LLVMErrorRef LLVMOrcCreateStaticLibrarySearchGeneratorForPath( LLVMOrcDefinitionGeneratorRef *Result, LLVMOrcObjectLayerRef ObjLayer, - const char *FileName, const char *TargetTriple) { + const char *FileName) { assert(Result && "Result can not be null"); assert(FileName && "Filename can not be null"); assert(ObjLayer && "ObjectLayer can not be null"); - if (TargetTriple) { - auto TT = Triple(TargetTriple); - auto LibrarySymsGenerator = - StaticLibraryDefinitionGenerator::Load(*unwrap(ObjLayer), FileName, TT); - if (!LibrarySymsGenerator) { - *Result = nullptr; - return wrap(LibrarySymsGenerator.takeError()); - } - *Result = wrap(LibrarySymsGenerator->release()); - return LLVMErrorSuccess; - } else { - auto LibrarySymsGenerator = - StaticLibraryDefinitionGenerator::Load(*unwrap(ObjLayer), FileName); - if (!LibrarySymsGenerator) { - *Result = nullptr; - return wrap(LibrarySymsGenerator.takeError()); - } - *Result = wrap(LibrarySymsGenerator->release()); - return LLVMErrorSuccess; + auto LibrarySymsGenerator = + StaticLibraryDefinitionGenerator::Load(*unwrap(ObjLayer), FileName); + if (!LibrarySymsGenerator) { + *Result = nullptr; + return wrap(LibrarySymsGenerator.takeError()); } + *Result = wrap(LibrarySymsGenerator->release()); + return LLVMErrorSuccess; } LLVMOrcThreadSafeContextRef LLVMOrcCreateNewThreadSafeContext(void) { @@ -859,9 +847,9 @@ LLVMErrorRef LLVMOrcObjectLayerAddObjectFile(LLVMOrcObjectLayerRef ObjLayer, *unwrap(JD), std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); } -LLVMErrorRef LLVMOrcLLJITAddObjectFileWithRT(LLVMOrcObjectLayerRef ObjLayer, - LLVMOrcResourceTrackerRef RT, - LLVMMemoryBufferRef ObjBuffer) { +LLVMErrorRef LLVMOrcObjectLayerAddObjectFileWithRT(LLVMOrcObjectLayerRef ObjLayer, + LLVMOrcResourceTrackerRef RT, + LLVMMemoryBufferRef ObjBuffer) { return wrap( unwrap(ObjLayer)->add(ResourceTrackerSP(unwrap(RT)), std::unique_ptr<MemoryBuffer>(unwrap(ObjBuffer)))); @@ -1210,8 +1198,8 @@ LLVMErrorRef LLVMOrcCreateLocalLazyCallThroughManager( const char *TargetTriple, LLVMOrcExecutionSessionRef ES, LLVMOrcJITTargetAddress ErrorHandlerAddr, LLVMOrcLazyCallThroughManagerRef *Result) { - auto LCTM = createLocalLazyCallThroughManager(Triple(TargetTriple), - *unwrap(ES), ErrorHandlerAddr); + auto LCTM = createLocalLazyCallThroughManager( + Triple(TargetTriple), *unwrap(ES), ExecutorAddr(ErrorHandlerAddr)); if (!LCTM) return wrap(LCTM.takeError()); diff --git a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 07b19b2e54f1..9ef333222028 100644 --- a/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -38,7 +38,8 @@ public: LookupResult Result; for (auto &KV : *InternedResult) - Result[*KV.first] = std::move(KV.second); + Result[*KV.first] = {KV.second.getAddress().getValue(), + KV.second.getFlags()}; OnResolved(Result); }; @@ -326,7 +327,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( } else if (AutoClaimObjectSymbols) ExtraSymbolsToClaim[InternedName] = Flags; - Symbols[InternedName] = JITEvaluatedSymbol(KV.second.getAddress(), Flags); + Symbols[InternedName] = {ExecutorAddr(KV.second.getAddress()), Flags}; } if (!ExtraSymbolsToClaim.empty()) { diff --git a/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp b/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp new file mode 100644 index 000000000000..ecf5e2915773 --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Shared/ObjectFormats.cpp @@ -0,0 +1,94 @@ +//===---------- ObjectFormats.cpp - Object format details for ORC ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ORC-specific object format details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" + +namespace llvm { +namespace orc { + +StringRef MachODataCommonSectionName = "__DATA,__common"; +StringRef MachODataDataSectionName = "__DATA,__data"; +StringRef MachOEHFrameSectionName = "__TEXT,__eh_frame"; +StringRef MachOCompactUnwindInfoSectionName = "__TEXT,__unwind_info"; +StringRef MachOModInitFuncSectionName = "__DATA,__mod_init_func"; +StringRef MachOObjCCatListSectionName = "__DATA,__objc_catlist"; +StringRef MachOObjCCatList2SectionName = "__DATA,__objc_catlist2"; +StringRef MachOObjCClassListSectionName = "__DATA,__objc_classlist"; +StringRef MachOObjCClassNameSectionName = "__TEXT,__objc_classname"; +StringRef MachOObjCClassRefsSectionName = "__DATA,__objc_classrefs"; +StringRef MachOObjCConstSectionName = "__DATA,__objc_const"; +StringRef MachOObjCDataSectionName = "__DATA,__objc_data"; +StringRef MachOObjCImageInfoSectionName = "__DATA,__objc_imageinfo"; +StringRef MachOObjCMethNameSectionName = "__TEXT,__objc_methname"; +StringRef MachOObjCMethTypeSectionName = "__TEXT,__objc_methtype"; +StringRef MachOObjCNLCatListSectionName = "__DATA,__objc_nlcatlist"; +StringRef MachOObjCSelRefsSectionName = "__DATA,__objc_selrefs"; +StringRef MachOSwift5ProtoSectionName = "__TEXT,__swift5_proto"; +StringRef MachOSwift5ProtosSectionName = "__TEXT,__swift5_protos"; +StringRef MachOSwift5TypesSectionName = "__TEXT,__swift5_types"; +StringRef MachOSwift5TypeRefSectionName = "__TEXT,__swift5_typeref"; +StringRef MachOSwift5FieldMetadataSectionName = "__TEXT,__swift5_fieldmd"; +StringRef MachOSwift5EntrySectionName = "__TEXT,__swift5_entry"; +StringRef MachOThreadBSSSectionName = "__DATA,__thread_bss"; +StringRef MachOThreadDataSectionName = "__DATA,__thread_data"; +StringRef MachOThreadVarsSectionName = "__DATA,__thread_vars"; + +StringRef MachOInitSectionNames[19] = { + MachOModInitFuncSectionName, MachOObjCCatListSectionName, + MachOObjCCatList2SectionName, MachOObjCClassListSectionName, + MachOObjCClassNameSectionName, MachOObjCClassRefsSectionName, + MachOObjCConstSectionName, MachOObjCDataSectionName, + MachOObjCImageInfoSectionName, MachOObjCMethNameSectionName, + MachOObjCMethTypeSectionName, MachOObjCNLCatListSectionName, + MachOObjCSelRefsSectionName, MachOSwift5ProtoSectionName, + MachOSwift5ProtosSectionName, MachOSwift5TypesSectionName, + MachOSwift5TypeRefSectionName, MachOSwift5FieldMetadataSectionName, + MachOSwift5EntrySectionName, +}; + +StringRef ELFEHFrameSectionName = ".eh_frame"; +StringRef ELFInitArrayFuncSectionName = ".init_array"; + +StringRef ELFThreadBSSSectionName = ".tbss"; +StringRef ELFThreadDataSectionName = ".tdata"; + +bool isMachOInitializerSection(StringRef SegName, StringRef SecName) { + for (auto &InitSection : MachOInitSectionNames) { + // Loop below assumes all MachO init sectios have a length-6 + // segment name. + assert(InitSection[6] == ',' && "Init section seg name has length != 6"); + if (InitSection.starts_with(SegName) && InitSection.substr(7) == SecName) + return true; + } + return false; +} + +bool isMachOInitializerSection(StringRef QualifiedName) { + for (auto &InitSection : MachOInitSectionNames) + if (InitSection == QualifiedName) + return true; + return false; +} + +bool isELFInitializerSection(StringRef SecName) { + if (SecName.consume_front(ELFInitArrayFuncSectionName) && + (SecName.empty() || SecName[0] == '.')) + return true; + return false; +} + +bool isCOFFInitializerSection(StringRef SecName) { + return SecName.startswith(".CRT"); +} + +} // namespace orc +} // namespace llvm diff --git a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp index 1bd10c9c6c0e..3d3ca891d881 100644 --- a/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp +++ b/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp @@ -49,7 +49,7 @@ Expected<int32_t> SimpleRemoteEPC::runAsMain(ExecutorAddr MainFnAddr, ArrayRef<std::string> Args) { int64_t Result = 0; if (auto Err = callSPSWrapper<rt::SPSRunAsMainSignature>( - RunAsMainAddr, Result, ExecutorAddr(MainFnAddr), Args)) + RunAsMainAddr, Result, MainFnAddr, Args)) return std::move(Err); return Result; } @@ -57,7 +57,7 @@ Expected<int32_t> SimpleRemoteEPC::runAsMain(ExecutorAddr MainFnAddr, Expected<int32_t> SimpleRemoteEPC::runAsVoidFunction(ExecutorAddr VoidFnAddr) { int32_t Result = 0; if (auto Err = callSPSWrapper<rt::SPSRunAsVoidFunctionSignature>( - RunAsVoidFunctionAddr, Result, ExecutorAddr(VoidFnAddr))) + RunAsVoidFunctionAddr, Result, VoidFnAddr)) return std::move(Err); return Result; } @@ -66,7 +66,7 @@ Expected<int32_t> SimpleRemoteEPC::runAsIntFunction(ExecutorAddr IntFnAddr, int Arg) { int32_t Result = 0; if (auto Err = callSPSWrapper<rt::SPSRunAsIntFunctionSignature>( - RunAsIntFunctionAddr, Result, ExecutorAddr(IntFnAddr), Arg)) + RunAsIntFunctionAddr, Result, IntFnAddr, Arg)) return std::move(Err); return Result; } @@ -126,23 +126,22 @@ SimpleRemoteEPC::handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, case SimpleRemoteEPCOpcode::Setup: dbgs() << "Setup"; assert(SeqNo == 0 && "Non-zero SeqNo for Setup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Setup?"); + assert(!TagAddr && "Non-zero TagAddr for Setup?"); break; case SimpleRemoteEPCOpcode::Hangup: dbgs() << "Hangup"; assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + assert(!TagAddr && "Non-zero TagAddr for Hangup?"); break; case SimpleRemoteEPCOpcode::Result: dbgs() << "Result"; - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + assert(!TagAddr && "Non-zero TagAddr for Result?"); break; case SimpleRemoteEPCOpcode::CallWrapper: dbgs() << "CallWrapper"; break; } - dbgs() << ", seqno = " << SeqNo - << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + dbgs() << ", seqno = " << SeqNo << ", tag-addr = " << TagAddr << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) << " bytes\n"; }); @@ -227,11 +226,11 @@ Error SimpleRemoteEPC::sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, case SimpleRemoteEPCOpcode::Hangup: dbgs() << "Hangup"; assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + assert(!TagAddr && "Non-zero TagAddr for Hangup?"); break; case SimpleRemoteEPCOpcode::Result: dbgs() << "Result"; - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + assert(!TagAddr && "Non-zero TagAddr for Result?"); break; case SimpleRemoteEPCOpcode::CallWrapper: dbgs() << "CallWrapper"; @@ -239,8 +238,7 @@ Error SimpleRemoteEPC::sendMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, default: llvm_unreachable("Invalid opcode"); } - dbgs() << ", seqno = " << SeqNo - << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + dbgs() << ", seqno = " << SeqNo << ", tag-addr = " << TagAddr << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) << " bytes\n"; }); @@ -317,13 +315,19 @@ Error SimpleRemoteEPC::setup(Setup S) { dbgs() << "SimpleRemoteEPC received setup message:\n" << " Triple: " << EI->TargetTriple << "\n" << " Page size: " << EI->PageSize << "\n" - << " Bootstrap symbols:\n"; + << " Bootstrap map" << (EI->BootstrapMap.empty() ? " empty" : ":") + << "\n"; + for (const auto &KV : EI->BootstrapMap) + dbgs() << " " << KV.first() << ": " << KV.second.size() + << "-byte SPS encoded buffer\n"; + dbgs() << " Bootstrap symbols" + << (EI->BootstrapSymbols.empty() ? " empty" : ":") << "\n"; for (const auto &KV : EI->BootstrapSymbols) - dbgs() << " " << KV.first() << ": " - << formatv("{0:x16}", KV.second.getValue()) << "\n"; + dbgs() << " " << KV.first() << ": " << KV.second << "\n"; }); TargetTriple = Triple(EI->TargetTriple); PageSize = EI->PageSize; + BootstrapMap = std::move(EI->BootstrapMap); BootstrapSymbols = std::move(EI->BootstrapSymbols); if (auto Err = getBootstrapSymbols( @@ -402,7 +406,7 @@ void SimpleRemoteEPC::handleCallWrapper( ExecutorAddr(), {WFR.data(), WFR.size()})) getExecutionSession().reportError(std::move(Err)); }, - TagAddr.getValue(), ArgBytes); + TagAddr, ArgBytes); }, "callWrapper task")); } diff --git a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp index b52d01318c0d..d4cbd1970d8f 100644 --- a/llvm/lib/ExecutionEngine/Orc/Speculation.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Speculation.cpp @@ -36,16 +36,15 @@ void ImplSymbolMap::trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD) { // Trigger Speculative Compiles. void Speculator::speculateForEntryPoint(Speculator *Ptr, uint64_t StubId) { assert(Ptr && " Null Address Received in orc_speculate_for "); - Ptr->speculateFor(StubId); + Ptr->speculateFor(ExecutorAddr(StubId)); } Error Speculator::addSpeculationRuntime(JITDylib &JD, MangleAndInterner &Mangle) { - JITEvaluatedSymbol ThisPtr(pointerToJITTargetAddress(this), - JITSymbolFlags::Exported); - JITEvaluatedSymbol SpeculateForEntryPtr( - pointerToJITTargetAddress(&speculateForEntryPoint), - JITSymbolFlags::Exported); + ExecutorSymbolDef ThisPtr(ExecutorAddr::fromPtr(this), + JITSymbolFlags::Exported); + ExecutorSymbolDef SpeculateForEntryPtr( + ExecutorAddr::fromPtr(&speculateForEntryPoint), JITSymbolFlags::Exported); return JD.define(absoluteSymbols({ {Mangle("__orc_speculator"), ThisPtr}, // Data Symbol {Mangle("__orc_speculate_for"), SpeculateForEntryPtr} // Callable Symbol diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp index 147f915f61d6..3f70dbf60437 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp @@ -132,11 +132,11 @@ Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( #if defined(LLVM_ON_UNIX) int NativeProt = 0; - if ((Segment.AG.getMemProt() & MemProt::Read) == MemProt::Read) + if ((Segment.RAG.Prot & MemProt::Read) == MemProt::Read) NativeProt |= PROT_READ; - if ((Segment.AG.getMemProt() & MemProt::Write) == MemProt::Write) + if ((Segment.RAG.Prot & MemProt::Write) == MemProt::Write) NativeProt |= PROT_WRITE; - if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) + if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec) NativeProt |= PROT_EXEC; if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt)) @@ -144,8 +144,7 @@ Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( #elif defined(_WIN32) - DWORD NativeProt = - getWindowsProtectionFlags(Segment.AG.getMemProt()); + DWORD NativeProt = getWindowsProtectionFlags(Segment.RAG.Prot); if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt, &NativeProt)) @@ -153,7 +152,7 @@ Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( #endif - if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) + if ((Segment.RAG.Prot & MemProt::Exec) == MemProt::Exec) sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(), Segment.Size); } diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp index 8296b03398a0..8eca874c48b8 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.cpp @@ -67,9 +67,9 @@ using namespace llvm; using namespace llvm::orc; // Register debug object, return error message or null for success. -static void registerJITLoaderGDBImpl(const char *ObjAddr, size_t Size) { +static void appendJITDebugDescriptor(const char *ObjAddr, size_t Size) { LLVM_DEBUG({ - dbgs() << "Registering debug object with GDB JIT interface " + dbgs() << "Adding debug object to GDB JIT interface " << formatv("([{0:x16} -- {1:x16}])", reinterpret_cast<uintptr_t>(ObjAddr), reinterpret_cast<uintptr_t>(ObjAddr + Size)) @@ -94,20 +94,20 @@ static void registerJITLoaderGDBImpl(const char *ObjAddr, size_t Size) { __jit_debug_descriptor.first_entry = E; __jit_debug_descriptor.relevant_entry = E; - - // Run into the rendezvous breakpoint. __jit_debug_descriptor.action_flag = JIT_REGISTER_FN; - __jit_debug_register_code(); } extern "C" orc::shared::CWrapperFunctionResult llvm_orc_registerJITLoaderGDBAllocAction(const char *Data, size_t Size) { using namespace orc::shared; - return WrapperFunction<SPSError(SPSExecutorAddrRange)>::handle( + return WrapperFunction<SPSError(SPSExecutorAddrRange, bool)>::handle( Data, Size, - [](ExecutorAddrRange R) { - registerJITLoaderGDBImpl(R.Start.toPtr<const char *>(), + [](ExecutorAddrRange R, bool AutoRegisterCode) { + appendJITDebugDescriptor(R.Start.toPtr<const char *>(), R.size()); + // Run into the rendezvous breakpoint. + if (AutoRegisterCode) + __jit_debug_register_code(); return Error::success(); }) .release(); @@ -116,11 +116,14 @@ llvm_orc_registerJITLoaderGDBAllocAction(const char *Data, size_t Size) { extern "C" orc::shared::CWrapperFunctionResult llvm_orc_registerJITLoaderGDBWrapper(const char *Data, uint64_t Size) { using namespace orc::shared; - return WrapperFunction<SPSError(SPSExecutorAddrRange)>::handle( + return WrapperFunction<SPSError(SPSExecutorAddrRange, bool)>::handle( Data, Size, - [](ExecutorAddrRange R) { - registerJITLoaderGDBImpl(R.Start.toPtr<const char *>(), + [](ExecutorAddrRange R, bool AutoRegisterCode) { + appendJITDebugDescriptor(R.Start.toPtr<const char *>(), R.size()); + // Run into the rendezvous breakpoint. + if (AutoRegisterCode) + __jit_debug_register_code(); return Error::success(); }) .release(); diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp index ce94bf1e039a..4da031716e32 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp @@ -132,9 +132,9 @@ Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) { assert(Seg.Size <= std::numeric_limits<size_t>::max()); if (auto EC = sys::Memory::protectMappedMemory( {Mem, static_cast<size_t>(Seg.Size)}, - toSysMemoryProtectionFlags(Seg.AG.getMemProt()))) + toSysMemoryProtectionFlags(Seg.RAG.Prot))) return BailOut(errorCodeToError(EC)); - if ((Seg.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) + if ((Seg.RAG.Prot & MemProt::Exec) == MemProt::Exec) sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); } diff --git a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp index 8ab0af3eab6e..67bc379f9821 100644 --- a/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp +++ b/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleRemoteEPCServer.cpp @@ -10,8 +10,8 @@ #include "llvm/ExecutionEngine/Orc/Shared/TargetProcessControlTypes.h" #include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/Host.h" #include "llvm/Support/Process.h" +#include "llvm/TargetParser/Host.h" #include "OrcRTBootstrap.h" @@ -68,23 +68,22 @@ SimpleRemoteEPCServer::handleMessage(SimpleRemoteEPCOpcode OpC, uint64_t SeqNo, case SimpleRemoteEPCOpcode::Setup: dbgs() << "Setup"; assert(SeqNo == 0 && "Non-zero SeqNo for Setup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Setup?"); + assert(!TagAddr && "Non-zero TagAddr for Setup?"); break; case SimpleRemoteEPCOpcode::Hangup: dbgs() << "Hangup"; assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + assert(!TagAddr && "Non-zero TagAddr for Hangup?"); break; case SimpleRemoteEPCOpcode::Result: dbgs() << "Result"; - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + assert(!TagAddr && "Non-zero TagAddr for Result?"); break; case SimpleRemoteEPCOpcode::CallWrapper: dbgs() << "CallWrapper"; break; } - dbgs() << ", seqno = " << SeqNo - << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + dbgs() << ", seqno = " << SeqNo << ", tag-addr = " << TagAddr << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) << " bytes\n"; }); @@ -158,23 +157,22 @@ Error SimpleRemoteEPCServer::sendMessage(SimpleRemoteEPCOpcode OpC, case SimpleRemoteEPCOpcode::Setup: dbgs() << "Setup"; assert(SeqNo == 0 && "Non-zero SeqNo for Setup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Setup?"); + assert(!TagAddr && "Non-zero TagAddr for Setup?"); break; case SimpleRemoteEPCOpcode::Hangup: dbgs() << "Hangup"; assert(SeqNo == 0 && "Non-zero SeqNo for Hangup?"); - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Hangup?"); + assert(!TagAddr && "Non-zero TagAddr for Hangup?"); break; case SimpleRemoteEPCOpcode::Result: dbgs() << "Result"; - assert(TagAddr.getValue() == 0 && "Non-zero TagAddr for Result?"); + assert(!TagAddr && "Non-zero TagAddr for Result?"); break; case SimpleRemoteEPCOpcode::CallWrapper: dbgs() << "CallWrapper"; break; } - dbgs() << ", seqno = " << SeqNo - << ", tag-addr = " << formatv("{0:x}", TagAddr.getValue()) + dbgs() << ", seqno = " << SeqNo << ", tag-addr = " << TagAddr << ", arg-buffer = " << formatv("{0:x}", ArgBytes.size()) << " bytes\n"; }); @@ -187,6 +185,7 @@ Error SimpleRemoteEPCServer::sendMessage(SimpleRemoteEPCOpcode OpC, } Error SimpleRemoteEPCServer::sendSetupMessage( + StringMap<std::vector<char>> BootstrapMap, StringMap<ExecutorAddr> BootstrapSymbols) { using namespace SimpleRemoteEPCDefaultBootstrapSymbolNames; @@ -198,6 +197,7 @@ Error SimpleRemoteEPCServer::sendSetupMessage( EI.PageSize = *PageSize; else return PageSize.takeError(); + EI.BootstrapMap = std::move(BootstrapMap); EI.BootstrapSymbols = std::move(BootstrapSymbols); assert(!EI.BootstrapSymbols.count(ExecutorSessionObjectName) && diff --git a/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp index b425eec5f6d6..62cab22a1c45 100644 --- a/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp +++ b/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp @@ -417,7 +417,7 @@ void PerfJITEventListener::NotifyCode(Expected<llvm::StringRef> &Symbol, rec.Prefix.Timestamp = perf_get_timestamp(); rec.CodeSize = CodeSize; - rec.Vma = 0; + rec.Vma = CodeAddr; rec.CodeAddr = CodeAddr; rec.Pid = Pid; rec.Tid = get_threadid(); diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp index 1d8f1ac8ac8a..9255311f992d 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -16,9 +16,9 @@ #include "Targets/RuntimeDyldCOFFThumb.h" #include "Targets/RuntimeDyldCOFFX86_64.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/Triple.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/FormatVariadic.h" +#include "llvm/TargetParser/Triple.h" using namespace llvm; using namespace llvm::object; diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 2fe49fefae2d..d439b1b4ebfb 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -15,12 +15,12 @@ #include "Targets/RuntimeDyldELFMips.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringRef.h" -#include "llvm/ADT/Triple.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Endian.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/TargetParser/Triple.h" using namespace llvm; using namespace llvm::object; @@ -426,13 +426,15 @@ void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section, break; case ELF::R_AARCH64_ABS16: { uint64_t Result = Value + Addend; - assert(static_cast<int64_t>(Result) >= INT16_MIN && Result < UINT16_MAX); + assert(Result == static_cast<uint64_t>(llvm::SignExtend64(Result, 16)) || + (Result >> 16) == 0); write(isBE, TargetPtr, static_cast<uint16_t>(Result & 0xffffU)); break; } case ELF::R_AARCH64_ABS32: { uint64_t Result = Value + Addend; - assert(static_cast<int64_t>(Result) >= INT32_MIN && Result < UINT32_MAX); + assert(Result == static_cast<uint64_t>(llvm::SignExtend64(Result, 32)) || + (Result >> 32) == 0); write(isBE, TargetPtr, static_cast<uint32_t>(Result & 0xffffffffU)); break; } @@ -477,7 +479,9 @@ void RuntimeDyldELF::resolveAArch64Relocation(const SectionEntry &Section, assert(isInt<16>(BranchImm)); - *TargetPtr &= 0xfff8001fU; + uint32_t RawInstr = *(support::little32_t *)TargetPtr; + *(support::little32_t *)TargetPtr = RawInstr & 0xfff8001fU; + // Immediate:15:2 goes in bits 18:5 of TBZ, TBNZ or32le(TargetPtr, (BranchImm & 0x0000FFFC) << 3); break; @@ -1282,6 +1286,7 @@ RuntimeDyldELF::processRelocationRef( } case SymbolRef::ST_Data: case SymbolRef::ST_Function: + case SymbolRef::ST_Other: case SymbolRef::ST_Unknown: { Value.SymbolName = TargetName.data(); Value.Addend = Addend; @@ -2405,6 +2410,7 @@ Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, } } + GOTOffsetMap.clear(); GOTSectionID = 0; CurrentGOTIndex = 0; diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index bf33a2dec18a..501417db421a 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -15,7 +15,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringMap.h" -#include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/RTDyldMemoryManager.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/ExecutionEngine/RuntimeDyldChecker.h" @@ -23,9 +22,10 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/Format.h" -#include "llvm/Support/Host.h" #include "llvm/Support/Mutex.h" #include "llvm/Support/SwapByteOrder.h" +#include "llvm/TargetParser/Host.h" +#include "llvm/TargetParser/Triple.h" #include <deque> #include <map> #include <system_error> diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h index 342c4221ff0c..da381986e9de 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h @@ -15,6 +15,7 @@ #define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFAARCH64_H #include "../RuntimeDyldCOFF.h" +#include "llvm/ADT/SmallString.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/Object/COFF.h" #include "llvm/Support/Endian.h" diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h index 3859f36ac4bd..22f1cf33158c 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h @@ -14,6 +14,7 @@ #define LLVM_LIB_EXECUTIONENGINE_RUNTIMEDYLD_TARGETS_RUNTIMEDYLDCOFFTHUMB_H #include "../RuntimeDyldCOFF.h" +#include "llvm/ADT/SmallString.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/Object/COFF.h" diff --git a/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp b/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp index b23e33039c35..436888730bfb 100644 --- a/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/SectionMemoryManager.cpp @@ -101,7 +101,7 @@ uint8_t *SectionMemoryManager::allocateSection( // FIXME: Initialize the Near member for each memory group to avoid // interleaving. std::error_code ec; - sys::MemoryBlock MB = MMapper.allocateMappedMemory( + sys::MemoryBlock MB = MMapper->allocateMappedMemory( Purpose, RequiredSize, &MemGroup.Near, sys::Memory::MF_READ | sys::Memory::MF_WRITE, ec); if (ec) { @@ -204,7 +204,7 @@ std::error_code SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, unsigned Permissions) { for (sys::MemoryBlock &MB : MemGroup.PendingMem) - if (std::error_code EC = MMapper.protectMappedMemory(MB, Permissions)) + if (std::error_code EC = MMapper->protectMappedMemory(MB, Permissions)) return EC; MemGroup.PendingMem.clear(); @@ -234,7 +234,7 @@ void SectionMemoryManager::invalidateInstructionCache() { SectionMemoryManager::~SectionMemoryManager() { for (MemoryGroup *Group : {&CodeMem, &RWDataMem, &RODataMem}) { for (sys::MemoryBlock &Block : Group->AllocatedMem) - MMapper.releaseMappedMemory(Block); + MMapper->releaseMappedMemory(Block); } } @@ -263,11 +263,14 @@ public: return sys::Memory::releaseMappedMemory(M); } }; - -DefaultMMapper DefaultMMapperInstance; } // namespace -SectionMemoryManager::SectionMemoryManager(MemoryMapper *MM) - : MMapper(MM ? *MM : DefaultMMapperInstance) {} +SectionMemoryManager::SectionMemoryManager(MemoryMapper *UnownedMM) + : MMapper(UnownedMM), OwnedMMapper(nullptr) { + if (!MMapper) { + OwnedMMapper = std::make_unique<DefaultMMapper>(); + MMapper = OwnedMMapper.get(); + } +} } // namespace llvm diff --git a/llvm/lib/ExecutionEngine/TargetSelect.cpp b/llvm/lib/ExecutionEngine/TargetSelect.cpp index c67a1a7661d6..72fb16fbf203 100644 --- a/llvm/lib/ExecutionEngine/TargetSelect.cpp +++ b/llvm/lib/ExecutionEngine/TargetSelect.cpp @@ -13,13 +13,13 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/Triple.h" #include "llvm/ExecutionEngine/ExecutionEngine.h" #include "llvm/IR/Module.h" -#include "llvm/MC/SubtargetFeature.h" #include "llvm/MC/TargetRegistry.h" -#include "llvm/Support/Host.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/TargetParser/Host.h" +#include "llvm/TargetParser/SubtargetFeature.h" +#include "llvm/TargetParser/Triple.h" using namespace llvm; @@ -89,7 +89,6 @@ TargetMachine *EngineBuilder::selectTarget(const Triple &TargetTriple, Options, RelocModel, CMModel, OptLevel, /*JIT*/ true); Target->Options.EmulatedTLS = EmulatedTLS; - Target->Options.ExplicitEmulatedTLS = true; assert(Target && "Could not allocate target machine!"); return Target; |