diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ExecutionEngine')
88 files changed, 4424 insertions, 1040 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/ExecutionEngine.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/ExecutionEngine.cpp index a14bd4d2c3fd..2a90b67bee4b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/ExecutionEngine.cpp @@ -395,7 +395,7 @@ void ExecutionEngine::runStaticConstructorsDestructors(Module &module, // Execute the ctor/dtor function! if (Function *F = dyn_cast<Function>(FP)) - runFunction(F, None); + runFunction(F, std::nullopt); // FIXME: It is marginally lame that we just do nothing here if we see an // entry we don't recognize. It might not be unreasonable for the verifier @@ -719,7 +719,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { APFloat apf = APFloat(APFloat::x87DoubleExtended(), GV.IntVal); uint64_t v; bool ignored; - (void)apf.convertToInteger(makeMutableArrayRef(v), BitWidth, + (void)apf.convertToInteger(MutableArrayRef(v), BitWidth, CE->getOpcode()==Instruction::FPToSI, APFloat::rmTowardZero, &ignored); GV.IntVal = v; // endian? diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp index 672fd7b991c2..dc9a07e3f212 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -21,6 +21,7 @@ #include "llvm/Target/CodeGenCWrappers.h" #include "llvm/Target/TargetOptions.h" #include <cstring> +#include <optional> using namespace llvm; @@ -199,7 +200,7 @@ LLVMBool LLVMCreateMCJITCompilerForModule( .setOptLevel((CodeGenOpt::Level)options.OptLevel) .setTargetOptions(targetOptions); bool JIT; - if (Optional<CodeModel::Model> CM = unwrap(options.CodeModel, JIT)) + if (std::optional<CodeModel::Model> CM = unwrap(options.CodeModel, JIT)) builder.setCodeModel(*CM); if (options.MCJMM) builder.setMCJITMemoryManager( diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c b/contrib/llvm-project/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c index 074e0735628a..50d64d70c98a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c @@ -258,7 +258,7 @@ iJIT_RegisterCallbackEx(void *userdata, iJIT_ModeChangedEx * This function allows the user to query in which mode, if at all, *VTune is running */ -ITT_EXTERN_C iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive() +ITT_EXTERN_C iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive(void) { if (!iJIT_DLL_is_missing) { @@ -273,7 +273,7 @@ ITT_EXTERN_C iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive() * on success: all functions load, iJIT_DLL_is_missing = 0, return value = 1 * on failure: all functions are NULL, iJIT_DLL_is_missing = 1, return value = 0 */ -static int loadiJIT_Funcs() +static int loadiJIT_Funcs(void) { static int bDllWasLoaded = 0; char *dllName = (char*)rcsid; /* !! Just to avoid unused code elimination */ @@ -416,7 +416,7 @@ static int loadiJIT_Funcs() * This function should be called by the user whenever a thread ends, * to free the thread "virtual stack" storage */ -ITT_EXTERN_C void JITAPI FinalizeThread() +ITT_EXTERN_C void JITAPI FinalizeThread(void) { if (threadLocalStorageHandle) { @@ -444,7 +444,7 @@ ITT_EXTERN_C void JITAPI FinalizeThread() * This function should be called by the user when the process ends, * to free the local storage index */ -ITT_EXTERN_C void JITAPI FinalizeProcess() +ITT_EXTERN_C void JITAPI FinalizeProcess(void) { if (m_libHandle) { @@ -469,7 +469,7 @@ ITT_EXTERN_C void JITAPI FinalizeProcess() * The function will return a unique method ID, the user should maintain * the ID for each method */ -ITT_EXTERN_C unsigned int JITAPI iJIT_GetNewMethodID() +ITT_EXTERN_C unsigned int JITAPI iJIT_GetNewMethodID(void) { static unsigned int methodID = 0x100000; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index c3ba5ebb36fb..29f481a1e4e8 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -30,7 +30,6 @@ #include "llvm/Support/Casting.h" #include "llvm/Support/DynamicLibrary.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/ManagedStatic.h" #include "llvm/Support/Mutex.h" #include "llvm/Support/raw_ostream.h" #include <cassert> @@ -57,16 +56,26 @@ using namespace llvm; -static ManagedStatic<sys::Mutex> FunctionsLock; +namespace { typedef GenericValue (*ExFunc)(FunctionType *, ArrayRef<GenericValue>); -static ManagedStatic<std::map<const Function *, ExFunc> > ExportedFunctions; -static ManagedStatic<std::map<std::string, ExFunc> > FuncNames; +typedef void (*RawFunc)(); +struct Functions { + sys::Mutex Lock; + std::map<const Function *, ExFunc> ExportedFunctions; + std::map<std::string, ExFunc> FuncNames; #ifdef USE_LIBFFI -typedef void (*RawFunc)(); -static ManagedStatic<std::map<const Function *, RawFunc> > RawFunctions; + std::map<const Function *, RawFunc> RawFunctions; #endif +}; + +Functions &getFunctions() { + static Functions F; + return F; +} + +} // anonymous namespace static Interpreter *TheInterpreter; @@ -107,15 +116,16 @@ static ExFunc lookupFunction(const Function *F) { ExtName += getTypeID(T); ExtName += ("_" + F->getName()).str(); - sys::ScopedLock Writer(*FunctionsLock); - ExFunc FnPtr = (*FuncNames)[ExtName]; + auto &Fns = getFunctions(); + sys::ScopedLock Writer(Fns.Lock); + ExFunc FnPtr = Fns.FuncNames[ExtName]; if (!FnPtr) - FnPtr = (*FuncNames)[("lle_X_" + F->getName()).str()]; + FnPtr = Fns.FuncNames[("lle_X_" + F->getName()).str()]; if (!FnPtr) // Try calling a generic function... if it exists... FnPtr = (ExFunc)(intptr_t)sys::DynamicLibrary::SearchForAddressOfSymbol( ("lle_X_" + F->getName()).str()); if (FnPtr) - ExportedFunctions->insert(std::make_pair(F, FnPtr)); // Cache for later + Fns.ExportedFunctions.insert(std::make_pair(F, FnPtr)); // Cache for later return FnPtr; } @@ -260,27 +270,29 @@ GenericValue Interpreter::callExternalFunction(Function *F, ArrayRef<GenericValue> ArgVals) { TheInterpreter = this; - std::unique_lock<sys::Mutex> Guard(*FunctionsLock); + auto &Fns = getFunctions(); + std::unique_lock<sys::Mutex> Guard(Fns.Lock); // Do a lookup to see if the function is in our cache... this should just be a // deferred annotation! - std::map<const Function *, ExFunc>::iterator FI = ExportedFunctions->find(F); - if (ExFunc Fn = (FI == ExportedFunctions->end()) ? lookupFunction(F) - : FI->second) { + std::map<const Function *, ExFunc>::iterator FI = + Fns.ExportedFunctions.find(F); + if (ExFunc Fn = (FI == Fns.ExportedFunctions.end()) ? lookupFunction(F) + : FI->second) { Guard.unlock(); return Fn(F->getFunctionType(), ArgVals); } #ifdef USE_LIBFFI - std::map<const Function *, RawFunc>::iterator RF = RawFunctions->find(F); + std::map<const Function *, RawFunc>::iterator RF = Fns.RawFunctions.find(F); RawFunc RawFn; - if (RF == RawFunctions->end()) { + if (RF == Fns.RawFunctions.end()) { RawFn = (RawFunc)(intptr_t) sys::DynamicLibrary::SearchForAddressOfSymbol(std::string(F->getName())); if (!RawFn) RawFn = (RawFunc)(intptr_t)getPointerToGlobalIfAvailable(F); if (RawFn != 0) - RawFunctions->insert(std::make_pair(F, RawFn)); // Cache for later + Fns.RawFunctions.insert(std::make_pair(F, RawFn)); // Cache for later } else { RawFn = RF->second; } @@ -496,16 +508,17 @@ static GenericValue lle_X_memcpy(FunctionType *FT, } void Interpreter::initializeExternalFunctions() { - sys::ScopedLock Writer(*FunctionsLock); - (*FuncNames)["lle_X_atexit"] = lle_X_atexit; - (*FuncNames)["lle_X_exit"] = lle_X_exit; - (*FuncNames)["lle_X_abort"] = lle_X_abort; - - (*FuncNames)["lle_X_printf"] = lle_X_printf; - (*FuncNames)["lle_X_sprintf"] = lle_X_sprintf; - (*FuncNames)["lle_X_sscanf"] = lle_X_sscanf; - (*FuncNames)["lle_X_scanf"] = lle_X_scanf; - (*FuncNames)["lle_X_fprintf"] = lle_X_fprintf; - (*FuncNames)["lle_X_memset"] = lle_X_memset; - (*FuncNames)["lle_X_memcpy"] = lle_X_memcpy; + auto &Fns = getFunctions(); + sys::ScopedLock Writer(Fns.Lock); + Fns.FuncNames["lle_X_atexit"] = lle_X_atexit; + Fns.FuncNames["lle_X_exit"] = lle_X_exit; + Fns.FuncNames["lle_X_abort"] = lle_X_abort; + + Fns.FuncNames["lle_X_printf"] = lle_X_printf; + Fns.FuncNames["lle_X_sprintf"] = lle_X_sprintf; + Fns.FuncNames["lle_X_sscanf"] = lle_X_sscanf; + Fns.FuncNames["lle_X_scanf"] = lle_X_scanf; + Fns.FuncNames["lle_X_fprintf"] = lle_X_fprintf; + Fns.FuncNames["lle_X_memset"] = lle_X_memset; + Fns.FuncNames["lle_X_memcpy"] = lle_X_memcpy; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp index 5727f7adb49c..d4235cfa2ccf 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Interpreter/Interpreter.cpp @@ -69,7 +69,7 @@ Interpreter::~Interpreter() { void Interpreter::runAtExitHandlers () { while (!AtExitHandlers.empty()) { - callFunction(AtExitHandlers.back(), None); + callFunction(AtExitHandlers.back(), std::nullopt); AtExitHandlers.pop_back(); run(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp new file mode 100644 index 000000000000..30c1579a1ba0 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.cpp @@ -0,0 +1,88 @@ +//===-- COFFDirectiveParser.cpp - JITLink coff directive parser --*- C++ -*===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MSVC COFF directive parser +// +//===----------------------------------------------------------------------===// + +#include "COFFDirectiveParser.h" + +#include <array> + +using namespace llvm; +using namespace jitlink; + +#define DEBUG_TYPE "jitlink" + +// Create prefix string literals used in Options.td +#define PREFIX(NAME, VALUE) \ + static constexpr StringLiteral NAME##_init[] = VALUE; \ + static constexpr ArrayRef<StringLiteral> NAME(NAME##_init, \ + std::size(NAME##_init) - 1); +#include "COFFOptions.inc" +#undef PREFIX + +static constexpr const StringLiteral PrefixTable_init[] = +#define PREFIX_UNION(VALUES) VALUES +#include "COFFOptions.inc" +#undef PREFIX_UNION + ; +static constexpr const ArrayRef<StringLiteral> + PrefixTable(PrefixTable_init, std::size(PrefixTable_init) - 1); + +// Create table mapping all options defined in COFFOptions.td +static constexpr opt::OptTable::Info infoTable[] = { +#define OPTION(X1, X2, ID, KIND, GROUP, ALIAS, X7, X8, X9, X10, X11, X12) \ + {X1, \ + X2, \ + X10, \ + X11, \ + COFF_OPT_##ID, \ + opt::Option::KIND##Class, \ + X9, \ + X8, \ + COFF_OPT_##GROUP, \ + COFF_OPT_##ALIAS, \ + X7, \ + X12}, +#include "COFFOptions.inc" +#undef OPTION +}; + +class COFFOptTable : public opt::PrecomputedOptTable { +public: + COFFOptTable() : PrecomputedOptTable(infoTable, PrefixTable, true) {} +}; + +static COFFOptTable optTable; + +Expected<opt::InputArgList> COFFDirectiveParser::parse(StringRef Str) { + SmallVector<StringRef, 16> Tokens; + SmallVector<const char *, 16> Buffer; + cl::TokenizeWindowsCommandLineNoCopy(Str, saver, Tokens); + for (StringRef Tok : Tokens) { + bool HasNul = Tok.end() != Str.end() && Tok.data()[Tok.size()] == '\0'; + Buffer.push_back(HasNul ? Tok.data() : saver.save(Tok).data()); + } + + unsigned missingIndex; + unsigned missingCount; + + auto Result = optTable.ParseArgs(Buffer, missingIndex, missingCount); + + if (missingCount) + return make_error<JITLinkError>(Twine("COFF directive parsing failed: ") + + Result.getArgString(missingIndex) + + " missing argument"); + LLVM_DEBUG({ + for (auto *arg : Result.filtered(COFF_OPT_UNKNOWN)) + dbgs() << "Unknown coff option argument: " << arg->getAsString(Result) + << "\n"; + }); + return std::move(Result); +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h new file mode 100644 index 000000000000..8d5e0f7314dd --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFDirectiveParser.h @@ -0,0 +1,48 @@ +//===--- COFFDirectiveParser.h - JITLink coff directive parser --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MSVC COFF directive parser +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_COFFDIRECTIVEPARSER_H +#define LLVM_EXECUTIONENGINE_JITLINK_COFFDIRECTIVEPARSER_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Option/Arg.h" +#include "llvm/Option/ArgList.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/StringSaver.h" + +namespace llvm { +namespace jitlink { + +enum { + COFF_OPT_INVALID = 0, +#define OPTION(_1, _2, ID, _4, _5, _6, _7, _8, _9, _10, _11, _12) COFF_OPT_##ID, +#include "COFFOptions.inc" +#undef OPTION +}; + +/// Parser for the MSVC specific preprocessor directives. +/// https://docs.microsoft.com/en-us/cpp/preprocessor/comment-c-cpp?view=msvc-160 +class COFFDirectiveParser { +public: + Expected<opt::InputArgList> parse(StringRef Str); + +private: + llvm::BumpPtrAllocator bAlloc; + llvm::StringSaver saver{bAlloc}; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_COFFDIRECTIVEPARSER_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp index 3a6162db75c4..782928c26084 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.cpp @@ -71,8 +71,8 @@ bool COFFLinkGraphBuilder::isComdatSection( Section &COFFLinkGraphBuilder::getCommonSection() { if (!CommonSection) - CommonSection = - &G->createSection(CommonSectionName, MemProt::Read | MemProt::Write); + CommonSection = &G->createSection(CommonSectionName, + orc::MemProt::Read | orc::MemProt::Write); return *CommonSection; } @@ -141,16 +141,14 @@ Error COFFLinkGraphBuilder::graphifySections() { << "Creating section for \"" << SectionName << "\"\n"; }); - // FIXME: Revisit crash when dropping IMAGE_SCN_MEM_DISCARDABLE sections - // Get the section's memory protection flags. - MemProt Prot = MemProt::None; + orc::MemProt Prot = orc::MemProt::Read; if ((*Sec)->Characteristics & COFF::IMAGE_SCN_MEM_EXECUTE) - Prot |= MemProt::Exec; + Prot |= orc::MemProt::Exec; if ((*Sec)->Characteristics & COFF::IMAGE_SCN_MEM_READ) - Prot |= MemProt::Read; + Prot |= orc::MemProt::Read; if ((*Sec)->Characteristics & COFF::IMAGE_SCN_MEM_WRITE) - Prot |= MemProt::Write; + Prot |= orc::MemProt::Write; // Look for existing sections first. auto *GraphSec = G->findSectionByName(SectionName); @@ -170,11 +168,16 @@ Error COFFLinkGraphBuilder::graphifySections() { if (auto Err = Obj.getSectionContents(*Sec, Data)) return Err; + auto CharData = ArrayRef<char>( + reinterpret_cast<const char *>(Data.data()), Data.size()); + + if (SectionName == getDirectiveSectionName()) + if (auto Err = handleDirectiveSection( + StringRef(CharData.data(), CharData.size()))) + return Err; + B = &G->createContentBlock( - *GraphSec, - ArrayRef<char>(reinterpret_cast<const char *>(Data.data()), - Data.size()), - orc::ExecutorAddr(getSectionAddress(Obj, *Sec)), + *GraphSec, CharData, orc::ExecutorAddr(getSectionAddress(Obj, *Sec)), (*Sec)->getAlignment(), 0); } @@ -224,17 +227,7 @@ Error COFFLinkGraphBuilder::graphifySymbols() { << " (index: " << SectionIndex << ") \n"; }); else if (Sym->isUndefined()) { - LLVM_DEBUG({ - dbgs() << " " << SymIndex - << ": Creating external graph symbol for COFF symbol \"" - << SymbolName << "\" in " - << getCOFFSectionName(SectionIndex, Sec, *Sym) - << " (index: " << SectionIndex << ") \n"; - }); - if (!ExternalSymbols.count(SymbolName)) - ExternalSymbols[SymbolName] = - &G->addExternalSymbol(SymbolName, Sym->getValue(), Linkage::Strong); - GSym = ExternalSymbols[SymbolName]; + GSym = createExternalSymbol(SymIndex, SymbolName, *Sym, Sec); } else if (Sym->isWeakExternal()) { auto *WeakExternal = Sym->getAux<object::coff_aux_weak_external>(); COFFSymbolIndex TagIndex = WeakExternal->TagIndex; @@ -268,12 +261,51 @@ Error COFFLinkGraphBuilder::graphifySymbols() { if (auto Err = flushWeakAliasRequests()) return Err; + if (auto Err = handleAlternateNames()) + return Err; + if (auto Err = calculateImplicitSizeOfSymbols()) return Err; return Error::success(); } +Error COFFLinkGraphBuilder::handleDirectiveSection(StringRef Str) { + auto Parsed = DirectiveParser.parse(Str); + if (!Parsed) + return Parsed.takeError(); + for (auto *Arg : *Parsed) { + StringRef S = Arg->getValue(); + switch (Arg->getOption().getID()) { + case COFF_OPT_alternatename: { + StringRef From, To; + std::tie(From, To) = S.split('='); + if (From.empty() || To.empty()) + return make_error<JITLinkError>( + "Invalid COFF /alternatename directive"); + AlternateNames[From] = To; + break; + } + case COFF_OPT_incl: { + auto DataCopy = G->allocateString(S); + StringRef StrCopy(DataCopy.data(), DataCopy.size()); + ExternalSymbols[StrCopy] = &G->addExternalSymbol(StrCopy, 0, false); + ExternalSymbols[StrCopy]->setLive(true); + break; + } + case COFF_OPT_export: + break; + default: { + LLVM_DEBUG({ + dbgs() << "Unknown coff directive: " << Arg->getSpelling() << "\n"; + }); + break; + } + } + } + return Error::success(); +} + Error COFFLinkGraphBuilder::flushWeakAliasRequests() { // Export the weak external symbols and alias it for (auto &WeakExternal : WeakExternalRequests) { @@ -290,22 +322,18 @@ Error COFFLinkGraphBuilder::flushWeakAliasRequests() { ? Scope::Default : Scope::Local; - // FIXME: Support this when there's a way to handle this. - if (!Target->isDefined()) - return make_error<JITLinkError>("Weak external symbol with external " - "symbol as alternative not supported."); - - jitlink::Symbol *NewSymbol = &G->addDefinedSymbol( - Target->getBlock(), Target->getOffset(), WeakExternal.SymbolName, - Target->getSize(), Linkage::Weak, S, Target->isCallable(), false); + auto NewSymbol = + createAliasSymbol(WeakExternal.SymbolName, Linkage::Weak, S, *Target); + if (!NewSymbol) + return NewSymbol.takeError(); setGraphSymbol(AliasSymbol->getSectionNumber(), WeakExternal.Alias, - *NewSymbol); + **NewSymbol); LLVM_DEBUG({ dbgs() << " " << WeakExternal.Alias << ": Creating weak external symbol for COFF symbol \"" << WeakExternal.SymbolName << "\" in section " << AliasSymbol->getSectionNumber() << "\n"; - dbgs() << " " << *NewSymbol << "\n"; + dbgs() << " " << **NewSymbol << "\n"; }); } else return make_error<JITLinkError>("Weak symbol alias requested but actual " @@ -315,6 +343,48 @@ Error COFFLinkGraphBuilder::flushWeakAliasRequests() { return Error::success(); } +Error COFFLinkGraphBuilder::handleAlternateNames() { + for (auto &KeyValue : AlternateNames) + if (DefinedSymbols.count(KeyValue.second) && + ExternalSymbols.count(KeyValue.first)) { + auto *Target = DefinedSymbols[KeyValue.second]; + auto *Alias = ExternalSymbols[KeyValue.first]; + G->makeDefined(*Alias, Target->getBlock(), Target->getOffset(), + Target->getSize(), Linkage::Weak, Scope::Local, false); + } + return Error::success(); +} + +Symbol *COFFLinkGraphBuilder::createExternalSymbol( + COFFSymbolIndex SymIndex, StringRef SymbolName, + object::COFFSymbolRef Symbol, const object::coff_section *Section) { + if (!ExternalSymbols.count(SymbolName)) + ExternalSymbols[SymbolName] = + &G->addExternalSymbol(SymbolName, Symbol.getValue(), false); + + LLVM_DEBUG({ + dbgs() << " " << SymIndex + << ": Creating external graph symbol for COFF symbol \"" + << SymbolName << "\" in " + << getCOFFSectionName(Symbol.getSectionNumber(), Section, Symbol) + << " (index: " << Symbol.getSectionNumber() << ") \n"; + }); + return ExternalSymbols[SymbolName]; +} + +Expected<Symbol *> COFFLinkGraphBuilder::createAliasSymbol(StringRef SymbolName, + Linkage L, Scope S, + Symbol &Target) { + if (!Target.isDefined()) { + // FIXME: Support this when there's a way to handle this. + return make_error<JITLinkError>("Weak external symbol with external " + "symbol as alternative not supported."); + } + return &G->addDefinedSymbol(Target.getBlock(), Target.getOffset(), SymbolName, + Target.getSize(), L, S, Target.isCallable(), + false); +} + // In COFF, most of the defined symbols don't contain the size information. // Hence, we calculate the "implicit" size of symbol by taking the delta of // offsets of consecutive symbols within a block. We maintain a balanced tree @@ -384,9 +454,11 @@ Expected<Symbol *> COFFLinkGraphBuilder::createDefinedSymbol( object::COFFSymbolRef Symbol, const object::coff_section *Section) { if (Symbol.isCommon()) { // FIXME: correct alignment - return &G->addCommonSymbol(SymbolName, Scope::Default, getCommonSection(), - orc::ExecutorAddr(), Symbol.getValue(), - Symbol.getValue(), false); + return &G->addDefinedSymbol( + G->createZeroFillBlock(getCommonSection(), Symbol.getValue(), + orc::ExecutorAddr(), Symbol.getValue(), 0), + 0, SymbolName, Symbol.getValue(), Linkage::Strong, Scope::Default, + false, false); } if (Symbol.isAbsolute()) return &G->addAbsoluteSymbol(SymbolName, @@ -413,10 +485,11 @@ Expected<Symbol *> COFFLinkGraphBuilder::createDefinedSymbol( if (Symbol.isExternal()) { // This is not a comdat sequence, export the symbol as it is if (!isComdatSection(Section)) { - - return &G->addDefinedSymbol( + auto GSym = &G->addDefinedSymbol( *B, Symbol.getValue(), SymbolName, 0, Linkage::Strong, Scope::Default, Symbol.getComplexType() == COFF::IMAGE_SYM_DTYPE_FUNCTION, false); + DefinedSymbols[SymbolName] = GSym; + return GSym; } else { if (!PendingComdatExports[Symbol.getSectionNumber()]) return make_error<JITLinkError>("No pending COMDAT export for symbol " + @@ -472,7 +545,6 @@ Expected<Symbol *> COFFLinkGraphBuilder::createDefinedSymbol( Expected<Symbol *> COFFLinkGraphBuilder::createCOMDATExportRequest( COFFSymbolIndex SymIndex, object::COFFSymbolRef Symbol, const object::coff_aux_section_definition *Definition) { - Block *B = getGraphBlock(Symbol.getSectionNumber()); Linkage L = Linkage::Strong; switch (Definition->Selection) { case COFF::IMAGE_COMDAT_SELECT_NODUPLICATES: { @@ -497,7 +569,8 @@ Expected<Symbol *> COFFLinkGraphBuilder::createCOMDATExportRequest( dbgs() << " " << SymIndex << ": Partially supported IMAGE_COMDAT_SELECT_LARGEST was used" " in section " - << Symbol.getSectionNumber() << "\n"; + << Symbol.getSectionNumber() << " (size: " << Definition->Length + << ")\n"; }); L = Linkage::Weak; break; @@ -512,9 +585,9 @@ Expected<Symbol *> COFFLinkGraphBuilder::createCOMDATExportRequest( formatv("{0:d}", Definition->Selection)); } } - PendingComdatExports[Symbol.getSectionNumber()] = {SymIndex, L}; - return &G->addAnonymousSymbol(*B, Symbol.getValue(), Definition->Length, - false, false); + PendingComdatExports[Symbol.getSectionNumber()] = {SymIndex, L, + Definition->Length}; + return nullptr; } // Process the second symbol of COMDAT sequence. @@ -522,29 +595,26 @@ Expected<Symbol *> COFFLinkGraphBuilder::exportCOMDATSymbol(COFFSymbolIndex SymIndex, StringRef SymbolName, object::COFFSymbolRef Symbol) { + Block *B = getGraphBlock(Symbol.getSectionNumber()); auto &PendingComdatExport = PendingComdatExports[Symbol.getSectionNumber()]; - COFFSymbolIndex TargetIndex = PendingComdatExport->SymbolIndex; - Linkage L = PendingComdatExport->Linkage; - jitlink::Symbol *Target = getGraphSymbol(TargetIndex); - assert(Target && "COMDAT leaader is invalid."); - assert((llvm::count_if(G->defined_symbols(), - [&](const jitlink::Symbol *Sym) { - return Sym->getName() == SymbolName; - }) == 0) && - "Duplicate defined symbol"); - Target->setName(SymbolName); - Target->setLinkage(L); - Target->setCallable(Symbol.getComplexType() == - COFF::IMAGE_SYM_DTYPE_FUNCTION); - Target->setScope(Scope::Default); + // NOTE: ComdatDef->Legnth is the size of "section" not size of symbol. + // We use zero symbol size to not reach out of bound of block when symbol + // offset is non-zero. + auto GSym = &G->addDefinedSymbol( + *B, Symbol.getValue(), SymbolName, 0, PendingComdatExport->Linkage, + Scope::Default, Symbol.getComplexType() == COFF::IMAGE_SYM_DTYPE_FUNCTION, + false); LLVM_DEBUG({ dbgs() << " " << SymIndex << ": Exporting COMDAT graph symbol for COFF symbol \"" << SymbolName << "\" in section " << Symbol.getSectionNumber() << "\n"; - dbgs() << " " << *Target << "\n"; + dbgs() << " " << *GSym << "\n"; }); - PendingComdatExport = None; - return Target; + setGraphSymbol(Symbol.getSectionNumber(), PendingComdatExport->SymbolIndex, + *GSym); + DefinedSymbols[SymbolName] = GSym; + PendingComdatExport = std::nullopt; + return GSym; } } // namespace jitlink diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h index f925f6d7aeef..0c0a1a536deb 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFLinkGraphBuilder.h @@ -18,6 +18,7 @@ #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/Object/COFF.h" +#include "COFFDirectiveParser.h" #include "EHFrameSupportImpl.h" #include "JITLinkGeneric.h" @@ -112,8 +113,9 @@ private: struct ComdatExportRequest { COFFSymbolIndex SymbolIndex; jitlink::Linkage Linkage; + orc::ExecutorAddrDiff Size; }; - std::vector<Optional<ComdatExportRequest>> PendingComdatExports; + std::vector<std::optional<ComdatExportRequest>> PendingComdatExports; // This represents a pending request to create a weak external symbol with a // name. @@ -132,6 +134,11 @@ private: Section &getCommonSection(); + Symbol *createExternalSymbol(COFFSymbolIndex SymIndex, StringRef SymbolName, + object::COFFSymbolRef Symbol, + const object::coff_section *Section); + Expected<Symbol *> createAliasSymbol(StringRef SymbolName, Linkage L, Scope S, + Symbol &Target); Expected<Symbol *> createDefinedSymbol(COFFSymbolIndex SymIndex, StringRef SymbolName, object::COFFSymbolRef Symbol, @@ -142,7 +149,10 @@ private: Expected<Symbol *> exportCOMDATSymbol(COFFSymbolIndex SymIndex, StringRef SymbolName, object::COFFSymbolRef Symbol); + + Error handleDirectiveSection(StringRef Str); Error flushWeakAliasRequests(); + Error handleAlternateNames(); Error calculateImplicitSizeOfSymbols(); static uint64_t getSectionAddress(const object::COFFObjectFile &Obj, @@ -152,18 +162,23 @@ private: static bool isComdatSection(const object::coff_section *Section); static unsigned getPointerSize(const object::COFFObjectFile &Obj); static support::endianness getEndianness(const object::COFFObjectFile &Obj); + static StringRef getDLLImportStubPrefix() { return "__imp_"; } + static StringRef getDirectiveSectionName() { return ".drectve"; } StringRef getCOFFSectionName(COFFSectionIndex SectionIndex, const object::coff_section *Sec, object::COFFSymbolRef Sym); const object::COFFObjectFile &Obj; std::unique_ptr<LinkGraph> G; + COFFDirectiveParser DirectiveParser; Section *CommonSection = nullptr; std::vector<Block *> GraphBlocks; std::vector<Symbol *> GraphSymbols; + DenseMap<StringRef, StringRef> AlternateNames; DenseMap<StringRef, Symbol *> ExternalSymbols; + DenseMap<StringRef, Symbol *> DefinedSymbols; }; template <typename RelocHandlerFunction> diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFOptions.td b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFOptions.td new file mode 100644 index 000000000000..0a0ce2fc76dd --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFFOptions.td @@ -0,0 +1,21 @@ +include "llvm/Option/OptParser.td" + +// link.exe accepts options starting with either a dash or a slash. + +// Flag that takes no arguments. +class F<string name> : Flag<["/", "-", "/?", "-?"], name>; + +// Flag that takes one argument after ":". +class P<string name> : + Joined<["/", "-", "/?", "-?"], name#":">; + +// Boolean flag which can be suffixed by ":no". Using it unsuffixed turns the +// flag on and using it suffixed by ":no" turns it off. +multiclass B_priv<string name> { + def "" : F<name>; + def _no : F<name#":no">; +} + +def export : P<"export">; +def alternatename : P<"alternatename">; +def incl : Joined<["/", "-", "/?", "-?"], "include:">;
\ No newline at end of file diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp index e2040dc95acc..b09dc769b81c 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/COFF_x86_64.cpp @@ -29,6 +29,9 @@ namespace { enum EdgeKind_coff_x86_64 : Edge::Kind { PCRel32 = x86_64::FirstPlatformRelocation, Pointer32NB, + Pointer64, + SectionIdx16, + SecRel32, }; class COFFJITLinker_x86_64 : public JITLinker<COFFJITLinker_x86_64> { @@ -108,6 +111,57 @@ private: Addend -= 1; break; } + case COFF::RelocationTypeAMD64::IMAGE_REL_AMD64_REL32_2: { + Kind = EdgeKind_coff_x86_64::PCRel32; + Addend = *reinterpret_cast<const support::little32_t *>(FixupPtr); + Addend -= 2; + break; + } + case COFF::RelocationTypeAMD64::IMAGE_REL_AMD64_REL32_3: { + Kind = EdgeKind_coff_x86_64::PCRel32; + Addend = *reinterpret_cast<const support::little32_t *>(FixupPtr); + Addend -= 3; + break; + } + case COFF::RelocationTypeAMD64::IMAGE_REL_AMD64_REL32_4: { + Kind = EdgeKind_coff_x86_64::PCRel32; + Addend = *reinterpret_cast<const support::little32_t *>(FixupPtr); + Addend -= 4; + break; + } + case COFF::RelocationTypeAMD64::IMAGE_REL_AMD64_REL32_5: { + Kind = EdgeKind_coff_x86_64::PCRel32; + Addend = *reinterpret_cast<const support::little32_t *>(FixupPtr); + Addend -= 5; + break; + } + case COFF::RelocationTypeAMD64::IMAGE_REL_AMD64_ADDR64: { + Kind = EdgeKind_coff_x86_64::Pointer64; + Addend = *reinterpret_cast<const support::little64_t *>(FixupPtr); + break; + } + case COFF::RelocationTypeAMD64::IMAGE_REL_AMD64_SECTION: { + Kind = EdgeKind_coff_x86_64::SectionIdx16; + Addend = *reinterpret_cast<const support::little16_t *>(FixupPtr); + uint64_t SectionIdx = 0; + if (COFFSymbol.isAbsolute()) + SectionIdx = getObject().getNumberOfSections() + 1; + else + SectionIdx = COFFSymbol.getSectionNumber(); + auto *AbsSym = &getGraph().addAbsoluteSymbol( + "secidx", orc::ExecutorAddr(SectionIdx), 2, Linkage::Strong, + Scope::Local, false); + GraphSymbol = AbsSym; + break; + } + case COFF::RelocationTypeAMD64::IMAGE_REL_AMD64_SECREL: { + // FIXME: SECREL to external symbol should be handled + if (!GraphSymbol->isDefined()) + return Error::success(); + Kind = EdgeKind_coff_x86_64::SecRel32; + Addend = *reinterpret_cast<const support::little32_t *>(FixupPtr); + break; + } default: { return make_error<JITLinkError>("Unsupported x86_64 relocation:" + formatv("{0:d}", Rel.getType())); @@ -150,6 +204,21 @@ public: E.setKind(x86_64::PCRel32); break; } + case EdgeKind_coff_x86_64::Pointer64: { + E.setKind(x86_64::Pointer64); + break; + } + case EdgeKind_coff_x86_64::SectionIdx16: { + E.setKind(x86_64::Pointer16); + break; + } + case EdgeKind_coff_x86_64::SecRel32: { + E.setAddend(E.getAddend() - + getSectionStart(E.getTarget().getBlock().getSection()) + .getValue()); + E.setKind(x86_64::Pointer32); + break; + } default: break; } @@ -160,6 +229,15 @@ public: private: static StringRef getImageBaseSymbolName() { return "__ImageBase"; } + + orc::ExecutorAddr getSectionStart(Section &Sec) { + if (!SectionStartCache.count(&Sec)) { + SectionRange Range(Sec); + SectionStartCache[&Sec] = Range.getStart(); + } + return SectionStartCache[&Sec]; + } + Expected<JITTargetAddress> getImageBaseAddress(LinkGraph &G, JITLinkContext &Ctx) { if (this->ImageBase) @@ -189,6 +267,8 @@ private: this->ImageBase = ImageBase; return ImageBase; } + + DenseMap<Section *, orc::ExecutorAddr> SectionStartCache; JITTargetAddress ImageBase = 0; }; @@ -213,6 +293,12 @@ const char *getCOFFX86RelocationKindName(Edge::Kind R) { return "PCRel32"; case Pointer32NB: return "Pointer32NB"; + case Pointer64: + return "Pointer64"; + case SectionIdx16: + return "SectionIdx16"; + case SecRel32: + return "SecRel32"; default: return x86_64::getEdgeKindName(R); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index 389fd14c0f29..86249591a9be 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -85,6 +85,32 @@ Error EHFrameEdgeFixer::operator()(LinkGraph &G) { return Error::success(); } +static Expected<size_t> readCFIRecordLength(const Block &B, + BinaryStreamReader &R) { + uint32_t Length; + if (auto Err = R.readInteger(Length)) + return std::move(Err); + + // If Length < 0xffffffff then use the regular length field, otherwise + // read the extended length field. + if (Length != 0xffffffff) + return Length; + + uint64_t ExtendedLength; + if (auto Err = R.readInteger(ExtendedLength)) + return std::move(Err); + + if (ExtendedLength > std::numeric_limits<size_t>::max()) + return make_error<JITLinkError>( + "In CFI record at " + + formatv("{0:x}", B.getAddress() + R.getOffset() - 12) + + ", extended length of " + formatv("{0:x}", ExtendedLength) + + " exceeds address-range max (" + + formatv("{0:x}", std::numeric_limits<size_t>::max())); + + return ExtendedLength; +} + Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { LLVM_DEBUG(dbgs() << " Processing block at " << B.getAddress() << "\n"); @@ -125,24 +151,11 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { }); // Get the record length. - size_t RecordRemaining; - { - uint32_t Length; - if (auto Err = BlockReader.readInteger(Length)) - return Err; - // If Length < 0xffffffff then use the regular length field, otherwise - // read the extended length field. - if (Length != 0xffffffff) - RecordRemaining = Length; - else { - uint64_t ExtendedLength; - if (auto Err = BlockReader.readInteger(ExtendedLength)) - return Err; - RecordRemaining = ExtendedLength; - } - } + Expected<size_t> RecordRemaining = readCFIRecordLength(B, BlockReader); + if (!RecordRemaining) + return RecordRemaining.takeError(); - if (BlockReader.bytesRemaining() < RecordRemaining) + if (BlockReader.bytesRemaining() < *RecordRemaining) return make_error<JITLinkError>( "Incomplete CFI record at " + formatv("{0:x16}", B.getAddress() + RecordStartOffset)); @@ -155,19 +168,19 @@ Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { if (CIEDelta == 0) { if (auto Err = processCIE(PC, B, RecordStartOffset, - CIEDeltaFieldOffset + RecordRemaining, + CIEDeltaFieldOffset + *RecordRemaining, CIEDeltaFieldOffset, BlockEdges)) return Err; } else { if (auto Err = processFDE(PC, B, RecordStartOffset, - CIEDeltaFieldOffset + RecordRemaining, + CIEDeltaFieldOffset + *RecordRemaining, CIEDeltaFieldOffset, CIEDelta, BlockEdges)) return Err; } // Move to the next record. BlockReader.setOffset(RecordStartOffset + CIEDeltaFieldOffset + - RecordRemaining); + *RecordRemaining); } return Error::success(); @@ -358,14 +371,25 @@ Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, PC, BlockEdges, CIEInfo->AddressEncoding, RecordReader, B, RecordReader.getOffset(), "PC begin")) { assert(*PCBegin && "PC-begin symbol not set"); - // Add a keep-alive edge from the FDE target to the FDE to ensure that the - // FDE is kept alive if its target is. - LLVM_DEBUG({ - dbgs() << " Adding keep-alive edge from target at " - << (*PCBegin)->getBlock().getAddress() << " to FDE at " - << RecordAddress << "\n"; - }); - (*PCBegin)->getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + if ((*PCBegin)->isDefined()) { + // Add a keep-alive edge from the FDE target to the FDE to ensure that the + // FDE is kept alive if its target is. + LLVM_DEBUG({ + dbgs() << " Adding keep-alive edge from target at " + << (*PCBegin)->getBlock().getAddress() << " to FDE at " + << RecordAddress << "\n"; + }); + (*PCBegin)->getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + } else { + LLVM_DEBUG({ + dbgs() << " WARNING: Not adding keep-alive edge to FDE at " + << RecordAddress << ", which points to " + << ((*PCBegin)->isExternal() ? "external" : "absolute") + << " symbol \"" << (*PCBegin)->getName() + << "\" -- FDE must be kept alive manually or it will be " + << "dead stripped.\n"; + }); + } } else return PCBegin.takeError(); @@ -639,6 +663,31 @@ Error InProcessEHFrameRegistrar::deregisterEHFrames( EHFrameSection.size()); } +EHFrameCFIBlockInspector EHFrameCFIBlockInspector::FromEdgeScan(Block &B) { + if (B.edges_empty()) + return EHFrameCFIBlockInspector(nullptr); + if (B.edges_size() == 1) + return EHFrameCFIBlockInspector(&*B.edges().begin()); + SmallVector<Edge *, 3> Es; + for (auto &E : B.edges()) + Es.push_back(&E); + assert(Es.size() >= 2 && Es.size() <= 3 && "Unexpected number of edges"); + llvm::sort(Es, [](const Edge *LHS, const Edge *RHS) { + return LHS->getOffset() < RHS->getOffset(); + }); + return EHFrameCFIBlockInspector(*Es[0], *Es[1], + Es.size() == 3 ? Es[2] : nullptr); + return EHFrameCFIBlockInspector(nullptr); +} + +EHFrameCFIBlockInspector::EHFrameCFIBlockInspector(Edge *PersonalityEdge) + : PersonalityEdge(PersonalityEdge) {} + +EHFrameCFIBlockInspector::EHFrameCFIBlockInspector(Edge &CIEEdge, + Edge &PCBeginEdge, + Edge *LSDAEdge) + : CIEEdge(&CIEEdge), PCBeginEdge(&PCBeginEdge), LSDAEdge(LSDAEdge) {} + LinkGraphPassFunction createEHFrameRecorderPass(const Triple &TT, StoreFrameRangeFunction StoreRangeAddress) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp index eb98e4ba4041..ef0f19a78571 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF.cpp @@ -14,6 +14,8 @@ #include "llvm/BinaryFormat/ELF.h" #include "llvm/ExecutionEngine/JITLink/ELF_aarch64.h" +#include "llvm/ExecutionEngine/JITLink/ELF_i386.h" +#include "llvm/ExecutionEngine/JITLink/ELF_loongarch.h" #include "llvm/ExecutionEngine/JITLink/ELF_riscv.h" #include "llvm/ExecutionEngine/JITLink/ELF_x86_64.h" #include "llvm/Object/ELF.h" @@ -67,10 +69,14 @@ createLinkGraphFromELFObject(MemoryBufferRef ObjectBuffer) { switch (*TargetMachineArch) { case ELF::EM_AARCH64: return createLinkGraphFromELFObject_aarch64(ObjectBuffer); + case ELF::EM_LOONGARCH: + return createLinkGraphFromELFObject_loongarch(ObjectBuffer); case ELF::EM_RISCV: return createLinkGraphFromELFObject_riscv(ObjectBuffer); case ELF::EM_X86_64: return createLinkGraphFromELFObject_x86_64(ObjectBuffer); + case ELF::EM_386: + return createLinkGraphFromELFObject_i386(ObjectBuffer); default: return make_error<JITLinkError>( "Unsupported target machine architecture in ELF object " + @@ -84,6 +90,10 @@ void link_ELF(std::unique_ptr<LinkGraph> G, case Triple::aarch64: link_ELF_aarch64(std::move(G), std::move(Ctx)); return; + case Triple::loongarch32: + case Triple::loongarch64: + link_ELF_loongarch(std::move(G), std::move(Ctx)); + return; case Triple::riscv32: case Triple::riscv64: link_ELF_riscv(std::move(G), std::move(Ctx)); @@ -91,6 +101,9 @@ void link_ELF(std::unique_ptr<LinkGraph> G, case Triple::x86_64: link_ELF_x86_64(std::move(G), std::move(Ctx)); return; + case Triple::x86: + link_ELF_i386(std::move(G), std::move(Ctx)); + return; default: Ctx->notifyFailed(make_error<JITLinkError>( "Unsupported target machine architecture in ELF link graph " + diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h index 2ab7ed61f71b..953a9f512784 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELFLinkGraphBuilder.h @@ -37,8 +37,8 @@ protected: Section &getCommonSection() { if (!CommonSection) - CommonSection = - &G->createSection(CommonSectionName, MemProt::Read | MemProt::Write); + CommonSection = &G->createSection( + CommonSectionName, orc::MemProt::Read | orc::MemProt::Write); return *CommonSection; } @@ -108,24 +108,49 @@ protected: Error graphifySections(); Error graphifySymbols(); - /// Traverse all matching relocation records in the given section. The handler - /// function Func should be callable with this signature: + /// Traverse all matching ELFT::Rela relocation records in the given section. + /// The handler function Func should be callable with this signature: /// Error(const typename ELFT::Rela &, /// const typename ELFT::Shdr &, Section &) /// - template <typename RelocHandlerFunction> - Error forEachRelocation(const typename ELFT::Shdr &RelSect, - RelocHandlerFunction &&Func, - bool ProcessDebugSections = false); + template <typename RelocHandlerMethod> + Error forEachRelaRelocation(const typename ELFT::Shdr &RelSect, + RelocHandlerMethod &&Func, + bool ProcessDebugSections = false); + + /// Traverse all matching ELFT::Rel relocation records in the given section. + /// The handler function Func should be callable with this signature: + /// Error(const typename ELFT::Rel &, + /// const typename ELFT::Shdr &, Section &) + /// + template <typename RelocHandlerMethod> + Error forEachRelRelocation(const typename ELFT::Shdr &RelSect, + RelocHandlerMethod &&Func, + bool ProcessDebugSections = false); + + /// Traverse all matching rela relocation records in the given section. + /// Convenience wrapper to allow passing a member function for the handler. + /// + template <typename ClassT, typename RelocHandlerMethod> + Error forEachRelaRelocation(const typename ELFT::Shdr &RelSect, + ClassT *Instance, RelocHandlerMethod &&Method, + bool ProcessDebugSections = false) { + return forEachRelaRelocation( + RelSect, + [Instance, Method](const auto &Rel, const auto &Target, auto &GS) { + return (Instance->*Method)(Rel, Target, GS); + }, + ProcessDebugSections); + } - /// Traverse all matching relocation records in the given section. Convenience - /// wrapper to allow passing a member function for the handler. + /// Traverse all matching rel relocation records in the given section. + /// Convenience wrapper to allow passing a member function for the handler. /// template <typename ClassT, typename RelocHandlerMethod> - Error forEachRelocation(const typename ELFT::Shdr &RelSect, ClassT *Instance, - RelocHandlerMethod &&Method, - bool ProcessDebugSections = false) { - return forEachRelocation( + Error forEachRelRelocation(const typename ELFT::Shdr &RelSect, + ClassT *Instance, RelocHandlerMethod &&Method, + bool ProcessDebugSections = false) { + return forEachRelRelocation( RelSect, [Instance, Method](const auto &Rel, const auto &Target, auto &GS) { return (Instance->*Method)(Rel, Target, GS); @@ -310,11 +335,11 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySections() { }); // Get the section's memory protection flags. - MemProt Prot; + orc::MemProt Prot; if (Sec.sh_flags & ELF::SHF_EXECINSTR) - Prot = MemProt::Read | MemProt::Exec; + Prot = orc::MemProt::Read | orc::MemProt::Exec; else - Prot = MemProt::Read | MemProt::Write; + Prot = orc::MemProt::Read | orc::MemProt::Write; // Look for existing sections first. auto *GraphSec = G->findSectionByName(*Name); @@ -402,26 +427,27 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { // Handle common symbols specially. if (Sym.isCommon()) { - Symbol &GSym = G->addCommonSymbol(*Name, Scope::Default, - getCommonSection(), orc::ExecutorAddr(), - Sym.st_size, Sym.getValue(), false); + Symbol &GSym = G->addDefinedSymbol( + G->createZeroFillBlock(getCommonSection(), Sym.st_size, + orc::ExecutorAddr(), Sym.getValue(), 0), + 0, *Name, Sym.st_size, Linkage::Strong, Scope::Default, false, false); setGraphSymbol(SymIndex, GSym); continue; } - // Map Visibility and Binding to Scope and Linkage: - Linkage L; - Scope S; - - if (auto LSOrErr = getSymbolLinkageAndScope(Sym, *Name)) - std::tie(L, S) = *LSOrErr; - else - return LSOrErr.takeError(); - if (Sym.isDefined() && (Sym.getType() == ELF::STT_NOTYPE || Sym.getType() == ELF::STT_FUNC || Sym.getType() == ELF::STT_OBJECT || Sym.getType() == ELF::STT_SECTION || Sym.getType() == ELF::STT_TLS)) { + + // Map Visibility and Binding to Scope and Linkage: + Linkage L; + Scope S; + if (auto LSOrErr = getSymbolLinkageAndScope(Sym, *Name)) + std::tie(L, S) = *LSOrErr; + else + return LSOrErr.takeError(); + // Handle extended tables. unsigned Shndx = Sym.st_shndx; if (Shndx == ELF::SHN_XINDEX) { @@ -460,7 +486,18 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { << ": Creating external graph symbol for ELF symbol \"" << *Name << "\"\n"; }); - auto &GSym = G->addExternalSymbol(*Name, Sym.st_size, L); + + if (Sym.getBinding() != ELF::STB_GLOBAL && + Sym.getBinding() != ELF::STB_WEAK) + return make_error<StringError>( + "Invalid symbol binding " + + Twine(static_cast<int>(Sym.getBinding())) + + " for external symbol " + *Name, + inconvertibleErrorCode()); + + // If L is Linkage::Weak that means this is a weakly referenced symbol. + auto &GSym = G->addExternalSymbol(*Name, Sym.st_size, + Sym.getBinding() == ELF::STB_WEAK); setGraphSymbol(SymIndex, GSym); } else { LLVM_DEBUG({ @@ -476,12 +513,11 @@ template <typename ELFT> Error ELFLinkGraphBuilder<ELFT>::graphifySymbols() { template <typename ELFT> template <typename RelocHandlerFunction> -Error ELFLinkGraphBuilder<ELFT>::forEachRelocation( +Error ELFLinkGraphBuilder<ELFT>::forEachRelaRelocation( const typename ELFT::Shdr &RelSect, RelocHandlerFunction &&Func, bool ProcessDebugSections) { - // Only look into sections that store relocation entries. - if (RelSect.sh_type != ELF::SHT_RELA && RelSect.sh_type != ELF::SHT_REL) + if (RelSect.sh_type != ELF::SHT_RELA) return Error::success(); // sh_info contains the section header index of the target (FixupSection), @@ -522,6 +558,53 @@ Error ELFLinkGraphBuilder<ELFT>::forEachRelocation( return Error::success(); } +template <typename ELFT> +template <typename RelocHandlerFunction> +Error ELFLinkGraphBuilder<ELFT>::forEachRelRelocation( + const typename ELFT::Shdr &RelSect, RelocHandlerFunction &&Func, + bool ProcessDebugSections) { + // Only look into sections that store relocation entries. + if (RelSect.sh_type != ELF::SHT_REL) + return Error::success(); + + // sh_info contains the section header index of the target (FixupSection), + // which is the section to which all relocations in RelSect apply. + auto FixupSection = Obj.getSection(RelSect.sh_info); + if (!FixupSection) + return FixupSection.takeError(); + + // Target sections have names in valid ELF object files. + Expected<StringRef> Name = Obj.getSectionName(**FixupSection); + if (!Name) + return Name.takeError(); + LLVM_DEBUG(dbgs() << " " << *Name << ":\n"); + + // Consider skipping these relocations. + if (!ProcessDebugSections && isDwarfSection(*Name)) { + LLVM_DEBUG(dbgs() << " skipped (dwarf section)\n\n"); + return Error::success(); + } + + // Lookup the link-graph node corresponding to the target section name. + auto *BlockToFix = getGraphBlock(RelSect.sh_info); + if (!BlockToFix) + return make_error<StringError>( + "Refencing a section that wasn't added to the graph: " + *Name, + inconvertibleErrorCode()); + + auto RelEntries = Obj.rels(RelSect); + if (!RelEntries) + return RelEntries.takeError(); + + // Let the callee process relocation entries one by one. + for (const typename ELFT::Rel &R : *RelEntries) + if (Error Err = Func(R, **FixupSection, *BlockToFix)) + return Err; + + LLVM_DEBUG(dbgs() << "\n"); + return Error::success(); +} + } // end namespace jitlink } // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp index 7d67e5ef343a..567d5a4dd47a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_aarch64.cpp @@ -129,8 +129,8 @@ private: using Base = ELFLinkGraphBuilder<ELFT>; using Self = ELFLinkGraphBuilder_aarch64<ELFT>; for (const auto &RelSect : Base::Sections) - if (Error Err = Base::forEachRelocation(RelSect, this, - &Self::addSingleRelocation)) + if (Error Err = Base::forEachRelaRelocation(RelSect, this, + &Self::addSingleRelocation)) return Err; return Error::success(); @@ -174,7 +174,7 @@ private: switch (*RelocKind) { case ELFCall26: { - Kind = aarch64::Branch26; + Kind = aarch64::Branch26PCRel; break; } case ELFAdrPage21: { @@ -297,23 +297,20 @@ private: break; } case ELFAdrGOTPage21: { - Kind = aarch64::GOTPage21; + Kind = aarch64::RequestGOTAndTransformToPage21; break; } case ELFLd64GOTLo12: { - Kind = aarch64::GOTPageOffset12; + Kind = aarch64::RequestGOTAndTransformToPageOffset12; break; } case ELFTLSDescAdrPage21: { - Kind = aarch64::TLSDescPage21; - break; - } - case ELFTLSDescAddLo12: { - Kind = aarch64::TLSDescPageOffset12; + Kind = aarch64::RequestTLSDescEntryAndTransformToPage21; break; } + case ELFTLSDescAddLo12: case ELFTLSDescLd64Lo12: { - Kind = aarch64::TLSDescPageOffset12; + Kind = aarch64::RequestTLSDescEntryAndTransformToPageOffset12; break; } case ELFTLSDescCall: { @@ -413,7 +410,7 @@ public: private: Section &getTLSInfoSection(LinkGraph &G) { if (!TLSInfoTable) - TLSInfoTable = &G.createSection(getSectionName(), MemProt::Read); + TLSInfoTable = &G.createSection(getSectionName(), orc::MemProt::Read); return *TLSInfoTable; } @@ -445,11 +442,11 @@ public: bool visitEdge(LinkGraph &G, Block *B, Edge &E) { Edge::Kind KindToSet = Edge::Invalid; switch (E.getKind()) { - case aarch64::TLSDescPage21: { + case aarch64::RequestTLSDescEntryAndTransformToPage21: { KindToSet = aarch64::Page21; break; } - case aarch64::TLSDescPageOffset12: { + case aarch64::RequestTLSDescEntryAndTransformToPageOffset12: { KindToSet = aarch64::PageOffset12; break; } @@ -481,14 +478,13 @@ public: private: Section &getTLSDescSection(LinkGraph &G) { if (!GOTSection) - GOTSection = &G.createSection(getSectionName(), MemProt::Read); + GOTSection = &G.createSection(getSectionName(), orc::MemProt::Read); return *GOTSection; } Symbol &getTLSDescResolver(LinkGraph &G) { if (!TLSDescResolver) - TLSDescResolver = - &G.addExternalSymbol("__tlsdesc_resolver", 8, Linkage::Strong); + TLSDescResolver = &G.addExternalSymbol("__tlsdesc_resolver", 8, false); return *TLSDescResolver; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp new file mode 100644 index 000000000000..1fee1b24b6bd --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_i386.cpp @@ -0,0 +1,255 @@ +//===----- ELF_i386.cpp - JIT linker implementation for ELF/i386 ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ELF/i386 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ELF_i386.h" +#include "DefineExternalSectionStartAndEndSymbols.h" +#include "ELFLinkGraphBuilder.h" +#include "JITLinkGeneric.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/i386.h" +#include "llvm/Object/ELFObjectFile.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; + +namespace { +constexpr StringRef ELFGOTSymbolName = "_GLOBAL_OFFSET_TABLE_"; + +Error buildTables_ELF_i386(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); + + i386::GOTTableManager GOT; + visitExistingEdges(G, GOT); + return Error::success(); +} +} // namespace + +namespace llvm::jitlink { + +class ELFJITLinker_i386 : public JITLinker<ELFJITLinker_i386> { + friend class JITLinker<ELFJITLinker_i386>; + +public: + ELFJITLinker_i386(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) { + getPassConfig().PostAllocationPasses.push_back( + [this](LinkGraph &G) { return getOrCreateGOTSymbol(G); }); + } + +private: + Symbol *GOTSymbol = nullptr; + + Error getOrCreateGOTSymbol(LinkGraph &G) { + auto DefineExternalGOTSymbolIfPresent = + createDefineExternalSectionStartAndEndSymbolsPass( + [&](LinkGraph &LG, Symbol &Sym) -> SectionRangeSymbolDesc { + if (Sym.getName() == ELFGOTSymbolName) + if (auto *GOTSection = G.findSectionByName( + i386::GOTTableManager::getSectionName())) { + GOTSymbol = &Sym; + return {*GOTSection, true}; + } + return {}; + }); + + // Try to attach _GLOBAL_OFFSET_TABLE_ to the GOT if it's defined as an + // external. + if (auto Err = DefineExternalGOTSymbolIfPresent(G)) + return Err; + + // If we succeeded then we're done. + if (GOTSymbol) + return Error::success(); + + // Otherwise look for a GOT section: If it already has a start symbol we'll + // record it, otherwise we'll create our own. + // If there's a GOT section but we didn't find an external GOT symbol... + if (auto *GOTSection = + G.findSectionByName(i386::GOTTableManager::getSectionName())) { + + // Check for an existing defined symbol. + for (auto *Sym : GOTSection->symbols()) + if (Sym->getName() == ELFGOTSymbolName) { + GOTSymbol = Sym; + return Error::success(); + } + + // If there's no defined symbol then create one. + SectionRange SR(*GOTSection); + + if (SR.empty()) { + GOTSymbol = + &G.addAbsoluteSymbol(ELFGOTSymbolName, orc::ExecutorAddr(), 0, + Linkage::Strong, Scope::Local, true); + } else { + GOTSymbol = + &G.addDefinedSymbol(*SR.getFirstBlock(), 0, ELFGOTSymbolName, 0, + Linkage::Strong, Scope::Local, false, true); + } + } + + return Error::success(); + } + + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { + return i386::applyFixup(G, B, E, GOTSymbol); + } +}; + +template <typename ELFT> +class ELFLinkGraphBuilder_i386 : public ELFLinkGraphBuilder<ELFT> { +private: + static Expected<i386::EdgeKind_i386> getRelocationKind(const uint32_t Type) { + using namespace i386; + switch (Type) { + case ELF::R_386_NONE: + return EdgeKind_i386::None; + case ELF::R_386_32: + return EdgeKind_i386::Pointer32; + case ELF::R_386_PC32: + return EdgeKind_i386::PCRel32; + case ELF::R_386_16: + return EdgeKind_i386::Pointer16; + case ELF::R_386_PC16: + return EdgeKind_i386::PCRel16; + case ELF::R_386_GOT32: + return EdgeKind_i386::RequestGOTAndTransformToDelta32FromGOT; + case ELF::R_386_GOTPC: + return EdgeKind_i386::Delta32; + case ELF::R_386_GOTOFF: + return EdgeKind_i386::Delta32FromGOT; + } + + return make_error<JITLinkError>("Unsupported i386 relocation:" + + formatv("{0:d}", Type)); + } + + Error addRelocations() override { + LLVM_DEBUG(dbgs() << "Adding relocations\n"); + using Base = ELFLinkGraphBuilder<ELFT>; + using Self = ELFLinkGraphBuilder_i386; + + for (const auto &RelSect : Base::Sections) { + // Validate the section to read relocation entries from. + if (RelSect.sh_type == ELF::SHT_RELA) + return make_error<StringError>( + "No SHT_RELA in valid i386 ELF object files", + inconvertibleErrorCode()); + + if (Error Err = Base::forEachRelRelocation(RelSect, this, + &Self::addSingleRelocation)) + return Err; + } + + return Error::success(); + } + + Error addSingleRelocation(const typename ELFT::Rel &Rel, + const typename ELFT::Shdr &FixupSection, + Block &BlockToFix) { + using Base = ELFLinkGraphBuilder<ELFT>; + + uint32_t SymbolIndex = Rel.getSymbol(false); + auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); + if (!ObjSymbol) + return ObjSymbol.takeError(); + + Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); + if (!GraphSymbol) + return make_error<StringError>( + formatv("Could not find symbol at given index, did you add it to " + "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", + SymbolIndex, (*ObjSymbol)->st_shndx, + Base::GraphSymbols.size()), + inconvertibleErrorCode()); + + Expected<i386::EdgeKind_i386> Kind = getRelocationKind(Rel.getType(false)); + if (!Kind) + return Kind.takeError(); + + auto FixupAddress = orc::ExecutorAddr(FixupSection.sh_addr) + Rel.r_offset; + int64_t Addend = 0; + + switch (*Kind) { + case i386::EdgeKind_i386::Delta32: { + const char *FixupContent = BlockToFix.getContent().data() + + (FixupAddress - BlockToFix.getAddress()); + Addend = *(const support::ulittle32_t *)FixupContent; + break; + } + default: + break; + } + + Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); + Edge GE(*Kind, Offset, *GraphSymbol, Addend); + LLVM_DEBUG({ + dbgs() << " "; + printEdge(dbgs(), BlockToFix, GE, i386::getEdgeKindName(*Kind)); + dbgs() << "\n"; + }); + + BlockToFix.addEdge(std::move(GE)); + return Error::success(); + } + +public: + ELFLinkGraphBuilder_i386(StringRef FileName, const object::ELFFile<ELFT> &Obj, + const Triple T) + : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, + i386::getEdgeKindName) {} +}; + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_i386(MemoryBufferRef ObjectBuffer) { + LLVM_DEBUG({ + dbgs() << "Building jitlink graph for new input " + << ObjectBuffer.getBufferIdentifier() << "...\n"; + }); + + auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); + if (!ELFObj) + return ELFObj.takeError(); + + assert((*ELFObj)->getArch() == Triple::x86 && + "Only i386 (little endian) is supported for now"); + + auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj); + return ELFLinkGraphBuilder_i386<object::ELF32LE>((*ELFObj)->getFileName(), + ELFObjFile.getELFFile(), + (*ELFObj)->makeTriple()) + .buildGraph(); +} + +void link_ELF_i386(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; + const Triple &TT = G->getTargetTriple(); + if (Ctx->shouldAddDefaultTargetPasses(TT)) { + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + + // Add an in-place GOT build pass. + Config.PostPrunePasses.push_back(buildTables_ELF_i386); + } + if (auto Err = Ctx->modifyPassConfig(*G, Config)) + return Ctx->notifyFailed(std::move(Err)); + + ELFJITLinker_i386::link(std::move(Ctx), std::move(G), std::move(Config)); +} + +} // namespace llvm::jitlink diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp new file mode 100644 index 000000000000..cd70217b4c0a --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_loongarch.cpp @@ -0,0 +1,209 @@ +//===--- ELF_loongarch.cpp - JIT linker implementation for ELF/loongarch --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// ELF/loongarch jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/ELF_loongarch.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/ExecutionEngine/JITLink/DWARFRecordSectionSplitter.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITLink/loongarch.h" +#include "llvm/Object/ELF.h" +#include "llvm/Object/ELFObjectFile.h" + +#include "EHFrameSupportImpl.h" +#include "ELFLinkGraphBuilder.h" +#include "JITLinkGeneric.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::loongarch; + +namespace { + +class ELFJITLinker_loongarch : public JITLinker<ELFJITLinker_loongarch> { + friend class JITLinker<ELFJITLinker_loongarch>; + +public: + ELFJITLinker_loongarch(std::unique_ptr<JITLinkContext> Ctx, + std::unique_ptr<LinkGraph> G, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(G), std::move(PassConfig)) {} + +private: + Error applyFixup(LinkGraph &G, Block &B, const Edge &E) const { + return loongarch::applyFixup(G, B, E); + } +}; + +template <typename ELFT> +class ELFLinkGraphBuilder_loongarch : public ELFLinkGraphBuilder<ELFT> { +private: + static Expected<loongarch::EdgeKind_loongarch> + getRelocationKind(const uint32_t Type) { + using namespace loongarch; + switch (Type) { + case ELF::R_LARCH_64: + return Pointer64; + case ELF::R_LARCH_32: + return Pointer32; + case ELF::R_LARCH_32_PCREL: + return Delta32; + case ELF::R_LARCH_B26: + return Branch26PCRel; + case ELF::R_LARCH_PCALA_HI20: + return Page20; + case ELF::R_LARCH_PCALA_LO12: + return PageOffset12; + case ELF::R_LARCH_GOT_PC_HI20: + return RequestGOTAndTransformToPage20; + case ELF::R_LARCH_GOT_PC_LO12: + return RequestGOTAndTransformToPageOffset12; + } + + return make_error<JITLinkError>( + "Unsupported loongarch relocation:" + formatv("{0:d}: ", Type) + + object::getELFRelocationTypeName(ELF::EM_LOONGARCH, Type)); + } + + Error addRelocations() override { + LLVM_DEBUG(dbgs() << "Processing relocations:\n"); + + using Base = ELFLinkGraphBuilder<ELFT>; + using Self = ELFLinkGraphBuilder_loongarch<ELFT>; + for (const auto &RelSect : Base::Sections) + if (Error Err = Base::forEachRelaRelocation(RelSect, this, + &Self::addSingleRelocation)) + return Err; + + return Error::success(); + } + + Error addSingleRelocation(const typename ELFT::Rela &Rel, + const typename ELFT::Shdr &FixupSect, + Block &BlockToFix) { + using Base = ELFLinkGraphBuilder<ELFT>; + + uint32_t SymbolIndex = Rel.getSymbol(false); + auto ObjSymbol = Base::Obj.getRelocationSymbol(Rel, Base::SymTabSec); + if (!ObjSymbol) + return ObjSymbol.takeError(); + + Symbol *GraphSymbol = Base::getGraphSymbol(SymbolIndex); + if (!GraphSymbol) + return make_error<StringError>( + formatv("Could not find symbol at given index, did you add it to " + "JITSymbolTable? index: {0}, shndx: {1} Size of table: {2}", + SymbolIndex, (*ObjSymbol)->st_shndx, + Base::GraphSymbols.size()), + inconvertibleErrorCode()); + + uint32_t Type = Rel.getType(false); + Expected<loongarch::EdgeKind_loongarch> Kind = getRelocationKind(Type); + if (!Kind) + return Kind.takeError(); + + int64_t Addend = Rel.r_addend; + auto FixupAddress = orc::ExecutorAddr(FixupSect.sh_addr) + Rel.r_offset; + Edge::OffsetT Offset = FixupAddress - BlockToFix.getAddress(); + Edge GE(*Kind, Offset, *GraphSymbol, Addend); + LLVM_DEBUG({ + dbgs() << " "; + printEdge(dbgs(), BlockToFix, GE, loongarch::getEdgeKindName(*Kind)); + dbgs() << "\n"; + }); + + BlockToFix.addEdge(std::move(GE)); + + return Error::success(); + } + +public: + ELFLinkGraphBuilder_loongarch(StringRef FileName, + const object::ELFFile<ELFT> &Obj, + const Triple T) + : ELFLinkGraphBuilder<ELFT>(Obj, std::move(T), FileName, + loongarch::getEdgeKindName) {} +}; + +Error buildTables_ELF_loongarch(LinkGraph &G) { + LLVM_DEBUG(dbgs() << "Visiting edges in graph:\n"); + + GOTTableManager GOT; + PLTTableManager PLT(GOT); + visitExistingEdges(G, GOT, PLT); + return Error::success(); +} + +} // namespace + +namespace llvm { +namespace jitlink { + +Expected<std::unique_ptr<LinkGraph>> +createLinkGraphFromELFObject_loongarch(MemoryBufferRef ObjectBuffer) { + LLVM_DEBUG({ + dbgs() << "Building jitlink graph for new input " + << ObjectBuffer.getBufferIdentifier() << "...\n"; + }); + + auto ELFObj = object::ObjectFile::createELFObjectFile(ObjectBuffer); + if (!ELFObj) + return ELFObj.takeError(); + + if ((*ELFObj)->getArch() == Triple::loongarch64) { + auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF64LE>>(**ELFObj); + return ELFLinkGraphBuilder_loongarch<object::ELF64LE>( + (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), + (*ELFObj)->makeTriple()) + .buildGraph(); + } + + assert((*ELFObj)->getArch() == Triple::loongarch32 && + "Invalid triple for LoongArch ELF object file"); + auto &ELFObjFile = cast<object::ELFObjectFile<object::ELF32LE>>(**ELFObj); + return ELFLinkGraphBuilder_loongarch<object::ELF32LE>( + (*ELFObj)->getFileName(), ELFObjFile.getELFFile(), + (*ELFObj)->makeTriple()) + .buildGraph(); +} + +void link_ELF_loongarch(std::unique_ptr<LinkGraph> G, + std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; + const Triple &TT = G->getTargetTriple(); + if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add eh-frame passses. + Config.PrePrunePasses.push_back(DWARFRecordSectionSplitter(".eh_frame")); + Config.PrePrunePasses.push_back( + EHFrameEdgeFixer(".eh_frame", G->getPointerSize(), Pointer32, Pointer64, + Delta32, Delta64, NegDelta32)); + Config.PrePrunePasses.push_back(EHFrameNullTerminator(".eh_frame")); + + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllSymbolsLive); + + // Add an in-place GOT/PLTStubs build pass. + Config.PostPrunePasses.push_back(buildTables_ELF_loongarch); + } + + if (auto Err = Ctx->modifyPassConfig(*G, Config)) + return Ctx->notifyFailed(std::move(Err)); + + ELFJITLinker_loongarch::link(std::move(Ctx), std::move(G), std::move(Config)); +} + +} // namespace jitlink +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp index c7596efe2bb8..90d3bbe6a276 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_riscv.cpp @@ -82,14 +82,14 @@ public: private: Section &getGOTSection() const { if (!GOTSection) - GOTSection = &G.createSection("$__GOT", MemProt::Read); + GOTSection = &G.createSection("$__GOT", orc::MemProt::Read); return *GOTSection; } Section &getStubsSection() const { if (!StubsSection) StubsSection = - &G.createSection("$__STUBS", MemProt::Read | MemProt::Exec); + &G.createSection("$__STUBS", orc::MemProt::Read | orc::MemProt::Exec); return *StubsSection; } @@ -205,12 +205,13 @@ private: return makeTargetOutOfRangeError(G, B, E); if (LLVM_UNLIKELY(!isAlignmentCorrect(Value, 2))) return makeAlignmentError(FixupAddress, Value, 2, E); - uint32_t Imm31_25 = - extractBits(Value, 5, 6) << 25 | extractBits(Value, 12, 1) << 31; - uint32_t Imm11_7 = - extractBits(Value, 1, 4) << 8 | extractBits(Value, 11, 1) << 7; + uint32_t Imm12 = extractBits(Value, 12, 1) << 31; + uint32_t Imm10_5 = extractBits(Value, 5, 6) << 25; + uint32_t Imm4_1 = extractBits(Value, 1, 4) << 8; + uint32_t Imm11 = extractBits(Value, 11, 1) << 7; uint32_t RawInstr = *(little32_t *)FixupPtr; - *(little32_t *)FixupPtr = (RawInstr & 0x1FFF07F) | Imm31_25 | Imm11_7; + *(little32_t *)FixupPtr = + (RawInstr & 0x1FFF07F) | Imm12 | Imm10_5 | Imm4_1 | Imm11; break; } case R_RISCV_JAL: { @@ -224,27 +225,8 @@ private: uint32_t Imm11 = extractBits(Value, 11, 1) << 20; uint32_t Imm19_12 = extractBits(Value, 12, 8) << 12; uint32_t RawInstr = *(little32_t *)FixupPtr; - *(little32_t *)FixupPtr = RawInstr | Imm20 | Imm10_1 | Imm11 | Imm19_12; - break; - } - case R_RISCV_HI20: { - int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); - int64_t Hi = Value + 0x800; - if (LLVM_UNLIKELY(!isInRangeForImm(Hi, 32))) - return makeTargetOutOfRangeError(G, B, E); - uint32_t RawInstr = *(little32_t *)FixupPtr; - *(little32_t *)FixupPtr = - (RawInstr & 0xFFF) | (static_cast<uint32_t>(Hi & 0xFFFFF000)); - break; - } - case R_RISCV_LO12_I: { - // FIXME: We assume that R_RISCV_HI20 is present in object code and pairs - // with current relocation R_RISCV_LO12_I. So here may need a check. - int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); - int32_t Lo = Value & 0xFFF; - uint32_t RawInstr = *(little32_t *)FixupPtr; *(little32_t *)FixupPtr = - (RawInstr & 0xFFFFF) | (static_cast<uint32_t>(Lo & 0xFFF) << 20); + (RawInstr & 0xFFF) | Imm20 | Imm10_1 | Imm11 | Imm19_12; break; } case R_RISCV_CALL: { @@ -261,6 +243,9 @@ private: RawInstrJalr | (static_cast<uint32_t>(Lo) << 20); break; } + // The relocations R_RISCV_CALL_PLT and R_RISCV_GOT_HI20 are handled by + // PerGraphGOTAndPLTStubsBuilder_ELF_riscv and are transformed into + // R_RISCV_CALL and R_RISCV_PCREL_HI20. case R_RISCV_PCREL_HI20: { int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; int64_t Hi = Value + 0x800; @@ -291,23 +276,65 @@ private: // pairs with current relocation R_RISCV_PCREL_LO12_S. So here may need a // check. auto RelHI20 = getRISCVPCRelHi20(E); + if (!RelHI20) + return RelHI20.takeError(); int64_t Value = RelHI20->getTarget().getAddress() + RelHI20->getAddend() - E.getTarget().getAddress(); int64_t Lo = Value & 0xFFF; - uint32_t Imm31_25 = extractBits(Lo, 5, 7) << 25; - uint32_t Imm11_7 = extractBits(Lo, 0, 5) << 7; + uint32_t Imm11_5 = extractBits(Lo, 5, 7) << 25; + uint32_t Imm4_0 = extractBits(Lo, 0, 5) << 7; uint32_t RawInstr = *(little32_t *)FixupPtr; - *(little32_t *)FixupPtr = (RawInstr & 0x1FFF07F) | Imm31_25 | Imm11_7; + *(little32_t *)FixupPtr = (RawInstr & 0x1FFF07F) | Imm11_5 | Imm4_0; break; } - case R_RISCV_ADD64: { + case R_RISCV_HI20: { + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); + int64_t Hi = Value + 0x800; + if (LLVM_UNLIKELY(!isInRangeForImm(Hi, 32))) + return makeTargetOutOfRangeError(G, B, E); + uint32_t RawInstr = *(little32_t *)FixupPtr; + *(little32_t *)FixupPtr = + (RawInstr & 0xFFF) | (static_cast<uint32_t>(Hi & 0xFFFFF000)); + break; + } + case R_RISCV_LO12_I: { + // FIXME: We assume that R_RISCV_HI20 is present in object code and pairs + // with current relocation R_RISCV_LO12_I. So here may need a check. + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); + int32_t Lo = Value & 0xFFF; + uint32_t RawInstr = *(little32_t *)FixupPtr; + *(little32_t *)FixupPtr = + (RawInstr & 0xFFFFF) | (static_cast<uint32_t>(Lo & 0xFFF) << 20); + break; + } + case R_RISCV_LO12_S: { + // FIXME: We assume that R_RISCV_HI20 is present in object code and pairs + // with current relocation R_RISCV_LO12_S. So here may need a check. + int64_t Value = (E.getTarget().getAddress() + E.getAddend()).getValue(); + int64_t Lo = Value & 0xFFF; + uint32_t Imm11_5 = extractBits(Lo, 5, 7) << 25; + uint32_t Imm4_0 = extractBits(Lo, 0, 5) << 7; + uint32_t RawInstr = *(little32_t *)FixupPtr; + *(little32_t *)FixupPtr = (RawInstr & 0x1FFF07F) | Imm11_5 | Imm4_0; + break; + } + case R_RISCV_ADD8: { + int64_t Value = + (E.getTarget().getAddress() + + *(reinterpret_cast<const uint8_t *>(FixupAddress.getValue())) + + E.getAddend()) + .getValue(); + *FixupPtr = static_cast<uint8_t>(Value); + break; + } + case R_RISCV_ADD16: { int64_t Value = (E.getTarget().getAddress() + - support::endian::read64le(reinterpret_cast<const void *>( + support::endian::read16le(reinterpret_cast<const void *>( FixupAddress.getValue())) + E.getAddend()) .getValue(); - *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); + *(little16_t *)FixupPtr = static_cast<uint16_t>(Value); break; } case R_RISCV_ADD32: { @@ -319,29 +346,27 @@ private: *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); break; } - case R_RISCV_ADD16: { + case R_RISCV_ADD64: { int64_t Value = (E.getTarget().getAddress() + - support::endian::read16le(reinterpret_cast<const void *>( + support::endian::read64le(reinterpret_cast<const void *>( FixupAddress.getValue())) + E.getAddend()) .getValue(); - *(little16_t *)FixupPtr = static_cast<uint32_t>(Value); + *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); break; } - case R_RISCV_ADD8: { + case R_RISCV_SUB8: { int64_t Value = - (E.getTarget().getAddress() + - *(reinterpret_cast<const uint8_t *>(FixupAddress.getValue())) + - E.getAddend()) - .getValue(); + *(reinterpret_cast<const uint8_t *>(FixupAddress.getValue())) - + E.getTarget().getAddress().getValue() - E.getAddend(); *FixupPtr = static_cast<uint8_t>(Value); break; } - case R_RISCV_SUB64: { - int64_t Value = support::endian::read64le(reinterpret_cast<const void *>( + case R_RISCV_SUB16: { + int64_t Value = support::endian::read16le(reinterpret_cast<const void *>( FixupAddress.getValue())) - E.getTarget().getAddress().getValue() - E.getAddend(); - *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); + *(little16_t *)FixupPtr = static_cast<uint32_t>(Value); break; } case R_RISCV_SUB32: { @@ -351,18 +376,46 @@ private: *(little32_t *)FixupPtr = static_cast<uint32_t>(Value); break; } - case R_RISCV_SUB16: { - int64_t Value = support::endian::read16le(reinterpret_cast<const void *>( + case R_RISCV_SUB64: { + int64_t Value = support::endian::read64le(reinterpret_cast<const void *>( FixupAddress.getValue())) - E.getTarget().getAddress().getValue() - E.getAddend(); - *(little16_t *)FixupPtr = static_cast<uint32_t>(Value); + *(little64_t *)FixupPtr = static_cast<uint64_t>(Value); break; } - case R_RISCV_SUB8: { - int64_t Value = - *(reinterpret_cast<const uint8_t *>(FixupAddress.getValue())) - - E.getTarget().getAddress().getValue() - E.getAddend(); - *FixupPtr = static_cast<uint8_t>(Value); + case R_RISCV_RVC_BRANCH: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + if (LLVM_UNLIKELY(!isInRangeForImm(Value >> 1, 8))) + return makeTargetOutOfRangeError(G, B, E); + if (LLVM_UNLIKELY(!isAlignmentCorrect(Value, 2))) + return makeAlignmentError(FixupAddress, Value, 2, E); + uint16_t Imm8 = extractBits(Value, 8, 1) << 12; + uint16_t Imm4_3 = extractBits(Value, 3, 2) << 10; + uint16_t Imm7_6 = extractBits(Value, 6, 2) << 5; + uint16_t Imm2_1 = extractBits(Value, 1, 2) << 3; + uint16_t Imm5 = extractBits(Value, 5, 1) << 2; + uint16_t RawInstr = *(little16_t *)FixupPtr; + *(little16_t *)FixupPtr = + (RawInstr & 0xE383) | Imm8 | Imm4_3 | Imm7_6 | Imm2_1 | Imm5; + break; + } + case R_RISCV_RVC_JUMP: { + int64_t Value = E.getTarget().getAddress() + E.getAddend() - FixupAddress; + if (LLVM_UNLIKELY(!isInRangeForImm(Value >> 1, 11))) + return makeTargetOutOfRangeError(G, B, E); + if (LLVM_UNLIKELY(!isAlignmentCorrect(Value, 2))) + return makeAlignmentError(FixupAddress, Value, 2, E); + uint16_t Imm11 = extractBits(Value, 11, 1) << 12; + uint16_t Imm4 = extractBits(Value, 4, 1) << 11; + uint16_t Imm9_8 = extractBits(Value, 8, 2) << 9; + uint16_t Imm10 = extractBits(Value, 10, 1) << 8; + uint16_t Imm6 = extractBits(Value, 6, 1) << 7; + uint16_t Imm7 = extractBits(Value, 7, 1) << 6; + uint16_t Imm3_1 = extractBits(Value, 1, 3) << 3; + uint16_t Imm5 = extractBits(Value, 5, 1) << 2; + uint16_t RawInstr = *(little16_t *)FixupPtr; + *(little16_t *)FixupPtr = (RawInstr & 0xE003) | Imm11 | Imm4 | Imm9_8 | + Imm10 | Imm6 | Imm7 | Imm3_1 | Imm5; break; } case R_RISCV_SUB6: { @@ -425,38 +478,44 @@ private: return EdgeKind_riscv::R_RISCV_BRANCH; case ELF::R_RISCV_JAL: return EdgeKind_riscv::R_RISCV_JAL; - case ELF::R_RISCV_HI20: - return EdgeKind_riscv::R_RISCV_HI20; - case ELF::R_RISCV_LO12_I: - return EdgeKind_riscv::R_RISCV_LO12_I; case ELF::R_RISCV_CALL: return EdgeKind_riscv::R_RISCV_CALL; + case ELF::R_RISCV_CALL_PLT: + return EdgeKind_riscv::R_RISCV_CALL_PLT; + case ELF::R_RISCV_GOT_HI20: + return EdgeKind_riscv::R_RISCV_GOT_HI20; case ELF::R_RISCV_PCREL_HI20: return EdgeKind_riscv::R_RISCV_PCREL_HI20; case ELF::R_RISCV_PCREL_LO12_I: return EdgeKind_riscv::R_RISCV_PCREL_LO12_I; case ELF::R_RISCV_PCREL_LO12_S: return EdgeKind_riscv::R_RISCV_PCREL_LO12_S; - case ELF::R_RISCV_GOT_HI20: - return EdgeKind_riscv::R_RISCV_GOT_HI20; - case ELF::R_RISCV_CALL_PLT: - return EdgeKind_riscv::R_RISCV_CALL_PLT; - case ELF::R_RISCV_ADD64: - return EdgeKind_riscv::R_RISCV_ADD64; - case ELF::R_RISCV_ADD32: - return EdgeKind_riscv::R_RISCV_ADD32; - case ELF::R_RISCV_ADD16: - return EdgeKind_riscv::R_RISCV_ADD16; + case ELF::R_RISCV_HI20: + return EdgeKind_riscv::R_RISCV_HI20; + case ELF::R_RISCV_LO12_I: + return EdgeKind_riscv::R_RISCV_LO12_I; + case ELF::R_RISCV_LO12_S: + return EdgeKind_riscv::R_RISCV_LO12_S; case ELF::R_RISCV_ADD8: return EdgeKind_riscv::R_RISCV_ADD8; - case ELF::R_RISCV_SUB64: - return EdgeKind_riscv::R_RISCV_SUB64; - case ELF::R_RISCV_SUB32: - return EdgeKind_riscv::R_RISCV_SUB32; - case ELF::R_RISCV_SUB16: - return EdgeKind_riscv::R_RISCV_SUB16; + case ELF::R_RISCV_ADD16: + return EdgeKind_riscv::R_RISCV_ADD16; + case ELF::R_RISCV_ADD32: + return EdgeKind_riscv::R_RISCV_ADD32; + case ELF::R_RISCV_ADD64: + return EdgeKind_riscv::R_RISCV_ADD64; case ELF::R_RISCV_SUB8: return EdgeKind_riscv::R_RISCV_SUB8; + case ELF::R_RISCV_SUB16: + return EdgeKind_riscv::R_RISCV_SUB16; + case ELF::R_RISCV_SUB32: + return EdgeKind_riscv::R_RISCV_SUB32; + case ELF::R_RISCV_SUB64: + return EdgeKind_riscv::R_RISCV_SUB64; + case ELF::R_RISCV_RVC_BRANCH: + return EdgeKind_riscv::R_RISCV_RVC_BRANCH; + case ELF::R_RISCV_RVC_JUMP: + return EdgeKind_riscv::R_RISCV_RVC_JUMP; case ELF::R_RISCV_SUB6: return EdgeKind_riscv::R_RISCV_SUB6; case ELF::R_RISCV_SET6: @@ -482,8 +541,8 @@ private: using Base = ELFLinkGraphBuilder<ELFT>; using Self = ELFLinkGraphBuilder_riscv<ELFT>; for (const auto &RelSect : Base::Sections) - if (Error Err = Base::forEachRelocation(RelSect, this, - &Self::addSingleRelocation)) + if (Error Err = Base::forEachRelaRelocation(RelSect, this, + &Self::addSingleRelocation)) return Err; return Error::success(); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp index 8f21274bd1a3..c9359522c248 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/ELF_x86_64.cpp @@ -27,7 +27,6 @@ using namespace llvm; using namespace llvm::jitlink; -using namespace llvm::jitlink::ELF_x86_64_Edges; namespace { @@ -69,7 +68,8 @@ public: private: Section &getTLSInfoSection(LinkGraph &G) { if (!TLSInfoTable) - TLSInfoTable = &G.createSection(ELFTLSInfoSectionName, MemProt::Read); + TLSInfoTable = + &G.createSection(ELFTLSInfoSectionName, orc::MemProt::Read); return *TLSInfoTable; } @@ -106,34 +106,48 @@ class ELFLinkGraphBuilder_x86_64 : public ELFLinkGraphBuilder<object::ELF64LE> { private: using ELFT = object::ELF64LE; - static Expected<ELF_x86_64_Edges::ELFX86RelocationKind> - getRelocationKind(const uint32_t Type) { + enum ELFX86RelocationKind : Edge::Kind { + Branch32 = Edge::FirstRelocation, + Pointer32Signed, + Pointer64, + PCRel32, + PCRel32GOTLoad, + PCRel32GOTLoadRelaxable, + PCRel32REXGOTLoadRelaxable, + PCRel32TLV, + PCRel64GOT, + GOTOFF64, + GOT64, + Delta64, + }; + + static Expected<ELFX86RelocationKind> getRelocationKind(const uint32_t Type) { switch (Type) { case ELF::R_X86_64_32S: - return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer32Signed; + return ELFX86RelocationKind::Pointer32Signed; case ELF::R_X86_64_PC32: - return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32; + return ELFX86RelocationKind::PCRel32; case ELF::R_X86_64_PC64: case ELF::R_X86_64_GOTPC64: - return ELF_x86_64_Edges::ELFX86RelocationKind::Delta64; + return ELFX86RelocationKind::Delta64; case ELF::R_X86_64_64: - return ELF_x86_64_Edges::ELFX86RelocationKind::Pointer64; + return ELFX86RelocationKind::Pointer64; case ELF::R_X86_64_GOTPCREL: - return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoad; + return ELFX86RelocationKind::PCRel32GOTLoad; case ELF::R_X86_64_GOTPCRELX: - return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32GOTLoadRelaxable; + return ELFX86RelocationKind::PCRel32GOTLoadRelaxable; case ELF::R_X86_64_REX_GOTPCRELX: - return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable; + return ELFX86RelocationKind::PCRel32REXGOTLoadRelaxable; case ELF::R_X86_64_GOTPCREL64: - return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel64GOT; + return ELFX86RelocationKind::PCRel64GOT; case ELF::R_X86_64_GOT64: - return ELF_x86_64_Edges::ELFX86RelocationKind::GOT64; + return ELFX86RelocationKind::GOT64; case ELF::R_X86_64_GOTOFF64: - return ELF_x86_64_Edges::ELFX86RelocationKind::GOTOFF64; + return ELFX86RelocationKind::GOTOFF64; case ELF::R_X86_64_PLT32: - return ELF_x86_64_Edges::ELFX86RelocationKind::Branch32; + return ELFX86RelocationKind::Branch32; case ELF::R_X86_64_TLSGD: - return ELF_x86_64_Edges::ELFX86RelocationKind::PCRel32TLV; + return ELFX86RelocationKind::PCRel32TLV; } return make_error<JITLinkError>( "Unsupported x86-64 relocation type " + formatv("{0:d}: ", Type) + @@ -152,8 +166,8 @@ private: "No SHT_REL in valid x64 ELF object files", inconvertibleErrorCode()); - if (Error Err = Base::forEachRelocation(RelSect, this, - &Self::addSingleRelocation)) + if (Error Err = Base::forEachRelaRelocation(RelSect, this, + &Self::addSingleRelocation)) return Err; } @@ -399,32 +413,5 @@ void link_ELF_x86_64(std::unique_ptr<LinkGraph> G, ELFJITLinker_x86_64::link(std::move(Ctx), std::move(G), std::move(Config)); } -const char *getELFX86RelocationKindName(Edge::Kind R) { - switch (R) { - case Branch32: - return "Branch32"; - case Pointer32Signed: - return "Pointer32Signed"; - case Pointer64: - return "Pointer64"; - case PCRel32: - return "PCRel32"; - case PCRel32GOTLoad: - return "PCRel32GOTLoad"; - case PCRel32GOTLoadRelaxable: - return "PCRel32GOTLoadRelaxable"; - case PCRel32REXGOTLoadRelaxable: - return "PCRel32REXGOTLoad"; - case PCRel64GOT: - return "PCRel64GOT"; - case Delta64: - return "Delta64"; - case GOT64: - return "GOT64"; - case GOTOFF64: - return "GOTOFF64"; - } - return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); -} } // end namespace jitlink } // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index 08fdc7c9e6b1..bd5b4d585550 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -195,7 +195,7 @@ Block &LinkGraph::splitBlock(Block &B, size_t SplitIndex, SplitBlockCache LocalBlockSymbolsCache; if (!Cache) Cache = &LocalBlockSymbolsCache; - if (*Cache == None) { + if (*Cache == std::nullopt) { *Cache = SplitBlockCache::value_type(); for (auto *Sym : B.getSection().symbols()) if (&Sym->getBlock() == &B) @@ -309,14 +309,14 @@ void LinkGraph::dump(raw_ostream &OS) { } OS << "Absolute symbols:\n"; - if (!llvm::empty(absolute_symbols())) { + if (!absolute_symbols().empty()) { for (auto *Sym : absolute_symbols()) OS << " " << Sym->getAddress() << ": " << *Sym << "\n"; } else OS << " none\n"; OS << "\nExternal symbols:\n"; - if (!llvm::empty(external_symbols())) { + if (!external_symbols().empty()) { for (auto *Sym : external_symbols()) OS << " " << Sym->getAddress() << ": " << *Sym << "\n"; } else diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 6d321a080829..17de84fa6e11 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -51,6 +51,9 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { Ctx->getMemoryManager().allocate( Ctx->getJITLinkDylib(), *G, [S = std::move(Self)](AllocResult AR) mutable { + // FIXME: Once MSVC implements c++17 order of evaluation rules for calls + // this can be simplified to + // S->linkPhase2(std::move(S), std::move(AR)); auto *TmpSelf = S.get(); TmpSelf->linkPhase2(std::move(S), std::move(AR)); }); @@ -88,8 +91,8 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, dbgs() << "No external symbols for " << G->getName() << ". Proceeding immediately with link phase 3.\n"; }); - // FIXME: Once callee expressions are defined to be sequenced before - // argument expressions (c++17) we can simplify this. See below. + // FIXME: Once MSVC implements c++17 order of evaluation rules for calls + // this can be simplified. See below. auto &TmpSelf = *Self; TmpSelf.linkPhase3(std::move(Self), AsyncLookupResult()); return; @@ -104,8 +107,8 @@ void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, // We're about to hand off ownership of ourself to the continuation. Grab a // pointer to the context so that we can call it to initiate the lookup. // - // FIXME: Once callee expressions are defined to be sequenced before argument - // expressions (c++17) we can simplify all this to: + // FIXME: Once MSVC implements c++17 order of evaluation rules for calls this + // can be simplified to: // // Ctx->lookup(std::move(UnresolvedExternals), // [Self=std::move(Self)](Expected<AsyncLookupResult> Result) { @@ -161,6 +164,9 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, return abandonAllocAndBailOut(std::move(Self), std::move(Err)); Alloc->finalize([S = std::move(Self)](FinalizeResult FR) mutable { + // FIXME: Once MSVC implements c++17 order of evaluation rules for calls + // this can be simplified to + // S->linkPhase2(std::move(S), std::move(AR)); auto *TmpSelf = S.get(); TmpSelf->linkPhase4(std::move(S), std::move(FR)); }); @@ -197,9 +203,8 @@ JITLinkContext::LookupMap JITLinkerBase::getExternalSymbolNames() const { assert(Sym->getName() != StringRef() && Sym->getName() != "" && "Externals must be named"); SymbolLookupFlags LookupFlags = - Sym->getLinkage() == Linkage::Weak - ? SymbolLookupFlags::WeaklyReferencedSymbol - : SymbolLookupFlags::RequiredSymbol; + Sym->isWeaklyReferenced() ? SymbolLookupFlags::WeaklyReferencedSymbol + : SymbolLookupFlags::RequiredSymbol; UnresolvedExternals[Sym->getName()] = LookupFlags; } return UnresolvedExternals; @@ -212,19 +217,41 @@ void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { assert(!Sym->getAddress() && "Symbol already resolved"); assert(!Sym->isDefined() && "Symbol being resolved is already defined"); auto ResultI = Result.find(Sym->getName()); - if (ResultI != Result.end()) + if (ResultI != Result.end()) { Sym->getAddressable().setAddress( orc::ExecutorAddr(ResultI->second.getAddress())); - else - assert(Sym->getLinkage() == Linkage::Weak && + Sym->setLinkage(ResultI->second.getFlags().isWeak() ? Linkage::Weak + : Linkage::Strong); + Sym->setScope(ResultI->second.getFlags().isExported() ? Scope::Default + : Scope::Hidden); + } else + assert(Sym->isWeaklyReferenced() && "Failed to resolve non-weak reference"); } LLVM_DEBUG({ dbgs() << "Externals after applying lookup result:\n"; - for (auto *Sym : G->external_symbols()) + for (auto *Sym : G->external_symbols()) { dbgs() << " " << Sym->getName() << ": " - << formatv("{0:x16}", Sym->getAddress().getValue()) << "\n"; + << formatv("{0:x16}", Sym->getAddress().getValue()); + switch (Sym->getLinkage()) { + case Linkage::Strong: + break; + case Linkage::Weak: + dbgs() << " (weak)"; + break; + } + switch (Sym->getScope()) { + case Scope::Local: + llvm_unreachable("External symbol should not have local linkage"); + case Scope::Hidden: + break; + case Scope::Default: + dbgs() << " (exported)"; + break; + } + dbgs() << "\n"; + } }); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index 1095fa5ce701..2c9244526536 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -128,8 +128,12 @@ private: // Copy Block data and apply fixups. LLVM_DEBUG(dbgs() << " Applying fixups.\n"); - assert((!B->isZeroFill() || B->edges_size() == 0) && - "Edges in zero-fill block?"); + assert((!B->isZeroFill() || all_of(B->edges(), + [](const Edge &E) { + return E.getKind() == + Edge::KeepAlive; + })) && + "Non-KeepAlive edges in zero-fill block?"); for (auto &E : B->edges()) { // Skip non-relocation edges. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index acb759d6ce79..bd44b86f3081 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -25,7 +25,7 @@ BasicLayout::BasicLayout(LinkGraph &G) : G(G) { for (auto &Sec : G.sections()) { // Skip empty sections. - if (empty(Sec.blocks())) + if (Sec.blocks().empty()) continue; auto &Seg = Segments[{Sec.getMemProt(), Sec.getMemDeallocPolicy()}]; @@ -89,7 +89,7 @@ BasicLayout::getContiguousPageBasedLayoutSizes(uint64_t PageSize) { inconvertibleErrorCode()); uint64_t SegSize = alignTo(Seg.ContentSize + Seg.ZeroFillSize, PageSize); - if (AG.getMemDeallocPolicy() == MemDeallocPolicy::Standard) + if (AG.getMemDeallocPolicy() == orc::MemDeallocPolicy::Standard) SegsSizes.StandardSegs += SegSize; else SegsSizes.FinalizeSegs += SegSize; @@ -146,7 +146,7 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, const JITLinkDylib *JD, SegmentMap Segments, OnCreatedFunction OnCreated) { - static_assert(AllocGroup::NumGroups == 16, + static_assert(orc::AllocGroup::NumGroups == 16, "AllocGroup has changed. Section names below must be updated"); StringRef AGSectionNames[] = { "__---.standard", "__R--.standard", "__-W-.standard", "__RW-.standard", @@ -156,7 +156,7 @@ void SimpleSegmentAlloc::Create(JITLinkMemoryManager &MemMgr, auto G = std::make_unique<LinkGraph>("", Triple(), 0, support::native, nullptr); - AllocGroupSmallMap<Block *> ContentBlocks; + orc::AllocGroupSmallMap<Block *> ContentBlocks; orc::ExecutorAddr NextAddr(0x100000); for (auto &KV : Segments) { @@ -213,7 +213,8 @@ SimpleSegmentAlloc & SimpleSegmentAlloc::operator=(SimpleSegmentAlloc &&) = default; SimpleSegmentAlloc::~SimpleSegmentAlloc() = default; -SimpleSegmentAlloc::SegmentInfo SimpleSegmentAlloc::getSegInfo(AllocGroup AG) { +SimpleSegmentAlloc::SegmentInfo +SimpleSegmentAlloc::getSegInfo(orc::AllocGroup AG) { auto I = ContentBlocks.find(AG); if (I != ContentBlocks.end()) { auto &B = *I->second; @@ -223,7 +224,8 @@ SimpleSegmentAlloc::SegmentInfo SimpleSegmentAlloc::getSegInfo(AllocGroup AG) { } SimpleSegmentAlloc::SimpleSegmentAlloc( - std::unique_ptr<LinkGraph> G, AllocGroupSmallMap<Block *> ContentBlocks, + std::unique_ptr<LinkGraph> G, + orc::AllocGroupSmallMap<Block *> ContentBlocks, std::unique_ptr<JITLinkMemoryManager::InFlightAlloc> Alloc) : G(std::move(G)), ContentBlocks(std::move(ContentBlocks)), Alloc(std::move(Alloc)) {} @@ -394,9 +396,10 @@ void InProcessMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, auto &AG = KV.first; auto &Seg = KV.second; - auto &SegAddr = (AG.getMemDeallocPolicy() == MemDeallocPolicy::Standard) - ? NextStandardSegAddr - : NextFinalizeSegAddr; + auto &SegAddr = + (AG.getMemDeallocPolicy() == orc::MemDeallocPolicy::Standard) + ? NextStandardSegAddr + : NextFinalizeSegAddr; Seg.WorkingMem = SegAddr.toPtr<char *>(); Seg.Addr = SegAddr; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp index 1bf12f438be0..987689993397 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -11,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "MachOLinkGraphBuilder.h" +#include <optional> #define DEBUG_TYPE "jitlink" @@ -51,7 +52,10 @@ MachOLinkGraphBuilder::MachOLinkGraphBuilder( : Obj(Obj), G(std::make_unique<LinkGraph>( std::string(Obj.getFileName()), std::move(TT), getPointerSize(Obj), - getEndianness(Obj), std::move(GetEdgeKindName))) {} + getEndianness(Obj), std::move(GetEdgeKindName))) { + auto &MachHeader = Obj.getHeader64(); + SubsectionsViaSymbols = MachHeader.flags & MachO::MH_SUBSECTIONS_VIA_SYMBOLS; +} void MachOLinkGraphBuilder::addCustomSectionParser( StringRef SectionName, SectionParserFunction Parser) { @@ -108,8 +112,8 @@ MachOLinkGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { Section &MachOLinkGraphBuilder::getCommonSection() { if (!CommonSection) - CommonSection = - &G->createSection(CommonSectionName, MemProt::Read | MemProt::Write); + CommonSection = &G->createSection(CommonSectionName, + orc::MemProt::Read | orc::MemProt::Write); return *CommonSection; } @@ -174,11 +178,11 @@ Error MachOLinkGraphBuilder::createNormalizedSections() { // Get prot flags. // FIXME: Make sure this test is correct (it's probably missing cases // as-is). - MemProt Prot; + orc::MemProt Prot; if (NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS) - Prot = MemProt::Read | MemProt::Exec; + Prot = orc::MemProt::Read | orc::MemProt::Exec; else - Prot = MemProt::Read | MemProt::Write; + Prot = orc::MemProt::Read | orc::MemProt::Write; auto FullyQualifiedName = G->allocateString(StringRef(NSec.SegName) + "," + NSec.SectName); @@ -257,7 +261,7 @@ Error MachOLinkGraphBuilder::createNormalizedSymbols() { if (Type & MachO::N_STAB) continue; - Optional<StringRef> Name; + std::optional<StringRef> Name; if (NStrX) { if (auto NameOrErr = SymRef.getName()) Name = *NameOrErr; @@ -347,19 +351,20 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { if (!NSym.Name) return make_error<JITLinkError>("Anonymous common symbol at index " + Twine(KV.first)); - NSym.GraphSymbol = &G->addCommonSymbol( - *NSym.Name, NSym.S, getCommonSection(), orc::ExecutorAddr(), - orc::ExecutorAddrDiff(NSym.Value), - 1ull << MachO::GET_COMM_ALIGN(NSym.Desc), - NSym.Desc & MachO::N_NO_DEAD_STRIP); + NSym.GraphSymbol = &G->addDefinedSymbol( + G->createZeroFillBlock(getCommonSection(), + orc::ExecutorAddrDiff(NSym.Value), + orc::ExecutorAddr(), + 1ull << MachO::GET_COMM_ALIGN(NSym.Desc), 0), + 0, *NSym.Name, orc::ExecutorAddrDiff(NSym.Value), Linkage::Strong, + NSym.S, false, NSym.Desc & MachO::N_NO_DEAD_STRIP); } else { if (!NSym.Name) return make_error<JITLinkError>("Anonymous external symbol at " "index " + Twine(KV.first)); NSym.GraphSymbol = &G->addExternalSymbol( - *NSym.Name, 0, - NSym.Desc & MachO::N_WEAK_REF ? Linkage::Weak : Linkage::Strong); + *NSym.Name, 0, (NSym.Desc & MachO::N_WEAK_REF) != 0); } break; case MachO::N_ABS: @@ -485,15 +490,24 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { } // Visit section symbols in order by popping off the reverse-sorted stack, - // building blocks for each alt-entry chain and creating symbols as we go. + // building graph symbols as we go. + // + // If MH_SUBSECTIONS_VIA_SYMBOLS is set we'll build a block for each + // alt-entry chain. + // + // If MH_SUBSECTIONS_VIA_SYMBOLS is not set then we'll just build one block + // for the whole section. while (!SecNSymStack.empty()) { SmallVector<NormalizedSymbol *, 8> BlockSyms; + // Get the symbols in this alt-entry chain, or the whole section (if + // !SubsectionsViaSymbols). BlockSyms.push_back(SecNSymStack.back()); SecNSymStack.pop_back(); while (!SecNSymStack.empty() && (isAltEntry(*SecNSymStack.back()) || - SecNSymStack.back()->Value == BlockSyms.back()->Value)) { + SecNSymStack.back()->Value == BlockSyms.back()->Value || + !SubsectionsViaSymbols)) { BlockSyms.push_back(SecNSymStack.back()); SecNSymStack.pop_back(); } @@ -524,7 +538,7 @@ Error MachOLinkGraphBuilder::graphifyRegularSymbols() { BlockStart, NSec.Alignment, BlockStart % NSec.Alignment); - Optional<orc::ExecutorAddr> LastCanonicalAddr; + std::optional<orc::ExecutorAddr> LastCanonicalAddr; auto SymEnd = BlockEnd; while (!BlockSyms.empty()) { auto &NSym = *BlockSyms.back(); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index 2951a8533098..ba6cfaf8aa94 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -37,8 +37,9 @@ protected: friend class MachOLinkGraphBuilder; private: - NormalizedSymbol(Optional<StringRef> Name, uint64_t Value, uint8_t Type, - uint8_t Sect, uint16_t Desc, Linkage L, Scope S) + NormalizedSymbol(std::optional<StringRef> Name, uint64_t Value, + uint8_t Type, uint8_t Sect, uint16_t Desc, Linkage L, + Scope S) : Name(Name), Value(Value), Type(Type), Sect(Sect), Desc(Desc), L(L), S(S) { assert((!Name || !Name->empty()) && "Name must be none or non-empty"); @@ -50,7 +51,7 @@ protected: NormalizedSymbol(NormalizedSymbol &&) = delete; NormalizedSymbol &operator=(NormalizedSymbol &&) = delete; - Optional<StringRef> Name; + std::optional<StringRef> Name; uint64_t Value = 0; uint8_t Type = 0; uint8_t Sect = 0; @@ -226,6 +227,7 @@ private: const object::MachOObjectFile &Obj; std::unique_ptr<LinkGraph> G; + bool SubsectionsViaSymbols = false; DenseMap<unsigned, NormalizedSection> IndexToSection; Section *CommonSection = nullptr; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 04194318498f..3380bb563140 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -332,7 +332,7 @@ private: if ((Instr & 0x7fffffff) != 0x14000000) return make_error<JITLinkError>("BRANCH26 target is not a B or BL " "instruction with a zero addend"); - Kind = aarch64::Branch26; + Kind = aarch64::Branch26PCRel; break; } case MachOPointer32: @@ -362,12 +362,12 @@ private: else return TargetSymbolOrErr.takeError(); Addend = TargetAddress - TargetSymbol->getAddress(); - Kind = aarch64::Pointer64Anon; + Kind = aarch64::Pointer64; break; } case MachOPage21: - case MachOTLVPage21: - case MachOGOTPage21: { + case MachOGOTPage21: + case MachOTLVPage21: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -380,10 +380,10 @@ private: if (*MachORelocKind == MachOPage21) { Kind = aarch64::Page21; - } else if (*MachORelocKind == MachOTLVPage21) { - Kind = aarch64::TLVPage21; } else if (*MachORelocKind == MachOGOTPage21) { - Kind = aarch64::GOTPage21; + Kind = aarch64::RequestGOTAndTransformToPage21; + } else if (*MachORelocKind == MachOTLVPage21) { + Kind = aarch64::RequestTLVPAndTransformToPage21; } break; } @@ -400,8 +400,8 @@ private: Kind = aarch64::PageOffset12; break; } - case MachOTLVPageOffset12: - case MachOGOTPageOffset12: { + case MachOGOTPageOffset12: + case MachOTLVPageOffset12: { if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) TargetSymbol = TargetSymbolOrErr->GraphSymbol; else @@ -412,10 +412,10 @@ private: "immediate instruction with a zero " "addend"); - if (*MachORelocKind == MachOTLVPageOffset12) { - Kind = aarch64::TLVPageOffset12; - } else if (*MachORelocKind == MachOGOTPageOffset12) { - Kind = aarch64::GOTPageOffset12; + if (*MachORelocKind == MachOGOTPageOffset12) { + Kind = aarch64::RequestGOTAndTransformToPageOffset12; + } else if (*MachORelocKind == MachOTLVPageOffset12) { + Kind = aarch64::RequestTLVPAndTransformToPageOffset12; } break; } @@ -425,7 +425,7 @@ private: else return TargetSymbolOrErr.takeError(); - Kind = aarch64::Delta32ToGOT; + Kind = aarch64::RequestGOTAndTransformToDelta32; break; case MachODelta32: case MachODelta64: { @@ -563,11 +563,8 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G, // Add eh-frame passses. // FIXME: Prune eh-frames for which compact-unwind is available once // we support compact-unwind registration with libunwind. - Config.PrePrunePasses.push_back( - DWARFRecordSectionSplitter("__TEXT,__eh_frame")); - Config.PrePrunePasses.push_back(EHFrameEdgeFixer( - "__TEXT,__eh_frame", 8, aarch64::Pointer32, aarch64::Pointer64, - aarch64::Delta32, aarch64::Delta64, aarch64::NegDelta32)); + Config.PrePrunePasses.push_back(createEHFrameSplitterPass_MachO_arm64()); + Config.PrePrunePasses.push_back(createEHFrameEdgeFixerPass_MachO_arm64()); // Add an in-place GOT/Stubs pass. Config.PostPrunePasses.push_back(buildTables_MachO_arm64); @@ -580,5 +577,16 @@ void link_MachO_arm64(std::unique_ptr<LinkGraph> G, MachOJITLinker_arm64::link(std::move(Ctx), std::move(G), std::move(Config)); } +LinkGraphPassFunction createEHFrameSplitterPass_MachO_arm64() { + return DWARFRecordSectionSplitter("__TEXT,__eh_frame"); +} + +LinkGraphPassFunction createEHFrameEdgeFixerPass_MachO_arm64() { + return EHFrameEdgeFixer("__TEXT,__eh_frame", aarch64::PointerSize, + aarch64::Pointer32, aarch64::Pointer64, + aarch64::Delta32, aarch64::Delta64, + aarch64::NegDelta32); +} + } // end namespace jitlink } // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 6dfd5548fcfd..be40b740a5a7 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -204,7 +204,7 @@ private: LLVM_DEBUG(dbgs() << "Processing relocations:\n"); - for (auto &S : Obj.sections()) { + for (const auto &S : Obj.sections()) { orc::ExecutorAddr SectionAddress(S.getAddress()); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MemoryFlags.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MemoryFlags.cpp deleted file mode 100644 index b73a310b2910..000000000000 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/MemoryFlags.cpp +++ /dev/null @@ -1,33 +0,0 @@ -//===------------- MemoryFlags.cpp - Memory allocation flags --------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#include "llvm/ExecutionEngine/JITLink/MemoryFlags.h" - -#define DEBUG_TYPE "jitlink" - -namespace llvm { -namespace jitlink { - -raw_ostream &operator<<(raw_ostream &OS, MemProt MP) { - return OS << (((MP & MemProt::Read) != MemProt::None) ? 'R' : '-') - << (((MP & MemProt::Write) != MemProt::None) ? 'W' : '-') - << (((MP & MemProt::Exec) != MemProt::None) ? 'X' : '-'); -} - -raw_ostream &operator<<(raw_ostream &OS, MemDeallocPolicy MDP) { - return OS << (MDP == MemDeallocPolicy::Standard ? "standard" : "finalize"); -} - -raw_ostream &operator<<(raw_ostream &OS, AllocGroup AG) { - return OS << '(' << AG.getMemProt() << ", " << AG.getMemDeallocPolicy() - << ')'; -} - -} // end namespace jitlink -} // end namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/SEHFrameSupport.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/SEHFrameSupport.h index f7689e4e4043..0d95fbf439b5 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/SEHFrameSupport.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/SEHFrameSupport.h @@ -58,4 +58,4 @@ private: } // end namespace jitlink } // end namespace llvm -#endif // LLVM_EXECUTIONENGINE_JITLINK_SEHFRAMESUPPORT_H
\ No newline at end of file +#endif // LLVM_EXECUTIONENGINE_JITLINK_SEHFRAMESUPPORT_H diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp index 9ecc71dfbb54..1011fa81f750 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/aarch64.cpp @@ -18,54 +18,53 @@ namespace llvm { namespace jitlink { namespace aarch64 { -const uint8_t NullGOTEntryContent[8] = {0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00}; +const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}; -const uint8_t StubContent[8] = { - 0x10, 0x00, 0x00, 0x58, // LDR x16, <literal> - 0x00, 0x02, 0x1f, 0xd6 // BR x16 +const char PointerJumpStubContent[12] = { + 0x10, 0x00, 0x00, (char)0x90u, // ADRP x16, <imm>@page21 + 0x10, 0x02, 0x40, (char)0xf9u, // LDR x16, [x16, <imm>@pageoff12] + 0x00, 0x02, 0x1f, (char)0xd6u // BR x16 }; const char *getEdgeKindName(Edge::Kind R) { switch (R) { - case Branch26: - return "Branch26"; case Pointer64: return "Pointer64"; - case Pointer64Anon: - return "Pointer64Anon"; - case Page21: - return "Page21"; - case PageOffset12: - return "PageOffset12"; - case MoveWide16: - return "MoveWide16"; - case GOTPage21: - return "GOTPage21"; - case GOTPageOffset12: - return "GOTPageOffset12"; - case TLVPage21: - return "TLVPage21"; - case TLVPageOffset12: - return "TLVPageOffset12"; - case TLSDescPage21: - return "TLSDescPage21"; - case TLSDescPageOffset12: - return "TLSDescPageOffset12"; - case Delta32ToGOT: - return "Delta32ToGOT"; - case PairedAddend: - return "PairedAddend"; - case LDRLiteral19: - return "LDRLiteral19"; - case Delta32: - return "Delta32"; + case Pointer32: + return "Pointer32"; case Delta64: return "Delta64"; - case NegDelta32: - return "NegDelta32"; + case Delta32: + return "Delta32"; case NegDelta64: return "NegDelta64"; + case NegDelta32: + return "NegDelta32"; + case Branch26PCRel: + return "Branch26PCRel"; + case MoveWide16: + return "MoveWide16"; + case LDRLiteral19: + return "LDRLiteral19"; + case Page21: + return "Page21"; + case PageOffset12: + return "PageOffset12"; + case RequestGOTAndTransformToPage21: + return "RequestGOTAndTransformToPage21"; + case RequestGOTAndTransformToPageOffset12: + return "RequestGOTAndTransformToPageOffset12"; + case RequestGOTAndTransformToDelta32: + return "RequestGOTAndTransformToDelta32"; + case RequestTLVPAndTransformToPage21: + return "RequestTLVPAndTransformToPage21"; + case RequestTLVPAndTransformToPageOffset12: + return "RequestTLVPAndTransformToPageOffset12"; + case RequestTLSDescEntryAndTransformToPage21: + return "RequestTLSDescEntryAndTransformToPage21"; + case RequestTLSDescEntryAndTransformToPageOffset12: + return "RequestTLSDescEntryAndTransformToPageOffset12"; default: return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/i386.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/i386.cpp new file mode 100644 index 000000000000..c2c5761cd272 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/i386.cpp @@ -0,0 +1,43 @@ +//===---- i386.cpp - Generic JITLink i386 edge kinds, utilities -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing i386 objects. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/i386.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm::jitlink::i386 { + +const char *getEdgeKindName(Edge::Kind K) { + switch (K) { + case None: + return "None"; + case Pointer32: + return "Pointer32"; + case PCRel32: + return "PCRel32"; + case Pointer16: + return "Pointer16"; + case PCRel16: + return "PCRel16"; + case Delta32: + return "Delta32"; + case Delta32FromGOT: + return "Delta32FromGOT"; + case RequestGOTAndTransformToDelta32FromGOT: + return "RequestGOTAndTransformToDelta32FromGOT"; + } + + return getGenericEdgeKindName(K); +} + +const char NullPointerContent[PointerSize] = {0x00, 0x00, 0x00, 0x00}; +} // namespace llvm::jitlink::i386 diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp new file mode 100644 index 000000000000..d1e44ec187cc --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/loongarch.cpp @@ -0,0 +1,60 @@ +//===--- loongarch.cpp - Generic JITLink loongarch edge kinds, utilities --===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic utilities for graphs representing loongarch objects. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/loongarch.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { +namespace loongarch { + +const char NullPointerContent[8] = {0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00}; + +const uint8_t LA64StubContent[StubEntrySize] = { + 0x14, 0x00, 0x00, 0x1a, // pcalau12i $t8, %page20(imm) + 0x94, 0x02, 0xc0, 0x28, // ld.d $t8, $t8, %pageoff12(imm) + 0x80, 0x02, 0x00, 0x4c // jr $t8 +}; + +const uint8_t LA32StubContent[StubEntrySize] = { + 0x14, 0x00, 0x00, 0x1a, // pcalau12i $t8, %page20(imm) + 0x94, 0x02, 0x80, 0x28, // ld.w $t8, $t8, %pageoff12(imm) + 0x80, 0x02, 0x00, 0x4c // jr $t8 +}; + +const char *getEdgeKindName(Edge::Kind K) { +#define KIND_NAME_CASE(K) \ + case K: \ + return #K; + + switch (K) { + KIND_NAME_CASE(Pointer64) + KIND_NAME_CASE(Pointer32) + KIND_NAME_CASE(Delta32) + KIND_NAME_CASE(NegDelta32) + KIND_NAME_CASE(Delta64) + KIND_NAME_CASE(Branch26PCRel) + KIND_NAME_CASE(Page20) + KIND_NAME_CASE(PageOffset12) + KIND_NAME_CASE(RequestGOTAndTransformToPage20) + KIND_NAME_CASE(RequestGOTAndTransformToPageOffset12) + default: + return getGenericEdgeKindName(K); + } +#undef KIND_NAME_CASE +} + +} // namespace loongarch +} // namespace jitlink +} // namespace llvm diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/riscv.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/riscv.cpp index 3848cc6b5f01..6ee92b065ca1 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/riscv.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/riscv.cpp @@ -28,36 +28,44 @@ const char *getEdgeKindName(Edge::Kind K) { return "R_RISCV_BRANCH"; case R_RISCV_JAL: return "R_RISCV_JAL"; - case R_RISCV_HI20: - return "R_RISCV_HI20"; - case R_RISCV_LO12_I: - return "R_RISCV_LO12_I"; + case R_RISCV_CALL: + return "R_RISCV_CALL"; + case R_RISCV_CALL_PLT: + return "R_RISCV_CALL_PLT"; + case R_RISCV_GOT_HI20: + return "R_RISCV_GOT_HI20"; case R_RISCV_PCREL_HI20: return "R_RISCV_PCREL_HI20"; case R_RISCV_PCREL_LO12_I: return "R_RISCV_PCREL_LO12_I"; case R_RISCV_PCREL_LO12_S: return "R_RISCV_PCREL_LO12_S"; - case R_RISCV_CALL: - return "R_RISCV_CALL"; - case R_RISCV_32_PCREL: - return "R_RISCV_32_PCREL"; - case R_RISCV_ADD64: - return "R_RISCV_ADD64"; - case R_RISCV_ADD32: - return "R_RISCV_ADD32"; - case R_RISCV_ADD16: - return "R_RISCV_ADD16"; + case R_RISCV_HI20: + return "R_RISCV_HI20"; + case R_RISCV_LO12_I: + return "R_RISCV_LO12_I"; + case R_RISCV_LO12_S: + return "R_RISCV_LO12_S"; case R_RISCV_ADD8: return "R_RISCV_ADD8"; - case R_RISCV_SUB64: - return "R_RISCV_SUB64"; - case R_RISCV_SUB32: - return "R_RISCV_SUB32"; - case R_RISCV_SUB16: - return "R_RISCV_SUB16"; + case R_RISCV_ADD16: + return "R_RISCV_ADD16"; + case R_RISCV_ADD32: + return "R_RISCV_ADD32"; + case R_RISCV_ADD64: + return "R_RISCV_ADD64"; case R_RISCV_SUB8: return "R_RISCV_SUB8"; + case R_RISCV_SUB16: + return "R_RISCV_SUB16"; + case R_RISCV_SUB32: + return "R_RISCV_SUB32"; + case R_RISCV_SUB64: + return "R_RISCV_SUB64"; + case R_RISCV_RVC_BRANCH: + return "R_RISCV_RVC_BRANCH"; + case R_RISCV_RVC_JUMP: + return "R_RISCV_RVC_JUMP"; case R_RISCV_SUB6: return "R_RISCV_SUB6"; case R_RISCV_SET6: @@ -68,6 +76,8 @@ const char *getEdgeKindName(Edge::Kind K) { return "R_RISCV_SET16"; case R_RISCV_SET32: return "R_RISCV_SET32"; + case R_RISCV_32_PCREL: + return "R_RISCV_32_PCREL"; } return getGenericEdgeKindName(K); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp index 393250a5578b..097e19e02530 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/JITLink/x86_64.cpp @@ -26,6 +26,8 @@ const char *getEdgeKindName(Edge::Kind K) { return "Pointer32"; case Pointer32Signed: return "Pointer32Signed"; + case Pointer16: + return "Pointer16"; case Delta64: return "Delta64"; case Delta32: diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp index 4ac901daa5c8..869b383dd064 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -261,10 +261,10 @@ void MCJIT::finalizeObject() { // Generate code for module is going to move objects out of the 'added' list, // so we need to copy that out before using it: SmallVector<Module*, 16> ModsToAdd; - for (auto M : OwnedModules.added()) + for (auto *M : OwnedModules.added()) ModsToAdd.push_back(M); - for (auto M : ModsToAdd) + for (auto *M : ModsToAdd) generateCodeForModule(M); finalizeLoadedModules(); @@ -659,9 +659,8 @@ void MCJIT::notifyObjectLoaded(const object::ObjectFile &Obj, static_cast<uint64_t>(reinterpret_cast<uintptr_t>(Obj.getData().data())); std::lock_guard<sys::Mutex> locked(lock); MemMgr->notifyObjectLoaded(this, Obj); - for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) { - EventListeners[I]->notifyObjectLoaded(Key, Obj, L); - } + for (JITEventListener *EL : EventListeners) + EL->notifyObjectLoaded(Key, Obj, L); } void MCJIT::notifyFreeingObject(const object::ObjectFile &Obj) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp new file mode 100644 index 000000000000..40716a7f9b61 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFPlatform.cpp @@ -0,0 +1,902 @@ +//===------- COFFPlatform.cpp - Utilities for executing COFF in Orc -------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/COFFPlatform.h" +#include "llvm/ExecutionEngine/Orc/DebugUtils.h" +#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" + +#include "llvm/Object/COFF.h" + +#include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h" + +#include "llvm/ExecutionEngine/JITLink/x86_64.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +namespace llvm { +namespace orc { +namespace shared { + +using SPSCOFFJITDylibDepInfo = SPSSequence<SPSExecutorAddr>; +using SPSCOFFJITDylibDepInfoMap = + SPSSequence<SPSTuple<SPSExecutorAddr, SPSCOFFJITDylibDepInfo>>; +using SPSCOFFObjectSectionsMap = + SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>; +using SPSCOFFRegisterObjectSectionsArgs = + SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap, bool>; +using SPSCOFFDeregisterObjectSectionsArgs = + SPSArgList<SPSExecutorAddr, SPSCOFFObjectSectionsMap>; + +} // namespace shared +} // namespace orc +} // namespace llvm +namespace { + +class COFFHeaderMaterializationUnit : public MaterializationUnit { +public: + COFFHeaderMaterializationUnit(COFFPlatform &CP, + const SymbolStringPtr &HeaderStartSymbol) + : MaterializationUnit(createHeaderInterface(CP, HeaderStartSymbol)), + CP(CP) {} + + StringRef getName() const override { return "COFFHeaderMU"; } + + void materialize(std::unique_ptr<MaterializationResponsibility> R) override { + unsigned PointerSize; + support::endianness Endianness; + const auto &TT = + CP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + + switch (TT.getArch()) { + case Triple::x86_64: + PointerSize = 8; + Endianness = support::endianness::little; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + + auto G = std::make_unique<jitlink::LinkGraph>( + "<COFFHeaderMU>", TT, PointerSize, Endianness, + jitlink::getGenericEdgeKindName); + auto &HeaderSection = G->createSection("__header", MemProt::Read); + auto &HeaderBlock = createHeaderBlock(*G, HeaderSection); + + // Init symbol is __ImageBase symbol. + auto &ImageBaseSymbol = G->addDefinedSymbol( + HeaderBlock, 0, *R->getInitializerSymbol(), HeaderBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, false, true); + + addImageBaseRelocationEdge(HeaderBlock, ImageBaseSymbol); + + CP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + } + + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} + +private: + struct HeaderSymbol { + const char *Name; + uint64_t Offset; + }; + + struct NTHeader { + support::ulittle32_t PEMagic; + object::coff_file_header FileHeader; + struct PEHeader { + object::pe32plus_header Header; + object::data_directory DataDirectory[COFF::NUM_DATA_DIRECTORIES + 1]; + } OptionalHeader; + }; + + struct HeaderBlockContent { + object::dos_header DOSHeader; + COFFHeaderMaterializationUnit::NTHeader NTHeader; + }; + + static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + HeaderBlockContent Hdr = {}; + + // Set up magic + Hdr.DOSHeader.Magic[0] = 'M'; + Hdr.DOSHeader.Magic[1] = 'Z'; + Hdr.DOSHeader.AddressOfNewExeHeader = + offsetof(HeaderBlockContent, NTHeader); + uint32_t PEMagic = *reinterpret_cast<const uint32_t *>(COFF::PEMagic); + Hdr.NTHeader.PEMagic = PEMagic; + Hdr.NTHeader.OptionalHeader.Header.Magic = COFF::PE32Header::PE32_PLUS; + + switch (G.getTargetTriple().getArch()) { + case Triple::x86_64: + Hdr.NTHeader.FileHeader.Machine = COFF::IMAGE_FILE_MACHINE_AMD64; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + + auto HeaderContent = G.allocateString( + StringRef(reinterpret_cast<const char *>(&Hdr), sizeof(Hdr))); + + return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, + 0); + } + + static void addImageBaseRelocationEdge(jitlink::Block &B, + jitlink::Symbol &ImageBase) { + auto ImageBaseOffset = offsetof(HeaderBlockContent, NTHeader) + + offsetof(NTHeader, OptionalHeader) + + offsetof(object::pe32plus_header, ImageBase); + B.addEdge(jitlink::x86_64::Pointer64, ImageBaseOffset, ImageBase, 0); + } + + static MaterializationUnit::Interface + createHeaderInterface(COFFPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol) { + SymbolFlagsMap HeaderSymbolFlags; + + HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; + + return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), + HeaderStartSymbol); + } + + COFFPlatform &CP; +}; + +} // end anonymous namespace + +namespace llvm { +namespace orc { + +Expected<std::unique_ptr<COFFPlatform>> +COFFPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + LoadDynamicLibrary LoadDynLibrary, bool StaticVCRuntime, + const char *VCRuntimePath, + std::optional<SymbolAliasMap> RuntimeAliases) { + auto &EPC = ES.getExecutorProcessControl(); + + // If the target is not supported then bail out immediately. + if (!supportedTarget(EPC.getTargetTriple())) + return make_error<StringError>("Unsupported COFFPlatform triple: " + + EPC.getTargetTriple().str(), + inconvertibleErrorCode()); + + // Create default aliases if the caller didn't supply any. + if (!RuntimeAliases) + RuntimeAliases = standardPlatformAliases(ES); + + // Define the aliases. + if (auto Err = PlatformJD.define(symbolAliases(std::move(*RuntimeAliases)))) + return std::move(Err); + + auto &HostFuncJD = ES.createBareJITDylib("$<PlatformRuntimeHostFuncJD>"); + + // Add JIT-dispatch function support symbols. + if (auto Err = HostFuncJD.define(absoluteSymbols( + {{ES.intern("__orc_rt_jit_dispatch"), + {EPC.getJITDispatchInfo().JITDispatchFunction.getValue(), + JITSymbolFlags::Exported}}, + {ES.intern("__orc_rt_jit_dispatch_ctx"), + {EPC.getJITDispatchInfo().JITDispatchContext.getValue(), + JITSymbolFlags::Exported}}}))) + return std::move(Err); + + PlatformJD.addToLinkOrder(HostFuncJD); + + // Create the instance. + Error Err = Error::success(); + auto P = std::unique_ptr<COFFPlatform>(new COFFPlatform( + ES, ObjLinkingLayer, PlatformJD, OrcRuntimePath, + std::move(LoadDynLibrary), StaticVCRuntime, VCRuntimePath, Err)); + if (Err) + return std::move(Err); + return std::move(P); +} + +Expected<MemoryBufferRef> COFFPlatform::getPerJDObjectFile() { + auto PerJDObj = OrcRuntimeArchive->findSym("__orc_rt_coff_per_jd_marker"); + if (!PerJDObj) + return PerJDObj.takeError(); + + if (!*PerJDObj) + return make_error<StringError>("Could not find per jd object file", + inconvertibleErrorCode()); + + auto Buffer = (*PerJDObj)->getAsBinary(); + if (!Buffer) + return Buffer.takeError(); + + return (*Buffer)->getMemoryBufferRef(); +} + +static void addAliases(ExecutionSession &ES, SymbolAliasMap &Aliases, + ArrayRef<std::pair<const char *, const char *>> AL) { + for (auto &KV : AL) { + auto AliasName = ES.intern(KV.first); + assert(!Aliases.count(AliasName) && "Duplicate symbol name in alias map"); + Aliases[std::move(AliasName)] = {ES.intern(KV.second), + JITSymbolFlags::Exported}; + } +} + +Error COFFPlatform::setupJITDylib(JITDylib &JD) { + if (auto Err = JD.define(std::make_unique<COFFHeaderMaterializationUnit>( + *this, COFFHeaderStartSymbol))) + return Err; + + if (auto Err = ES.lookup({&JD}, COFFHeaderStartSymbol).takeError()) + return Err; + + // Define the CXX aliases. + SymbolAliasMap CXXAliases; + addAliases(ES, CXXAliases, requiredCXXAliases()); + if (auto Err = JD.define(symbolAliases(std::move(CXXAliases)))) + return Err; + + auto PerJDObj = getPerJDObjectFile(); + if (!PerJDObj) + return PerJDObj.takeError(); + + auto I = getObjectFileInterface(ES, *PerJDObj); + if (!I) + return I.takeError(); + + if (auto Err = ObjLinkingLayer.add( + JD, MemoryBuffer::getMemBuffer(*PerJDObj, false), std::move(*I))) + return Err; + + if (!Bootstrapping) { + auto ImportedLibs = StaticVCRuntime + ? VCRuntimeBootstrap->loadStaticVCRuntime(JD) + : VCRuntimeBootstrap->loadDynamicVCRuntime(JD); + if (!ImportedLibs) + return ImportedLibs.takeError(); + for (auto &Lib : *ImportedLibs) + if (auto Err = LoadDynLibrary(JD, Lib)) + return Err; + if (StaticVCRuntime) + if (auto Err = VCRuntimeBootstrap->initializeStaticVCRuntime(JD)) + return Err; + } + + JD.addGenerator(DLLImportDefinitionGenerator::Create(ES, ObjLinkingLayer)); + return Error::success(); +} + +Error COFFPlatform::teardownJITDylib(JITDylib &JD) { + std::lock_guard<std::mutex> Lock(PlatformMutex); + auto I = JITDylibToHeaderAddr.find(&JD); + if (I != JITDylibToHeaderAddr.end()) { + assert(HeaderAddrToJITDylib.count(I->second) && + "HeaderAddrToJITDylib missing entry"); + HeaderAddrToJITDylib.erase(I->second); + JITDylibToHeaderAddr.erase(I); + } + return Error::success(); +} + +Error COFFPlatform::notifyAdding(ResourceTracker &RT, + const MaterializationUnit &MU) { + auto &JD = RT.getJITDylib(); + const auto &InitSym = MU.getInitializerSymbol(); + if (!InitSym) + return Error::success(); + + RegisteredInitSymbols[&JD].add(InitSym, + SymbolLookupFlags::WeaklyReferencedSymbol); + + LLVM_DEBUG({ + dbgs() << "COFFPlatform: Registered init symbol " << *InitSym << " for MU " + << MU.getName() << "\n"; + }); + return Error::success(); +} + +Error COFFPlatform::notifyRemoving(ResourceTracker &RT) { + llvm_unreachable("Not supported yet"); +} + +SymbolAliasMap COFFPlatform::standardPlatformAliases(ExecutionSession &ES) { + SymbolAliasMap Aliases; + addAliases(ES, Aliases, standardRuntimeUtilityAliases()); + return Aliases; +} + +ArrayRef<std::pair<const char *, const char *>> +COFFPlatform::requiredCXXAliases() { + static const std::pair<const char *, const char *> RequiredCXXAliases[] = { + {"_CxxThrowException", "__orc_rt_coff_cxx_throw_exception"}, + {"_onexit", "__orc_rt_coff_onexit_per_jd"}, + {"atexit", "__orc_rt_coff_atexit_per_jd"}}; + + return ArrayRef<std::pair<const char *, const char *>>(RequiredCXXAliases); +} + +ArrayRef<std::pair<const char *, const char *>> +COFFPlatform::standardRuntimeUtilityAliases() { + static const std::pair<const char *, const char *> + StandardRuntimeUtilityAliases[] = { + {"__orc_rt_run_program", "__orc_rt_coff_run_program"}, + {"__orc_rt_jit_dlerror", "__orc_rt_coff_jit_dlerror"}, + {"__orc_rt_jit_dlopen", "__orc_rt_coff_jit_dlopen"}, + {"__orc_rt_jit_dlclose", "__orc_rt_coff_jit_dlclose"}, + {"__orc_rt_jit_dlsym", "__orc_rt_coff_jit_dlsym"}, + {"__orc_rt_log_error", "__orc_rt_log_error_to_stderr"}}; + + return ArrayRef<std::pair<const char *, const char *>>( + StandardRuntimeUtilityAliases); +} + +bool COFFPlatform::supportedTarget(const Triple &TT) { + switch (TT.getArch()) { + case Triple::x86_64: + return true; + default: + return false; + } +} + +COFFPlatform::COFFPlatform(ExecutionSession &ES, + ObjectLinkingLayer &ObjLinkingLayer, + JITDylib &PlatformJD, const char *OrcRuntimePath, + LoadDynamicLibrary LoadDynLibrary, + bool StaticVCRuntime, const char *VCRuntimePath, + Error &Err) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer), + LoadDynLibrary(std::move(LoadDynLibrary)), + StaticVCRuntime(StaticVCRuntime), + COFFHeaderStartSymbol(ES.intern("__ImageBase")) { + ErrorAsOutParameter _(&Err); + + // Create a generator for the ORC runtime archive. + auto OrcRuntimeArchiveGenerator = + StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, OrcRuntimePath); + if (!OrcRuntimeArchiveGenerator) { + Err = OrcRuntimeArchiveGenerator.takeError(); + return; + } + + auto ArchiveBuffer = MemoryBuffer::getFile(OrcRuntimePath); + if (!ArchiveBuffer) { + Err = createFileError(OrcRuntimePath, ArchiveBuffer.getError()); + return; + } + OrcRuntimeArchiveBuffer = std::move(*ArchiveBuffer); + OrcRuntimeArchive = + std::make_unique<object::Archive>(*OrcRuntimeArchiveBuffer, Err); + if (Err) + return; + + Bootstrapping.store(true); + ObjLinkingLayer.addPlugin(std::make_unique<COFFPlatformPlugin>(*this)); + + // Load vc runtime + auto VCRT = + COFFVCRuntimeBootstrapper::Create(ES, ObjLinkingLayer, VCRuntimePath); + if (!VCRT) { + Err = VCRT.takeError(); + return; + } + VCRuntimeBootstrap = std::move(*VCRT); + + for (auto &Lib : (*OrcRuntimeArchiveGenerator)->getImportedDynamicLibraries()) + DylibsToPreload.insert(Lib); + + auto ImportedLibs = + StaticVCRuntime ? VCRuntimeBootstrap->loadStaticVCRuntime(PlatformJD) + : VCRuntimeBootstrap->loadDynamicVCRuntime(PlatformJD); + if (!ImportedLibs) { + Err = ImportedLibs.takeError(); + return; + } + + for (auto &Lib : *ImportedLibs) + DylibsToPreload.insert(Lib); + + PlatformJD.addGenerator(std::move(*OrcRuntimeArchiveGenerator)); + + // PlatformJD hasn't been set up by the platform yet (since we're creating + // the platform now), so set it up. + if (auto E2 = setupJITDylib(PlatformJD)) { + Err = std::move(E2); + return; + } + + for (auto& Lib : DylibsToPreload) + if (auto E2 = LoadDynLibrary(PlatformJD, Lib)) { + Err = std::move(E2); + return; + } + + if (StaticVCRuntime) + if (auto E2 = VCRuntimeBootstrap->initializeStaticVCRuntime(PlatformJD)) { + Err = std::move(E2); + return; + } + + // Associate wrapper function tags with JIT-side function implementations. + if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { + Err = std::move(E2); + return; + } + + // Lookup addresses of runtime functions callable by the platform, + // call the platform bootstrap function to initialize the platform-state + // object in the executor. + if (auto E2 = bootstrapCOFFRuntime(PlatformJD)) { + Err = std::move(E2); + return; + } + + Bootstrapping.store(false); + JDBootstrapStates.clear(); +} + +Expected<COFFPlatform::JITDylibDepMap> +COFFPlatform::buildJDDepMap(JITDylib &JD) { + return ES.runSessionLocked([&]() -> Expected<JITDylibDepMap> { + JITDylibDepMap JDDepMap; + + SmallVector<JITDylib *, 16> Worklist({&JD}); + while (!Worklist.empty()) { + auto CurJD = Worklist.back(); + Worklist.pop_back(); + + auto &DM = JDDepMap[CurJD]; + CurJD->withLinkOrderDo([&](const JITDylibSearchOrder &O) { + DM.reserve(O.size()); + for (auto &KV : O) { + if (KV.first == CurJD) + continue; + { + // Bare jitdylibs not known to the platform + std::lock_guard<std::mutex> Lock(PlatformMutex); + if (!JITDylibToHeaderAddr.count(KV.first)) { + LLVM_DEBUG({ + dbgs() << "JITDylib unregistered to COFFPlatform detected in " + "LinkOrder: " + << CurJD->getName() << "\n"; + }); + continue; + } + } + DM.push_back(KV.first); + // Push unvisited entry. + if (!JDDepMap.count(KV.first)) { + Worklist.push_back(KV.first); + JDDepMap[KV.first] = {}; + } + } + }); + } + return std::move(JDDepMap); + }); +} + +void COFFPlatform::pushInitializersLoop(PushInitializersSendResultFn SendResult, + JITDylibSP JD, + JITDylibDepMap &JDDepMap) { + SmallVector<JITDylib *, 16> Worklist({JD.get()}); + DenseSet<JITDylib *> Visited({JD.get()}); + DenseMap<JITDylib *, SymbolLookupSet> NewInitSymbols; + ES.runSessionLocked([&]() { + while (!Worklist.empty()) { + auto CurJD = Worklist.back(); + Worklist.pop_back(); + + auto RISItr = RegisteredInitSymbols.find(CurJD); + if (RISItr != RegisteredInitSymbols.end()) { + NewInitSymbols[CurJD] = std::move(RISItr->second); + RegisteredInitSymbols.erase(RISItr); + } + + for (auto *DepJD : JDDepMap[CurJD]) + if (!Visited.count(DepJD)) { + Worklist.push_back(DepJD); + Visited.insert(DepJD); + } + } + }); + + // If there are no further init symbols to look up then send the link order + // (as a list of header addresses) to the caller. + if (NewInitSymbols.empty()) { + // Build the dep info map to return. + COFFJITDylibDepInfoMap DIM; + DIM.reserve(JDDepMap.size()); + for (auto &KV : JDDepMap) { + std::lock_guard<std::mutex> Lock(PlatformMutex); + COFFJITDylibDepInfo DepInfo; + DepInfo.reserve(KV.second.size()); + for (auto &Dep : KV.second) { + DepInfo.push_back(JITDylibToHeaderAddr[Dep]); + } + auto H = JITDylibToHeaderAddr[KV.first]; + DIM.push_back(std::make_pair(H, std::move(DepInfo))); + } + SendResult(DIM); + return; + } + + // Otherwise issue a lookup and re-run this phase when it completes. + lookupInitSymbolsAsync( + [this, SendResult = std::move(SendResult), &JD, + JDDepMap = std::move(JDDepMap)](Error Err) mutable { + if (Err) + SendResult(std::move(Err)); + else + pushInitializersLoop(std::move(SendResult), JD, JDDepMap); + }, + ES, std::move(NewInitSymbols)); +} + +void COFFPlatform::rt_pushInitializers(PushInitializersSendResultFn SendResult, + ExecutorAddr JDHeaderAddr) { + JITDylibSP JD; + { + std::lock_guard<std::mutex> Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(JDHeaderAddr); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + LLVM_DEBUG({ + dbgs() << "COFFPlatform::rt_pushInitializers(" << JDHeaderAddr << ") "; + if (JD) + dbgs() << "pushing initializers for " << JD->getName() << "\n"; + else + dbgs() << "No JITDylib for header address.\n"; + }); + + if (!JD) { + SendResult( + make_error<StringError>("No JITDylib with header addr " + + formatv("{0:x}", JDHeaderAddr.getValue()), + inconvertibleErrorCode())); + return; + } + + auto JDDepMap = buildJDDepMap(*JD); + if (!JDDepMap) { + SendResult(JDDepMap.takeError()); + return; + } + + pushInitializersLoop(std::move(SendResult), JD, *JDDepMap); +} + +void COFFPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, + ExecutorAddr Handle, StringRef SymbolName) { + LLVM_DEBUG({ + dbgs() << "COFFPlatform::rt_lookupSymbol(\"" + << formatv("{0:x}", Handle.getValue()) << "\")\n"; + }); + + JITDylib *JD = nullptr; + + { + std::lock_guard<std::mutex> Lock(PlatformMutex); + auto I = HeaderAddrToJITDylib.find(Handle); + if (I != HeaderAddrToJITDylib.end()) + JD = I->second; + } + + if (!JD) { + LLVM_DEBUG({ + dbgs() << " No JITDylib for handle " + << formatv("{0:x}", Handle.getValue()) << "\n"; + }); + SendResult(make_error<StringError>("No JITDylib associated with handle " + + formatv("{0:x}", Handle.getValue()), + inconvertibleErrorCode())); + return; + } + + // Use functor class to work around XL build compiler issue on AIX. + class RtLookupNotifyComplete { + public: + RtLookupNotifyComplete(SendSymbolAddressFn &&SendResult) + : SendResult(std::move(SendResult)) {} + void operator()(Expected<SymbolMap> Result) { + if (Result) { + assert(Result->size() == 1 && "Unexpected result map count"); + SendResult(ExecutorAddr(Result->begin()->second.getAddress())); + } else { + SendResult(Result.takeError()); + } + } + + private: + SendSymbolAddressFn SendResult; + }; + + ES.lookup( + LookupKind::DLSym, {{JD, JITDylibLookupFlags::MatchExportedSymbolsOnly}}, + SymbolLookupSet(ES.intern(SymbolName)), SymbolState::Ready, + RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister); +} + +Error COFFPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { + ExecutionSession::JITDispatchHandlerAssociationMap WFs; + + using LookupSymbolSPSSig = + SPSExpected<SPSExecutorAddr>(SPSExecutorAddr, SPSString); + WFs[ES.intern("__orc_rt_coff_symbol_lookup_tag")] = + ES.wrapAsyncWithSPS<LookupSymbolSPSSig>(this, + &COFFPlatform::rt_lookupSymbol); + using PushInitializersSPSSig = + SPSExpected<SPSCOFFJITDylibDepInfoMap>(SPSExecutorAddr); + WFs[ES.intern("__orc_rt_coff_push_initializers_tag")] = + ES.wrapAsyncWithSPS<PushInitializersSPSSig>( + this, &COFFPlatform::rt_pushInitializers); + + return ES.registerJITDispatchHandlers(PlatformJD, std::move(WFs)); +} + +Error COFFPlatform::runBootstrapInitializers(JDBootstrapState &BState) { + llvm::sort(BState.Initializers); + if (auto Err = + runBootstrapSubsectionInitializers(BState, ".CRT$XIA", ".CRT$XIZ")) + return Err; + + if (auto Err = runSymbolIfExists(*BState.JD, "__run_after_c_init")) + return Err; + + if (auto Err = + runBootstrapSubsectionInitializers(BState, ".CRT$XCA", ".CRT$XCZ")) + return Err; + return Error::success(); +} + +Error COFFPlatform::runBootstrapSubsectionInitializers(JDBootstrapState &BState, + StringRef Start, + StringRef End) { + for (auto &Initializer : BState.Initializers) + if (Initializer.first >= Start && Initializer.first <= End && + Initializer.second) { + auto Res = + ES.getExecutorProcessControl().runAsVoidFunction(Initializer.second); + if (!Res) + return Res.takeError(); + } + return Error::success(); +} + +Error COFFPlatform::bootstrapCOFFRuntime(JITDylib &PlatformJD) { + // Lookup of runtime symbols causes the collection of initializers if + // it's static linking setting. + if (auto Err = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), + { + {ES.intern("__orc_rt_coff_platform_bootstrap"), + &orc_rt_coff_platform_bootstrap}, + {ES.intern("__orc_rt_coff_platform_shutdown"), + &orc_rt_coff_platform_shutdown}, + {ES.intern("__orc_rt_coff_register_jitdylib"), + &orc_rt_coff_register_jitdylib}, + {ES.intern("__orc_rt_coff_deregister_jitdylib"), + &orc_rt_coff_deregister_jitdylib}, + {ES.intern("__orc_rt_coff_register_object_sections"), + &orc_rt_coff_register_object_sections}, + {ES.intern("__orc_rt_coff_deregister_object_sections"), + &orc_rt_coff_deregister_object_sections}, + })) + return Err; + + // Call bootstrap functions + if (auto Err = ES.callSPSWrapper<void()>(orc_rt_coff_platform_bootstrap)) + return Err; + + // Do the pending jitdylib registration actions that we couldn't do + // because orc runtime was not linked fully. + for (auto KV : JDBootstrapStates) { + auto &JDBState = KV.second; + if (auto Err = ES.callSPSWrapper<void(SPSString, SPSExecutorAddr)>( + orc_rt_coff_register_jitdylib, JDBState.JDName, + JDBState.HeaderAddr)) + return Err; + + for (auto &ObjSectionMap : JDBState.ObjectSectionsMaps) + if (auto Err = ES.callSPSWrapper<void(SPSExecutorAddr, + SPSCOFFObjectSectionsMap, bool)>( + orc_rt_coff_register_object_sections, JDBState.HeaderAddr, + ObjSectionMap, false)) + return Err; + } + + // Run static initializers collected in bootstrap stage. + for (auto KV : JDBootstrapStates) { + auto &JDBState = KV.second; + if (auto Err = runBootstrapInitializers(JDBState)) + return Err; + } + + return Error::success(); +} + +Error COFFPlatform::runSymbolIfExists(JITDylib &PlatformJD, + StringRef SymbolName) { + ExecutorAddr jit_function; + auto AfterCLookupErr = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), + {{ES.intern(SymbolName), &jit_function}}); + if (!AfterCLookupErr) { + auto Res = ES.getExecutorProcessControl().runAsVoidFunction(jit_function); + if (!Res) + return Res.takeError(); + return Error::success(); + } + if (!AfterCLookupErr.isA<SymbolsNotFound>()) + return AfterCLookupErr; + consumeError(std::move(AfterCLookupErr)); + return Error::success(); +} + +void COFFPlatform::COFFPlatformPlugin::modifyPassConfig( + MaterializationResponsibility &MR, jitlink::LinkGraph &LG, + jitlink::PassConfiguration &Config) { + + bool IsBootstrapping = CP.Bootstrapping.load(); + + if (auto InitSymbol = MR.getInitializerSymbol()) { + if (InitSymbol == CP.COFFHeaderStartSymbol) { + Config.PostAllocationPasses.push_back( + [this, &MR, IsBootstrapping](jitlink::LinkGraph &G) { + return associateJITDylibHeaderSymbol(G, MR, IsBootstrapping); + }); + return; + } + Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { + return preserveInitializerSections(G, MR); + }); + } + + if (!IsBootstrapping) + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerObjectPlatformSections(G, JD); + }); + else + Config.PostFixupPasses.push_back( + [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + return registerObjectPlatformSectionsInBootstrap(G, JD); + }); +} + +ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap +COFFPlatform::COFFPlatformPlugin::getSyntheticSymbolDependencies( + MaterializationResponsibility &MR) { + std::lock_guard<std::mutex> Lock(PluginMutex); + auto I = InitSymbolDeps.find(&MR); + if (I != InitSymbolDeps.end()) { + SyntheticSymbolDependenciesMap Result; + Result[MR.getInitializerSymbol()] = std::move(I->second); + InitSymbolDeps.erase(&MR); + return Result; + } + return SyntheticSymbolDependenciesMap(); +} + +Error COFFPlatform::COFFPlatformPlugin::associateJITDylibHeaderSymbol( + jitlink::LinkGraph &G, MaterializationResponsibility &MR, + bool IsBootstraping) { + auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { + return Sym->getName() == *CP.COFFHeaderStartSymbol; + }); + assert(I != G.defined_symbols().end() && "Missing COFF header start symbol"); + + auto &JD = MR.getTargetJITDylib(); + std::lock_guard<std::mutex> Lock(CP.PlatformMutex); + auto HeaderAddr = (*I)->getAddress(); + CP.JITDylibToHeaderAddr[&JD] = HeaderAddr; + CP.HeaderAddrToJITDylib[HeaderAddr] = &JD; + if (!IsBootstraping) { + G.allocActions().push_back( + {cantFail(WrapperFunctionCall::Create< + SPSArgList<SPSString, SPSExecutorAddr>>( + CP.orc_rt_coff_register_jitdylib, JD.getName(), HeaderAddr)), + cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( + CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))}); + } else { + G.allocActions().push_back( + {{}, + cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( + CP.orc_rt_coff_deregister_jitdylib, HeaderAddr))}); + JDBootstrapState BState; + BState.JD = &JD; + BState.JDName = JD.getName(); + BState.HeaderAddr = HeaderAddr; + CP.JDBootstrapStates.emplace(&JD, BState); + } + + return Error::success(); +} + +Error COFFPlatform::COFFPlatformPlugin::registerObjectPlatformSections( + jitlink::LinkGraph &G, JITDylib &JD) { + COFFObjectSectionsMap ObjSecs; + auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD]; + assert(HeaderAddr && "Must be registered jitdylib"); + for (auto &S : G.sections()) { + jitlink::SectionRange Range(S); + if (Range.getSize()) + ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange())); + } + + G.allocActions().push_back( + {cantFail(WrapperFunctionCall::Create<SPSCOFFRegisterObjectSectionsArgs>( + CP.orc_rt_coff_register_object_sections, HeaderAddr, ObjSecs, true)), + cantFail( + WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>( + CP.orc_rt_coff_deregister_object_sections, HeaderAddr, + ObjSecs))}); + + return Error::success(); +} + +Error COFFPlatform::COFFPlatformPlugin::preserveInitializerSections( + jitlink::LinkGraph &G, MaterializationResponsibility &MR) { + JITLinkSymbolSet InitSectionSymbols; + for (auto &Sec : G.sections()) + if (COFFPlatform::isInitializerSection(Sec.getName())) + for (auto *B : Sec.blocks()) + if (!B->edges_empty()) + InitSectionSymbols.insert( + &G.addAnonymousSymbol(*B, 0, 0, false, true)); + + std::lock_guard<std::mutex> Lock(PluginMutex); + InitSymbolDeps[&MR] = InitSectionSymbols; + return Error::success(); +} + +Error COFFPlatform::COFFPlatformPlugin:: + registerObjectPlatformSectionsInBootstrap(jitlink::LinkGraph &G, + JITDylib &JD) { + std::lock_guard<std::mutex> Lock(CP.PlatformMutex); + auto HeaderAddr = CP.JITDylibToHeaderAddr[&JD]; + COFFObjectSectionsMap ObjSecs; + for (auto &S : G.sections()) { + jitlink::SectionRange Range(S); + if (Range.getSize()) + ObjSecs.push_back(std::make_pair(S.getName().str(), Range.getRange())); + } + + G.allocActions().push_back( + {{}, + cantFail( + WrapperFunctionCall::Create<SPSCOFFDeregisterObjectSectionsArgs>( + CP.orc_rt_coff_deregister_object_sections, HeaderAddr, + ObjSecs))}); + + auto &BState = CP.JDBootstrapStates[&JD]; + BState.ObjectSectionsMaps.push_back(std::move(ObjSecs)); + + // Collect static initializers + for (auto &S : G.sections()) + if (COFFPlatform::isInitializerSection(S.getName())) + for (auto *B : S.blocks()) { + if (B->edges_empty()) + continue; + for (auto &E : B->edges()) + BState.Initializers.push_back(std::make_pair( + S.getName().str(), + ExecutorAddr(E.getTarget().getAddress() + E.getAddend()))); + } + + return Error::success(); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp new file mode 100644 index 000000000000..d9316fab2de3 --- /dev/null +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/COFFVCRuntimeSupport.cpp @@ -0,0 +1,184 @@ +//===------- COFFVCRuntimeSupport.cpp - VC runtime support in ORC ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/COFFVCRuntimeSupport.h" + +#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/Support/VirtualFileSystem.h" +#include "llvm/WindowsDriver/MSVCPaths.h" + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::orc; +using namespace llvm::orc::shared; + +Expected<std::unique_ptr<COFFVCRuntimeBootstrapper>> +COFFVCRuntimeBootstrapper::Create(ExecutionSession &ES, + ObjectLinkingLayer &ObjLinkingLayer, + const char *RuntimePath) { + return std::unique_ptr<COFFVCRuntimeBootstrapper>( + new COFFVCRuntimeBootstrapper(ES, ObjLinkingLayer, RuntimePath)); +} + +COFFVCRuntimeBootstrapper::COFFVCRuntimeBootstrapper( + ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, + const char *RuntimePath) + : ES(ES), ObjLinkingLayer(ObjLinkingLayer) { + if (RuntimePath) + this->RuntimePath = RuntimePath; +} + +Expected<std::vector<std::string>> +COFFVCRuntimeBootstrapper::loadStaticVCRuntime(JITDylib &JD, + bool DebugVersion) { + StringRef VCLibs[] = {"libvcruntime.lib", "libcmt.lib", "libcpmt.lib"}; + StringRef UCRTLibs[] = {"libucrt.lib"}; + std::vector<std::string> ImportedLibraries; + if (auto Err = loadVCRuntime(JD, ImportedLibraries, ArrayRef(VCLibs), + ArrayRef(UCRTLibs))) + return std::move(Err); + return ImportedLibraries; +} + +Expected<std::vector<std::string>> +COFFVCRuntimeBootstrapper::loadDynamicVCRuntime(JITDylib &JD, + bool DebugVersion) { + StringRef VCLibs[] = {"vcruntime.lib", "msvcrt.lib", "msvcprt.lib"}; + StringRef UCRTLibs[] = {"ucrt.lib"}; + std::vector<std::string> ImportedLibraries; + if (auto Err = loadVCRuntime(JD, ImportedLibraries, ArrayRef(VCLibs), + ArrayRef(UCRTLibs))) + return std::move(Err); + return ImportedLibraries; +} + +Error COFFVCRuntimeBootstrapper::loadVCRuntime( + JITDylib &JD, std::vector<std::string> &ImportedLibraries, + ArrayRef<StringRef> VCLibs, ArrayRef<StringRef> UCRTLibs) { + MSVCToolchainPath Path; + if (!RuntimePath.empty()) { + Path.UCRTSdkLib = RuntimePath; + Path.VCToolchainLib = RuntimePath; + } else { + auto ToolchainPath = getMSVCToolchainPath(); + if (!ToolchainPath) + return ToolchainPath.takeError(); + Path = *ToolchainPath; + } + LLVM_DEBUG({ + dbgs() << "Using VC toolchain pathes\n"; + dbgs() << " VC toolchain path: " << Path.VCToolchainLib << "\n"; + dbgs() << " UCRT path: " << Path.UCRTSdkLib << "\n"; + }); + + auto LoadLibrary = [&](SmallString<256> LibPath, StringRef LibName) -> Error { + sys::path::append(LibPath, LibName); + + auto G = StaticLibraryDefinitionGenerator::Load(ObjLinkingLayer, + LibPath.c_str()); + if (!G) + return G.takeError(); + + for (auto &Lib : (*G)->getImportedDynamicLibraries()) + ImportedLibraries.push_back(Lib); + + JD.addGenerator(std::move(*G)); + + return Error::success(); + }; + for (auto &Lib : UCRTLibs) + if (auto Err = LoadLibrary(Path.UCRTSdkLib, Lib)) + return Err; + + for (auto &Lib : VCLibs) + if (auto Err = LoadLibrary(Path.VCToolchainLib, Lib)) + return Err; + ImportedLibraries.push_back("ntdll.dll"); + ImportedLibraries.push_back("Kernel32.dll"); + + return Error::success(); +} + +Error COFFVCRuntimeBootstrapper::initializeStaticVCRuntime(JITDylib &JD) { + ExecutorAddr jit_scrt_initialize, jit_scrt_dllmain_before_initialize_c, + jit_scrt_initialize_type_info, + jit_scrt_initialize_default_local_stdio_options; + if (auto Err = lookupAndRecordAddrs( + ES, LookupKind::Static, makeJITDylibSearchOrder(&JD), + {{ES.intern("__scrt_initialize_crt"), &jit_scrt_initialize}, + {ES.intern("__scrt_dllmain_before_initialize_c"), + &jit_scrt_dllmain_before_initialize_c}, + {ES.intern("?__scrt_initialize_type_info@@YAXXZ"), + &jit_scrt_initialize_type_info}, + {ES.intern("__scrt_initialize_default_local_stdio_options"), + &jit_scrt_initialize_default_local_stdio_options}})) + return Err; + + auto RunVoidInitFunc = [&](ExecutorAddr Addr) -> Error { + if (auto Res = ES.getExecutorProcessControl().runAsVoidFunction(Addr)) + return Error::success(); + else + return Res.takeError(); + }; + + auto R = + ES.getExecutorProcessControl().runAsIntFunction(jit_scrt_initialize, 0); + if (!R) + return R.takeError(); + + if (auto Err = RunVoidInitFunc(jit_scrt_dllmain_before_initialize_c)) + return Err; + + if (auto Err = RunVoidInitFunc(jit_scrt_initialize_type_info)) + return Err; + + if (auto Err = + RunVoidInitFunc(jit_scrt_initialize_default_local_stdio_options)) + return Err; + + SymbolAliasMap Alias; + Alias[ES.intern("__run_after_c_init")] = { + ES.intern("__scrt_dllmain_after_initialize_c"), JITSymbolFlags::Exported}; + if (auto Err = JD.define(symbolAliases(Alias))) + return Err; + + return Error::success(); +} + +Expected<COFFVCRuntimeBootstrapper::MSVCToolchainPath> +COFFVCRuntimeBootstrapper::getMSVCToolchainPath() { + std::string VCToolChainPath; + ToolsetLayout VSLayout; + IntrusiveRefCntPtr<vfs::FileSystem> VFS = vfs::getRealFileSystem(); + if (!findVCToolChainViaCommandLine(*VFS, std::nullopt, std::nullopt, + std::nullopt, VCToolChainPath, VSLayout) && + !findVCToolChainViaEnvironment(*VFS, VCToolChainPath, VSLayout) && + !findVCToolChainViaSetupConfig(*VFS, VCToolChainPath, VSLayout) && + !findVCToolChainViaRegistry(VCToolChainPath, VSLayout)) + return make_error<StringError>("Couldn't find msvc toolchain.", + inconvertibleErrorCode()); + + std::string UniversalCRTSdkPath; + std::string UCRTVersion; + if (!getUniversalCRTSdkDir(*VFS, std::nullopt, std::nullopt, std::nullopt, + UniversalCRTSdkPath, UCRTVersion)) + return make_error<StringError>("Couldn't find universal sdk.", + inconvertibleErrorCode()); + + MSVCToolchainPath ToolchainPath; + SmallString<256> VCToolchainLib(VCToolChainPath); + sys::path::append(VCToolchainLib, "lib", "x64"); + ToolchainPath.VCToolchainLib = VCToolchainLib; + + SmallString<256> UCRTSdkLib(UniversalCRTSdkPath); + sys::path::append(UCRTSdkLib, "Lib", UCRTVersion, "ucrt", "x64"); + ToolchainPath.UCRTSdkLib = UCRTSdkLib; + return ToolchainPath; +} diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index e2a0cadb6348..6448adaa0ceb 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -102,14 +102,14 @@ private: CompileOnDemandLayer &Parent; }; -Optional<CompileOnDemandLayer::GlobalValueSet> +std::optional<CompileOnDemandLayer::GlobalValueSet> CompileOnDemandLayer::compileRequested(GlobalValueSet Requested) { return std::move(Requested); } -Optional<CompileOnDemandLayer::GlobalValueSet> +std::optional<CompileOnDemandLayer::GlobalValueSet> CompileOnDemandLayer::compileWholeModule(GlobalValueSet Requested) { - return None; + return std::nullopt; } CompileOnDemandLayer::CompileOnDemandLayer( @@ -237,7 +237,7 @@ void CompileOnDemandLayer::expandPartition(GlobalValueSet &Partition) { bool ContainsGlobalVariables = false; std::vector<const GlobalValue *> GVsToAdd; - for (auto *GV : Partition) + for (const auto *GV : Partition) if (isa<GlobalAlias>(GV)) GVsToAdd.push_back( cast<GlobalValue>(cast<GlobalAlias>(GV)->getAliasee())); @@ -252,7 +252,7 @@ void CompileOnDemandLayer::expandPartition(GlobalValueSet &Partition) { for (auto &G : M.globals()) GVsToAdd.push_back(&G); - for (auto *GV : GVsToAdd) + for (const auto *GV : GVsToAdd) Partition.insert(GV); } @@ -287,7 +287,7 @@ void CompileOnDemandLayer::emitPartition( // Take a 'None' partition to mean the whole module (as opposed to an empty // partition, which means "materialize nothing"). Emit the whole module // unmodified to the base layer. - if (GVsToExtract == None) { + if (GVsToExtract == std::nullopt) { Defs.clear(); BaseLayer.emit(std::move(R), std::move(TSM)); return; @@ -336,13 +336,13 @@ void CompileOnDemandLayer::emitPartition( { std::vector<const GlobalValue*> HashGVs; HashGVs.reserve(GVsToExtract->size()); - for (auto *GV : *GVsToExtract) + for (const auto *GV : *GVsToExtract) HashGVs.push_back(GV); llvm::sort(HashGVs, [](const GlobalValue *LHS, const GlobalValue *RHS) { return LHS->getName() < RHS->getName(); }); hash_code HC(0); - for (auto *GV : HashGVs) { + for (const auto *GV : HashGVs) { assert(GV->hasName() && "All GVs to extract should be named by now"); auto GVName = GV->getName(); HC = hash_combine(HC, hash_combine_range(GVName.begin(), GVName.end())); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp index dd80630a33c1..4a9d0d470a8e 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -17,6 +17,7 @@ #include <condition_variable> #include <future> +#include <optional> #define DEBUG_TYPE "orc" @@ -970,10 +971,9 @@ Error JITDylib::resolve(MaterializationResponsibility &MR, SymbolsInErrorState.insert(KV.first); else { auto Flags = KV.second.getFlags(); - Flags &= ~(JITSymbolFlags::Weak | JITSymbolFlags::Common); + Flags &= ~JITSymbolFlags::Common; assert(Flags == - (SymI->second.getFlags() & - ~(JITSymbolFlags::Weak | JITSymbolFlags::Common)) && + (SymI->second.getFlags() & ~JITSymbolFlags::Common) && "Resolved flags should match the declared flags"); Worklist.push_back( @@ -1482,8 +1482,8 @@ void JITDylib::dump(raw_ostream &OS) { void JITDylib::MaterializingInfo::addQuery( std::shared_ptr<AsynchronousSymbolQuery> Q) { - auto I = std::lower_bound( - PendingQueries.rbegin(), PendingQueries.rend(), Q->getRequiredState(), + auto I = llvm::lower_bound( + llvm::reverse(PendingQueries), Q->getRequiredState(), [](const std::shared_ptr<AsynchronousSymbolQuery> &V, SymbolState S) { return V->getRequiredState() <= S; }); @@ -2244,8 +2244,8 @@ void ExecutionSession::dump(raw_ostream &OS) { void ExecutionSession::dispatchOutstandingMUs() { LLVM_DEBUG(dbgs() << "Dispatching MaterializationUnits...\n"); while (true) { - Optional<std::pair<std::unique_ptr<MaterializationUnit>, - std::unique_ptr<MaterializationResponsibility>>> + std::optional<std::pair<std::unique_ptr<MaterializationUnit>, + std::unique_ptr<MaterializationResponsibility>>> JMU; { @@ -2285,9 +2285,10 @@ Error ExecutionSession::removeResourceTracker(ResourceTracker &RT) { Error Err = Error::success(); + auto &JD = RT.getJITDylib(); for (auto *L : reverse(CurrentResourceManagers)) - Err = - joinErrors(std::move(Err), L->handleRemoveResources(RT.getKeyUnsafe())); + Err = joinErrors(std::move(Err), + L->handleRemoveResources(JD, RT.getKeyUnsafe())); for (auto &Q : QueriesToFail) Q->handleFailed( @@ -2316,7 +2317,8 @@ void ExecutionSession::transferResourceTracker(ResourceTracker &DstRT, auto &JD = DstRT.getJITDylib(); JD.transferTracker(DstRT, SrcRT); for (auto *L : reverse(ResourceManagers)) - L->handleTransferResources(DstRT.getKeyUnsafe(), SrcRT.getKeyUnsafe()); + L->handleTransferResources(JD, DstRT.getKeyUnsafe(), + SrcRT.getKeyUnsafe()); }); } @@ -2427,7 +2429,7 @@ void ExecutionSession::OL_applyQueryPhase1( // Add any non-candidates from the last JITDylib (if any) back on to the // list of definition candidates for this JITDylib, reset definition - // non-candiates to the empty set. + // non-candidates to the empty set. SymbolLookupSet Tmp; std::swap(IPLS->DefGeneratorNonCandidates, Tmp); IPLS->DefGeneratorCandidates.append(std::move(Tmp)); @@ -2909,13 +2911,13 @@ Error ExecutionSession::OL_notifyResolved(MaterializationResponsibility &MR, }); #ifndef NDEBUG for (auto &KV : Symbols) { - auto WeakFlags = JITSymbolFlags::Weak | JITSymbolFlags::Common; auto I = MR.SymbolFlags.find(KV.first); assert(I != MR.SymbolFlags.end() && "Resolving symbol outside this responsibility set"); assert(!I->second.hasMaterializationSideEffectsOnly() && "Can't resolve materialization-side-effects-only symbol"); - assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) && + assert((KV.second.getFlags() & ~JITSymbolFlags::Common) == + (I->second & ~JITSymbolFlags::Common) && "Resolving symbol with incorrect flags"); } #endif diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp index 1e68ea1225e6..02c3e617df68 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebugObjectManagerPlugin.cpp @@ -289,6 +289,9 @@ ELFDebugObject::CreateArchType(MemoryBufferRef Buffer, continue; HasDwarfSection |= isDwarfSection(*Name); + if (!(Header.sh_flags & ELF::SHF_ALLOC)) + continue; + auto Wrapped = std::make_unique<ELFDebugObjectSection<ELFT>>(&Header); if (Error Err = DebugObj->recordSection(*Name, std::move(Wrapped))) return std::move(Err); @@ -370,7 +373,9 @@ Error ELFDebugObject::recordSection( return Err; auto ItInserted = Sections.try_emplace(Name, std::move(Section)); if (!ItInserted.second) - return make_error<StringError>("Duplicate section", + return make_error<StringError>("In " + Buffer->getBufferIdentifier() + + ", encountered duplicate section \"" + + Name + "\" while building debug object", inconvertibleErrorCode()); return Error::success(); } @@ -487,7 +492,8 @@ Error DebugObjectManagerPlugin::notifyFailed( return Error::success(); } -void DebugObjectManagerPlugin::notifyTransferringResources(ResourceKey DstKey, +void DebugObjectManagerPlugin::notifyTransferringResources(JITDylib &JD, + ResourceKey DstKey, ResourceKey SrcKey) { // Debug objects are stored by ResourceKey only after registration. // Thus, pending objects don't need to be updated here. @@ -502,7 +508,8 @@ void DebugObjectManagerPlugin::notifyTransferringResources(ResourceKey DstKey, } } -Error DebugObjectManagerPlugin::notifyRemovingResources(ResourceKey Key) { +Error DebugObjectManagerPlugin::notifyRemovingResources(JITDylib &JD, + ResourceKey Key) { // Removing the resource for a pending object fails materialization, so they // get cleaned up in the notifyFailed() handler. std::lock_guard<std::mutex> Lock(RegisteredObjsLock); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp index 3c44fe81b4a9..15e7ffb2f75a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/DebuggerSupportPlugin.cpp @@ -137,7 +137,7 @@ public: SmallVector<DebugSectionInfo, 12> DebugSecInfos; size_t NumSections = 0; for (auto &Sec : G.sections()) { - if (llvm::empty(Sec.blocks())) + if (Sec.blocks().empty()) continue; ++NumSections; @@ -189,7 +189,7 @@ public: // Copy debug section blocks and symbols. orc::ExecutorAddr NextBlockAddr(MachOContainerBlock->getSize()); for (auto &SI : DebugSecInfos) { - assert(!llvm::empty(SI.Sec->blocks()) && "Empty debug info section?"); + assert(!SI.Sec->blocks().empty() && "Empty debug info section?"); // Update addresses in debug section. LLVM_DEBUG({ @@ -390,12 +390,12 @@ Error GDBJITDebugInfoRegistrationPlugin::notifyFailed( } Error GDBJITDebugInfoRegistrationPlugin::notifyRemovingResources( - ResourceKey K) { + JITDylib &JD, ResourceKey K) { return Error::success(); } void GDBJITDebugInfoRegistrationPlugin::notifyTransferringResources( - ResourceKey DstKey, ResourceKey SrcKey) {} + JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) {} void GDBJITDebugInfoRegistrationPlugin::modifyPassConfig( MaterializationResponsibility &MR, LinkGraph &LG, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp index e7ca636c83e9..00032e4dca3f 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ELFNixPlatform.cpp @@ -16,6 +16,7 @@ #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" +#include <optional> #define DEBUG_TYPE "orc" @@ -62,7 +63,7 @@ public: "<DSOHandleMU>", TT, PointerSize, Endianness, jitlink::getGenericEdgeKindName); auto &DSOHandleSection = - G->createSection(".data.__dso_handle", jitlink::MemProt::Read); + G->createSection(".data.__dso_handle", MemProt::Read); auto &DSOHandleBlock = G->createContentBlock( DSOHandleSection, getDSOHandleContent(PointerSize), orc::ExecutorAddr(), 8, 0); @@ -110,7 +111,7 @@ Expected<std::unique_ptr<ELFNixPlatform>> ELFNixPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, const char *OrcRuntimePath, - Optional<SymbolAliasMap> RuntimeAliases) { + std::optional<SymbolAliasMap> RuntimeAliases) { auto &EPC = ES.getExecutorProcessControl(); @@ -850,7 +851,7 @@ Error ELFNixPlatform::ELFNixPlatformPlugin::fixTLVSectionsAndEdges( auto *TLSInfoEntrySection = G.findSectionByName("$__TLSINFO"); if (TLSInfoEntrySection) { - Optional<uint64_t> Key; + std::optional<uint64_t> Key; { std::lock_guard<std::mutex> Lock(MP.PlatformMutex); auto I = MP.JITDylibToPThreadKey.find(&JD); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp index c591acdd646b..30d641ee00cf 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDebugObjectRegistrar.cpp @@ -16,12 +16,17 @@ namespace llvm { namespace orc { -Expected<std::unique_ptr<EPCDebugObjectRegistrar>> -createJITLoaderGDBRegistrar(ExecutionSession &ES) { +Expected<std::unique_ptr<EPCDebugObjectRegistrar>> createJITLoaderGDBRegistrar( + ExecutionSession &ES, + std::optional<ExecutorAddr> RegistrationFunctionDylib) { auto &EPC = ES.getExecutorProcessControl(); - auto ProcessHandle = EPC.loadDylib(nullptr); - if (!ProcessHandle) - return ProcessHandle.takeError(); + + if (!RegistrationFunctionDylib) { + if (auto D = EPC.loadDylib(nullptr)) + RegistrationFunctionDylib = *D; + else + return D.takeError(); + } SymbolStringPtr RegisterFn = EPC.getTargetTriple().isOSBinFormatMachO() @@ -31,7 +36,8 @@ createJITLoaderGDBRegistrar(ExecutionSession &ES) { SymbolLookupSet RegistrationSymbols; RegistrationSymbols.add(RegisterFn); - auto Result = EPC.lookupSymbols({{*ProcessHandle, RegistrationSymbols}}); + auto Result = + EPC.lookupSymbols({{*RegistrationFunctionDylib, RegistrationSymbols}}); if (!Result) return Result.takeError(); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp index ba154aaecd1a..1adcc9156957 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.cpp @@ -54,7 +54,7 @@ Error EPCDynamicLibrarySearchGenerator::tryToGenerate( for (auto &KV : LookupSymbols) { if (*ResultI) NewSymbols[KV.first] = - JITEvaluatedSymbol(*ResultI, JITSymbolFlags::Exported); + JITEvaluatedSymbol(ResultI->getValue(), JITSymbolFlags::Exported); ++ResultI; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp index 256ce94690f0..3aa94a7f43e2 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCEHFrameRegistrar.cpp @@ -16,17 +16,22 @@ using namespace llvm::orc::shared; namespace llvm { namespace orc { -Expected<std::unique_ptr<EPCEHFrameRegistrar>> -EPCEHFrameRegistrar::Create(ExecutionSession &ES) { +Expected<std::unique_ptr<EPCEHFrameRegistrar>> EPCEHFrameRegistrar::Create( + ExecutionSession &ES, + std::optional<ExecutorAddr> RegistrationFunctionsDylib) { // FIXME: Proper mangling here -- we really need to decouple linker mangling // from DataLayout. // Find the addresses of the registration/deregistration functions in the // executor process. auto &EPC = ES.getExecutorProcessControl(); - auto ProcessHandle = EPC.loadDylib(nullptr); - if (!ProcessHandle) - return ProcessHandle.takeError(); + + if (!RegistrationFunctionsDylib) { + if (auto D = EPC.loadDylib(nullptr)) + RegistrationFunctionsDylib = *D; + else + return D.takeError(); + } std::string RegisterWrapperName, DeregisterWrapperName; if (EPC.getTargetTriple().isOSBinFormatMachO()) { @@ -40,7 +45,8 @@ EPCEHFrameRegistrar::Create(ExecutionSession &ES) { RegistrationSymbols.add(EPC.intern(RegisterWrapperName)); RegistrationSymbols.add(EPC.intern(DeregisterWrapperName)); - auto Result = EPC.lookupSymbols({{*ProcessHandle, RegistrationSymbols}}); + auto Result = + EPC.lookupSymbols({{*RegistrationFunctionsDylib, RegistrationSymbols}}); if (!Result) return Result.takeError(); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp index 6c47c5c5f7bb..e70749cdfab2 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericDylibManager.cpp @@ -73,7 +73,7 @@ EPCGenericDylibManager::CreateWithDefaultBootstrapSymbols( Expected<tpctypes::DylibHandle> EPCGenericDylibManager::open(StringRef Path, uint64_t Mode) { - Expected<tpctypes::DylibHandle> H(0); + Expected<tpctypes::DylibHandle> H((ExecutorAddr())); if (auto Err = EPC.callSPSWrapper<rt::SPSSimpleExecutorDylibManagerOpenSignature>( SAs.Open, H, SAs.Instance, Path, Mode)) diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp index 75cc30753f41..a3d857c3bfc4 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericJITLinkMemoryManager.cpp @@ -47,8 +47,7 @@ public: for (auto &KV : Segs) { assert(KV.second.ContentSize <= std::numeric_limits<size_t>::max()); FR.Segments.push_back(tpctypes::SegFinalizeRequest{ - tpctypes::toWireProtectionFlags( - toSysMemoryProtectionFlags(KV.first.getMemProt())), + KV.first, KV.second.Addr, alignTo(KV.second.ContentSize + KV.second.ZeroFillSize, Parent.EPC.getPageSize()), diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp index cdac367e11a3..ec82081937e2 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCGenericRTDyldMemoryManager.cpp @@ -94,8 +94,8 @@ uint8_t *EPCGenericRTDyldMemoryManager::allocateDataSection( } void EPCGenericRTDyldMemoryManager::reserveAllocationSpace( - uintptr_t CodeSize, uint32_t CodeAlign, uintptr_t RODataSize, - uint32_t RODataAlign, uintptr_t RWDataSize, uint32_t RWDataAlign) { + uintptr_t CodeSize, Align CodeAlign, uintptr_t RODataSize, + Align RODataAlign, uintptr_t RWDataSize, Align RWDataAlign) { { std::lock_guard<std::mutex> Lock(M); @@ -103,15 +103,15 @@ void EPCGenericRTDyldMemoryManager::reserveAllocationSpace( if (!ErrMsg.empty()) return; - if (!isPowerOf2_32(CodeAlign) || CodeAlign > EPC.getPageSize()) { + if (CodeAlign > EPC.getPageSize()) { ErrMsg = "Invalid code alignment in reserveAllocationSpace"; return; } - if (!isPowerOf2_32(RODataAlign) || RODataAlign > EPC.getPageSize()) { + if (RODataAlign > EPC.getPageSize()) { ErrMsg = "Invalid ro-data alignment in reserveAllocationSpace"; return; } - if (!isPowerOf2_32(RWDataAlign) || RWDataAlign > EPC.getPageSize()) { + if (RWDataAlign > EPC.getPageSize()) { ErrMsg = "Invalid rw-data alignment in reserveAllocationSpace"; return; } @@ -142,7 +142,7 @@ void EPCGenericRTDyldMemoryManager::reserveAllocationSpace( } std::lock_guard<std::mutex> Lock(M); - Unmapped.push_back(AllocGroup()); + Unmapped.push_back(SectionAllocGroup()); Unmapped.back().RemoteCode = { *TargetAllocAddr, ExecutorAddrDiff(alignTo(CodeSize, EPC.getPageSize()))}; Unmapped.back().RemoteROData = { @@ -170,10 +170,11 @@ void EPCGenericRTDyldMemoryManager::registerEHFrames(uint8_t *Addr, return; ExecutorAddr LA(LoadAddr); - for (auto &Alloc : llvm::reverse(Unfinalized)) { - if (Alloc.RemoteCode.contains(LA) || Alloc.RemoteROData.contains(LA) || - Alloc.RemoteRWData.contains(LA)) { - Alloc.UnfinalizedEHFrames.push_back({LA, Size}); + for (auto &SecAllocGroup : llvm::reverse(Unfinalized)) { + if (SecAllocGroup.RemoteCode.contains(LA) || + SecAllocGroup.RemoteROData.contains(LA) || + SecAllocGroup.RemoteRWData.contains(LA)) { + SecAllocGroup.UnfinalizedEHFrames.push_back({LA, Size}); return; } } @@ -204,35 +205,29 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { LLVM_DEBUG(dbgs() << "Allocator " << (void *)this << " finalizing:\n"); // If there's an error then bail out here. - std::vector<AllocGroup> Allocs; + std::vector<SectionAllocGroup> SecAllocGroups; { std::lock_guard<std::mutex> Lock(M); if (ErrMsg && !this->ErrMsg.empty()) { *ErrMsg = std::move(this->ErrMsg); return true; } - std::swap(Allocs, Unfinalized); + std::swap(SecAllocGroups, Unfinalized); } // Loop over unfinalized objects to make finalization requests. - for (auto &ObjAllocs : Allocs) { + for (auto &SecAllocGroup : SecAllocGroups) { - tpctypes::WireProtectionFlags SegProts[3] = { - tpctypes::toWireProtectionFlags( - static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_EXEC)), - tpctypes::toWireProtectionFlags(sys::Memory::MF_READ), - tpctypes::toWireProtectionFlags( - static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_WRITE))}; + MemProt SegMemProts[3] = {MemProt::Read | MemProt::Exec, MemProt::Read, + MemProt::Read | MemProt::Write}; - ExecutorAddrRange *RemoteAddrs[3] = {&ObjAllocs.RemoteCode, - &ObjAllocs.RemoteROData, - &ObjAllocs.RemoteRWData}; + ExecutorAddrRange *RemoteAddrs[3] = {&SecAllocGroup.RemoteCode, + &SecAllocGroup.RemoteROData, + &SecAllocGroup.RemoteRWData}; - std::vector<Alloc> *SegSections[3] = {&ObjAllocs.CodeAllocs, - &ObjAllocs.RODataAllocs, - &ObjAllocs.RWDataAllocs}; + std::vector<SectionAlloc> *SegSections[3] = {&SecAllocGroup.CodeAllocs, + &SecAllocGroup.RODataAllocs, + &SecAllocGroup.RWDataAllocs}; tpctypes::FinalizeRequest FR; std::unique_ptr<char[]> AggregateContents[3]; @@ -240,7 +235,7 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { for (unsigned I = 0; I != 3; ++I) { FR.Segments.push_back({}); auto &Seg = FR.Segments.back(); - Seg.Prot = SegProts[I]; + Seg.AG = SegMemProts[I]; Seg.Addr = RemoteAddrs[I]->Start; for (auto &SecAlloc : *SegSections[I]) { Seg.Size = alignTo(Seg.Size, SecAlloc.Align); @@ -261,7 +256,7 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { Seg.Content = {AggregateContents[I].get(), SecOffset}; } - for (auto &Frame : ObjAllocs.UnfinalizedEHFrames) + for (auto &Frame : SecAllocGroup.UnfinalizedEHFrames) FR.Actions.push_back( {cantFail( WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( @@ -297,7 +292,8 @@ bool EPCGenericRTDyldMemoryManager::finalizeMemory(std::string *ErrMsg) { } void EPCGenericRTDyldMemoryManager::mapAllocsToRemoteAddrs( - RuntimeDyld &Dyld, std::vector<Alloc> &Allocs, ExecutorAddr NextAddr) { + RuntimeDyld &Dyld, std::vector<SectionAlloc> &Allocs, + ExecutorAddr NextAddr) { for (auto &Alloc : Allocs) { NextAddr.setValue(alignTo(NextAddr.getValue(), Alloc.Align)); LLVM_DEBUG({ diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp index 48aaab96e71f..ddfb30500c7b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/EPCIndirectionUtils.cpp @@ -250,6 +250,9 @@ EPCIndirectionUtils::Create(ExecutorProcessControl &EPC) { case Triple::x86: return CreateWithABI<OrcI386>(EPC); + case Triple::loongarch64: + return CreateWithABI<OrcLoongArch64>(EPC); + case Triple::mips: return CreateWithABI<OrcMips32Be>(EPC); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 95cf89ec3f8b..377a59993eb0 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" +#include "llvm/ExecutionEngine/JITLink/x86_64.h" #include "llvm/ExecutionEngine/Orc/Layer.h" #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" #include "llvm/IR/Constants.h" @@ -350,7 +351,6 @@ StaticLibraryDefinitionGenerator::Create( Error StaticLibraryDefinitionGenerator::tryToGenerate( LookupState &LS, LookupKind K, JITDylib &JD, JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { - // Don't materialize symbols from static archives unless this is a static // lookup. if (K != LookupKind::Static) @@ -364,16 +364,11 @@ Error StaticLibraryDefinitionGenerator::tryToGenerate( for (const auto &KV : Symbols) { const auto &Name = KV.first; - auto Child = Archive->findSym(*Name); - if (!Child) - return Child.takeError(); - if (*Child == None) + if (!ObjectFilesMap.count(Name)) continue; - auto ChildBuffer = (*Child)->getMemoryBufferRef(); - if (!ChildBuffer) - return ChildBuffer.takeError(); + auto ChildBuffer = ObjectFilesMap[Name]; ChildBufferInfos.insert( - {ChildBuffer->getBuffer(), ChildBuffer->getBufferIdentifier()}); + {ChildBuffer.getBuffer(), ChildBuffer.getBufferIdentifier()}); } for (auto ChildBufferInfo : ChildBufferInfos) { @@ -392,15 +387,163 @@ Error StaticLibraryDefinitionGenerator::tryToGenerate( return Error::success(); } +Error StaticLibraryDefinitionGenerator::buildObjectFilesMap() { + DenseMap<uint64_t, MemoryBufferRef> MemoryBuffers; + DenseSet<uint64_t> Visited; + DenseSet<uint64_t> Excluded; + for (auto &S : Archive->symbols()) { + StringRef SymName = S.getName(); + auto Member = S.getMember(); + if (!Member) + return Member.takeError(); + auto DataOffset = Member->getDataOffset(); + if (!Visited.count(DataOffset)) { + Visited.insert(DataOffset); + auto Child = Member->getAsBinary(); + if (!Child) + return Child.takeError(); + if ((*Child)->isCOFFImportFile()) { + ImportedDynamicLibraries.insert((*Child)->getFileName().str()); + Excluded.insert(DataOffset); + continue; + } + MemoryBuffers[DataOffset] = (*Child)->getMemoryBufferRef(); + } + if (!Excluded.count(DataOffset)) + ObjectFilesMap[L.getExecutionSession().intern(SymName)] = + MemoryBuffers[DataOffset]; + } + + return Error::success(); +} + StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator( ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, GetObjectFileInterface GetObjFileInterface, Error &Err) : L(L), GetObjFileInterface(std::move(GetObjFileInterface)), ArchiveBuffer(std::move(ArchiveBuffer)), Archive(std::make_unique<object::Archive>(*this->ArchiveBuffer, Err)) { - + ErrorAsOutParameter _(&Err); if (!this->GetObjFileInterface) this->GetObjFileInterface = getObjectFileInterface; + if (!Err) + Err = buildObjectFilesMap(); +} + +std::unique_ptr<DLLImportDefinitionGenerator> +DLLImportDefinitionGenerator::Create(ExecutionSession &ES, + ObjectLinkingLayer &L) { + return std::unique_ptr<DLLImportDefinitionGenerator>( + new DLLImportDefinitionGenerator(ES, L)); +} + +Error DLLImportDefinitionGenerator::tryToGenerate( + LookupState &LS, LookupKind K, JITDylib &JD, + JITDylibLookupFlags JDLookupFlags, const SymbolLookupSet &Symbols) { + JITDylibSearchOrder LinkOrder; + JD.withLinkOrderDo([&](const JITDylibSearchOrder &LO) { + LinkOrder.reserve(LO.size()); + for (auto &KV : LO) { + if (KV.first == &JD) + continue; + LinkOrder.push_back(KV); + } + }); + + // FIXME: if regular symbol name start with __imp_ we have to issue lookup of + // both __imp_ and stripped name and use the lookup information to resolve the + // real symbol name. + SymbolLookupSet LookupSet; + DenseMap<StringRef, SymbolLookupFlags> ToLookUpSymbols; + for (auto &KV : Symbols) { + StringRef Deinterned = *KV.first; + if (Deinterned.startswith(getImpPrefix())) + Deinterned = Deinterned.drop_front(StringRef(getImpPrefix()).size()); + // Don't degrade the required state + if (ToLookUpSymbols.count(Deinterned) && + ToLookUpSymbols[Deinterned] == SymbolLookupFlags::RequiredSymbol) + continue; + ToLookUpSymbols[Deinterned] = KV.second; + } + + for (auto &KV : ToLookUpSymbols) + LookupSet.add(ES.intern(KV.first), KV.second); + + auto Resolved = + ES.lookup(LinkOrder, LookupSet, LookupKind::DLSym, SymbolState::Resolved); + if (!Resolved) + return Resolved.takeError(); + + auto G = createStubsGraph(*Resolved); + if (!G) + return G.takeError(); + return L.add(JD, std::move(*G)); +} + +Expected<unsigned> +DLLImportDefinitionGenerator::getTargetPointerSize(const Triple &TT) { + switch (TT.getArch()) { + case Triple::x86_64: + return 8; + default: + return make_error<StringError>( + "architecture unsupported by DLLImportDefinitionGenerator", + inconvertibleErrorCode()); + } +} + +Expected<support::endianness> +DLLImportDefinitionGenerator::getTargetEndianness(const Triple &TT) { + switch (TT.getArch()) { + case Triple::x86_64: + return support::endianness::little; + default: + return make_error<StringError>( + "architecture unsupported by DLLImportDefinitionGenerator", + inconvertibleErrorCode()); + } +} + +Expected<std::unique_ptr<jitlink::LinkGraph>> +DLLImportDefinitionGenerator::createStubsGraph(const SymbolMap &Resolved) { + Triple TT = ES.getExecutorProcessControl().getTargetTriple(); + auto PointerSize = getTargetEndianness(TT); + if (!PointerSize) + return PointerSize.takeError(); + auto Endianness = getTargetEndianness(TT); + if (!Endianness) + return Endianness.takeError(); + + auto G = std::make_unique<jitlink::LinkGraph>( + "<DLLIMPORT_STUBS>", TT, *PointerSize, *Endianness, + jitlink::getGenericEdgeKindName); + jitlink::Section &Sec = + G->createSection(getSectionName(), MemProt::Read | MemProt::Exec); + + for (auto &KV : Resolved) { + jitlink::Symbol &Target = G->addAbsoluteSymbol( + *KV.first, ExecutorAddr(KV.second.getAddress()), *PointerSize, + jitlink::Linkage::Strong, jitlink::Scope::Local, false); + + // Create __imp_ symbol + jitlink::Symbol &Ptr = + jitlink::x86_64::createAnonymousPointer(*G, Sec, &Target); + auto NameCopy = G->allocateString(Twine(getImpPrefix()) + *KV.first); + StringRef NameCopyRef = StringRef(NameCopy.data(), NameCopy.size()); + Ptr.setName(NameCopyRef); + Ptr.setLinkage(jitlink::Linkage::Strong); + Ptr.setScope(jitlink::Scope::Default); + + // Create PLT stub + // FIXME: check PLT stub of data symbol is not accessed + jitlink::Block &StubBlock = + jitlink::x86_64::createPointerJumpStubBlock(*G, Sec, Ptr); + G->addDefinedSymbol(StubBlock, 0, *KV.first, StubBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, true, + false); + } + + return std::move(G); } } // End namespace orc. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp index 412b9f95ea62..361fcd4a2e9c 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ExecutorProcessControl.cpp @@ -75,12 +75,10 @@ SelfExecutorProcessControl::Create( Expected<tpctypes::DylibHandle> SelfExecutorProcessControl::loadDylib(const char *DylibPath) { std::string ErrMsg; - auto Dylib = std::make_unique<sys::DynamicLibrary>( - sys::DynamicLibrary::getPermanentLibrary(DylibPath, &ErrMsg)); - if (!Dylib->isValid()) + auto Dylib = sys::DynamicLibrary::getPermanentLibrary(DylibPath, &ErrMsg); + if (!Dylib.isValid()) return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); - DynamicLibraries.push_back(std::move(Dylib)); - return pointerToJITTargetAddress(DynamicLibraries.back().get()); + return ExecutorAddr::fromPtr(Dylib.getOSSpecificHandle()); } Expected<std::vector<tpctypes::LookupResult>> @@ -88,26 +86,20 @@ SelfExecutorProcessControl::lookupSymbols(ArrayRef<LookupRequest> Request) { std::vector<tpctypes::LookupResult> R; for (auto &Elem : Request) { - auto *Dylib = jitTargetAddressToPointer<sys::DynamicLibrary *>(Elem.Handle); - assert(llvm::any_of(DynamicLibraries, - [=](const std::unique_ptr<sys::DynamicLibrary> &DL) { - return DL.get() == Dylib; - }) && - "Invalid handle"); - - R.push_back(std::vector<JITTargetAddress>()); + sys::DynamicLibrary Dylib(Elem.Handle.toPtr<void *>()); + R.push_back(std::vector<ExecutorAddr>()); for (auto &KV : Elem.Symbols) { auto &Sym = KV.first; std::string Tmp((*Sym).data() + !!GlobalManglingPrefix, (*Sym).size() - !!GlobalManglingPrefix); - void *Addr = Dylib->getAddressOfSymbol(Tmp.c_str()); + void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str()); if (!Addr && KV.second == SymbolLookupFlags::RequiredSymbol) { // FIXME: Collect all failing symbols before erroring out. SymbolNameVector MissingSymbols; MissingSymbols.push_back(Sym); return make_error<SymbolsNotFound>(SSP, std::move(MissingSymbols)); } - R.back().push_back(pointerToJITTargetAddress(Addr)); + R.back().push_back(ExecutorAddr::fromPtr(Addr)); } } @@ -121,6 +113,18 @@ SelfExecutorProcessControl::runAsMain(ExecutorAddr MainFnAddr, return orc::runAsMain(MainFnAddr.toPtr<MainTy>(), Args); } +Expected<int32_t> +SelfExecutorProcessControl::runAsVoidFunction(ExecutorAddr VoidFnAddr) { + using VoidTy = int (*)(); + return orc::runAsVoidFunction(VoidFnAddr.toPtr<VoidTy>()); +} + +Expected<int32_t> +SelfExecutorProcessControl::runAsIntFunction(ExecutorAddr IntFnAddr, int Arg) { + using IntTy = int (*)(int); + return orc::runAsIntFunction(IntFnAddr.toPtr<IntTy>(), Arg); +} + void SelfExecutorProcessControl::callWrapperAsync(ExecutorAddr WrapperFnAddr, IncomingWFRHandler SendResult, ArrayRef<char> ArgBuffer) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 38cab526704f..989bb094cc25 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -137,6 +137,11 @@ createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, return CCMgrT::Create(ES, ErrorHandlerAddress); } + case Triple::loongarch64: { + typedef orc::LocalJITCompileCallbackManager<orc::OrcLoongArch64> CCMgrT; + return CCMgrT::Create(ES, ErrorHandlerAddress); + } + case Triple::mips: { typedef orc::LocalJITCompileCallbackManager<orc::OrcMips32Be> CCMgrT; return CCMgrT::Create(ES, ErrorHandlerAddress); @@ -192,6 +197,12 @@ createLocalIndirectStubsManagerBuilder(const Triple &T) { orc::LocalIndirectStubsManager<orc::OrcI386>>(); }; + case Triple::loongarch64: + return []() { + return std::make_unique< + orc::LocalIndirectStubsManager<orc::OrcLoongArch64>>(); + }; + case Triple::mips: return [](){ return std::make_unique< @@ -407,7 +418,7 @@ Error addFunctionPointerRelocationsToCurrentSymbol(jitlink::Symbol &Sym, auto SymStartInBlock = (const uint8_t *)B.getContent().data() + Sym.getOffset(); auto SymSize = Sym.getSize() ? Sym.getSize() : B.getSize() - Sym.getOffset(); - auto Content = makeArrayRef(SymStartInBlock, SymSize); + auto Content = ArrayRef(SymStartInBlock, SymSize); LLVM_DEBUG(dbgs() << "Adding self-relocations to " << Sym.getName() << "\n"); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index 1926ef1ecc72..bc84988e3254 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -9,12 +9,14 @@ #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h" #include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/RegisterEHFrames.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" @@ -81,6 +83,53 @@ Function *addHelperAndWrapper(Module &M, StringRef WrapperName, return WrapperFn; } +class ORCPlatformSupport : public LLJIT::PlatformSupport { +public: + ORCPlatformSupport(orc::LLJIT &J) : J(J) {} + + Error initialize(orc::JITDylib &JD) override { + using llvm::orc::shared::SPSExecutorAddr; + using llvm::orc::shared::SPSString; + using SPSDLOpenSig = SPSExecutorAddr(SPSString, int32_t); + enum dlopen_mode : int32_t { + ORC_RT_RTLD_LAZY = 0x1, + ORC_RT_RTLD_NOW = 0x2, + ORC_RT_RTLD_LOCAL = 0x4, + ORC_RT_RTLD_GLOBAL = 0x8 + }; + + if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlopen_wrapper")) { + return J.getExecutionSession().callSPSWrapper<SPSDLOpenSig>( + *WrapperAddr, DSOHandles[&JD], JD.getName(), + int32_t(ORC_RT_RTLD_LAZY)); + } else + return WrapperAddr.takeError(); + } + + Error deinitialize(orc::JITDylib &JD) override { + using llvm::orc::shared::SPSExecutorAddr; + using SPSDLCloseSig = int32_t(SPSExecutorAddr); + + if (auto WrapperAddr = J.lookup("__orc_rt_jit_dlclose_wrapper")) { + int32_t result; + auto E = J.getExecutionSession().callSPSWrapper<SPSDLCloseSig>( + *WrapperAddr, result, DSOHandles[&JD]); + if (E) + return E; + else if (result) + return make_error<StringError>("dlclose failed", + inconvertibleErrorCode()); + DSOHandles.erase(&JD); + } else + return WrapperAddr.takeError(); + return Error::success(); + } + +private: + orc::LLJIT &J; + DenseMap<orc::JITDylib *, orc::ExecutorAddr> DSOHandles; +}; + class GenericLLVMIRPlatformSupport; /// orc::Platform component of Generic LLVM IR Platform support. @@ -667,6 +716,7 @@ Error LLJITBuilderState::prepareForConstruction() { if (!CreateObjectLinkingLayer) { auto &TT = JTMB->getTargetTriple(); if (TT.getArch() == Triple::riscv64 || + TT.getArch() == Triple::loongarch64 || (TT.isOSBinFormatMachO() && (TT.getArch() == Triple::aarch64 || TT.getArch() == Triple::x86_64))) { @@ -676,8 +726,12 @@ Error LLJITBuilderState::prepareForConstruction() { [](ExecutionSession &ES, const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> { auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES); - ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>( - ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>())); + if (auto EHFrameRegistrar = EPCEHFrameRegistrar::Create(ES)) + ObjLinkingLayer->addPlugin( + std::make_unique<EHFrameRegistrationPlugin>( + ES, std::move(*EHFrameRegistrar))); + else + return EHFrameRegistrar.takeError(); return std::move(ObjLinkingLayer); }; } @@ -746,6 +800,11 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { Layer->setAutoClaimResponsibilityForObjectSymbols(true); } + if (S.JTMB->getTargetTriple().isOSBinFormatELF() && + (S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64 || + S.JTMB->getTargetTriple().getArch() == Triple::ArchType::ppc64le)) + Layer->setAutoClaimResponsibilityForObjectSymbols(true); + // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence // errors from some GCC / libstdc++ bots. Remove this conversion (i.e. // just return ObjLinkingLayer) once those bots are upgraded. @@ -875,6 +934,13 @@ Error LLJIT::applyDataLayout(Module &M) { return Error::success(); } +Error setUpOrcPlatform(LLJIT& J) { + LLVM_DEBUG( + { dbgs() << "Setting up orc platform support for LLJIT\n"; }); + J.setPlatformSupport(std::make_unique<ORCPlatformSupport>(J)); + return Error::success(); +} + void setUpGenericLLVMIRPlatform(LLJIT &J) { LLVM_DEBUG( { dbgs() << "Setting up GenericLLVMIRPlatform support for LLJIT\n"; }); @@ -950,5 +1016,12 @@ LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { CODLayer->setCloneToNewContextOnEmit(true); } +// In-process LLJIT uses eh-frame section wrappers via EPC, so we need to force +// them to be linked in. +LLVM_ATTRIBUTE_USED void linkComponents() { + errs() << (void *)&llvm_orc_registerEHFrameSectionWrapper + << (void *)&llvm_orc_deregisterEHFrameSectionWrapper; +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp index 4a50f2d7a153..95380d912392 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Layer.cpp @@ -77,11 +77,14 @@ IRMaterializationUnit::IRMaterializationUnit( // Otherwise we just need a normal linker mangling. auto MangledName = Mangle(G.getName()); SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G); + if (G.getComdat() && + G.getComdat()->getSelectionKind() != Comdat::NoDeduplicate) + SymbolFlags[MangledName] |= JITSymbolFlags::Weak; SymbolToDefinition[MangledName] = &G; } // If we need an init symbol for this module then create one. - if (!llvm::empty(getStaticInitGVs(M))) { + if (!getStaticInitGVs(M).empty()) { size_t Counter = 0; do { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp index 20b655bdf4b1..c0a740d42dbd 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -119,6 +119,10 @@ createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES, case Triple::x86: return LocalLazyCallThroughManager::Create<OrcI386>(ES, ErrorHandlerAddr); + case Triple::loongarch64: + return LocalLazyCallThroughManager::Create<OrcLoongArch64>( + ES, ErrorHandlerAddr); + case Triple::mips: return LocalLazyCallThroughManager::Create<OrcMips32Be>(ES, ErrorHandlerAddr); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp index 3452267e4df4..59c63d38458b 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LookupAndRecordAddrs.cpp @@ -73,7 +73,7 @@ Error lookupAndRecordAddrs( inconvertibleErrorCode()); for (unsigned I = 0; I != Pairs.size(); ++I) - Pairs[I].second->setValue(Result->front()[I]); + *Pairs[I].second = Result->front()[I]; return Error::success(); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index d5274b06a76f..914a1b5afc71 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -15,6 +15,7 @@ #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" +#include <optional> #define DEBUG_TYPE "orc" @@ -57,6 +58,29 @@ public: namespace { +std::unique_ptr<jitlink::LinkGraph> createPlatformGraph(MachOPlatform &MOP, + std::string Name) { + unsigned PointerSize; + support::endianness Endianness; + const auto &TT = + MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + + switch (TT.getArch()) { + case Triple::aarch64: + case Triple::x86_64: + PointerSize = 8; + Endianness = support::endianness::little; + break; + default: + llvm_unreachable("Unrecognized architecture"); + } + + return std::make_unique<jitlink::LinkGraph>(std::move(Name), TT, PointerSize, + Endianness, + jitlink::getGenericEdgeKindName); +} + +// Generates a MachO header. class MachOHeaderMaterializationUnit : public MaterializationUnit { public: MachOHeaderMaterializationUnit(MachOPlatform &MOP, @@ -67,41 +91,28 @@ public: StringRef getName() const override { return "MachOHeaderMU"; } void materialize(std::unique_ptr<MaterializationResponsibility> R) override { - unsigned PointerSize; - support::endianness Endianness; - const auto &TT = - MOP.getExecutionSession().getExecutorProcessControl().getTargetTriple(); + auto G = createPlatformGraph(MOP, "<MachOHeaderMU>"); + addMachOHeader(*G, MOP, R->getInitializerSymbol()); + MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + } - switch (TT.getArch()) { - case Triple::aarch64: - case Triple::x86_64: - PointerSize = 8; - Endianness = support::endianness::little; - break; - default: - llvm_unreachable("Unrecognized architecture"); - } + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} - auto G = std::make_unique<jitlink::LinkGraph>( - "<MachOHeaderMU>", TT, PointerSize, Endianness, - jitlink::getGenericEdgeKindName); - auto &HeaderSection = G->createSection("__header", jitlink::MemProt::Read); - auto &HeaderBlock = createHeaderBlock(*G, HeaderSection); + static void addMachOHeader(jitlink::LinkGraph &G, MachOPlatform &MOP, + const SymbolStringPtr &InitializerSymbol) { + auto &HeaderSection = G.createSection("__header", MemProt::Read); + auto &HeaderBlock = createHeaderBlock(G, HeaderSection); // Init symbol is header-start symbol. - G->addDefinedSymbol(HeaderBlock, 0, *R->getInitializerSymbol(), - HeaderBlock.getSize(), jitlink::Linkage::Strong, - jitlink::Scope::Default, false, true); + G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol, + HeaderBlock.getSize(), jitlink::Linkage::Strong, + jitlink::Scope::Default, false, true); for (auto &HS : AdditionalHeaderSymbols) - G->addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, - HeaderBlock.getSize(), jitlink::Linkage::Strong, - jitlink::Scope::Default, false, true); - - MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, + false, true); } - void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} - private: struct HeaderSymbol { const char *Name; @@ -163,7 +174,82 @@ private: constexpr MachOHeaderMaterializationUnit::HeaderSymbol MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[]; +// Creates a Bootstrap-Complete LinkGraph to run deferred actions. +class MachOPlatformCompleteBootstrapMaterializationUnit + : public MaterializationUnit { +public: + MachOPlatformCompleteBootstrapMaterializationUnit( + MachOPlatform &MOP, StringRef PlatformJDName, + SymbolStringPtr CompleteBootstrapSymbol, shared::AllocActions DeferredAAs, + ExecutorAddr PlatformBootstrap, ExecutorAddr PlatformShutdown, + ExecutorAddr RegisterJITDylib, ExecutorAddr DeregisterJITDylib, + ExecutorAddr MachOHeaderAddr) + : MaterializationUnit( + {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}), + MOP(MOP), PlatformJDName(PlatformJDName), + CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)), + DeferredAAs(std::move(DeferredAAs)), + PlatformBootstrap(PlatformBootstrap), + PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib), + DeregisterJITDylib(DeregisterJITDylib), + MachOHeaderAddr(MachOHeaderAddr) {} + + StringRef getName() const override { + return "MachOPlatformCompleteBootstrap"; + } + + void materialize(std::unique_ptr<MaterializationResponsibility> R) override { + using namespace jitlink; + auto G = createPlatformGraph(MOP, "<OrcRTCompleteBootstrap>"); + auto &PlaceholderSection = + G->createSection("__orc_rt_cplt_bs", MemProt::Read); + auto &PlaceholderBlock = + G->createZeroFillBlock(PlaceholderSection, 1, ExecutorAddr(), 1, 0); + G->addDefinedSymbol(PlaceholderBlock, 0, *CompleteBootstrapSymbol, 1, + Linkage::Strong, Scope::Hidden, false, true); + + // Reserve space for the stolen actions, plus two extras. + G->allocActions().reserve(DeferredAAs.size() + 2); + + // 1. Bootstrap the platform support code. + G->allocActions().push_back( + {cantFail(WrapperFunctionCall::Create<SPSArgList<>>(PlatformBootstrap)), + cantFail( + WrapperFunctionCall::Create<SPSArgList<>>(PlatformShutdown))}); + + // 2. Register the platform JITDylib. + G->allocActions().push_back( + {cantFail(WrapperFunctionCall::Create< + SPSArgList<SPSString, SPSExecutorAddr>>( + RegisterJITDylib, PlatformJDName, MachOHeaderAddr)), + cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( + DeregisterJITDylib, MachOHeaderAddr))}); + + // 3. Add the deferred actions to the graph. + std::move(DeferredAAs.begin(), DeferredAAs.end(), + std::back_inserter(G->allocActions())); + + MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); + } + + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} + +private: + MachOPlatform &MOP; + StringRef PlatformJDName; + SymbolStringPtr CompleteBootstrapSymbol; + shared::AllocActions DeferredAAs; + ExecutorAddr PlatformBootstrap; + ExecutorAddr PlatformShutdown; + ExecutorAddr RegisterJITDylib; + ExecutorAddr DeregisterJITDylib; + ExecutorAddr MachOHeaderAddr; +}; + +StringRef DataCommonSectionName = "__DATA,__common"; +StringRef DataDataSectionName = "__DATA,__data"; StringRef EHFrameSectionName = "__TEXT,__eh_frame"; +StringRef CompactUnwindInfoSectionName = "__TEXT,__unwind_info"; StringRef ModInitFuncSectionName = "__DATA,__mod_init_func"; StringRef ObjCClassListSectionName = "__DATA,__objc_classlist"; StringRef ObjCImageInfoSectionName = "__DATA,__objc_image_info"; @@ -187,7 +273,7 @@ namespace orc { Expected<std::unique_ptr<MachOPlatform>> MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, const char *OrcRuntimePath, - Optional<SymbolAliasMap> RuntimeAliases) { + std::optional<SymbolAliasMap> RuntimeAliases) { auto &EPC = ES.getExecutorProcessControl(); @@ -335,52 +421,115 @@ MachOPlatform::MachOPlatform( ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr<DefinitionGenerator> OrcRuntimeGenerator, Error &Err) - : ES(ES), ObjLinkingLayer(ObjLinkingLayer), - MachOHeaderStartSymbol(ES.intern("___dso_handle")) { + : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer) { ErrorAsOutParameter _(&Err); - ObjLinkingLayer.addPlugin(std::make_unique<MachOPlatformPlugin>(*this)); - PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); - // Force linking of eh-frame registration functions. - if (auto Err2 = lookupAndRecordAddrs( - ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), - {{ES.intern("___orc_rt_macho_register_ehframe_section"), - &orc_rt_macho_register_ehframe_section}, - {ES.intern("___orc_rt_macho_deregister_ehframe_section"), - &orc_rt_macho_deregister_ehframe_section}})) { - Err = std::move(Err2); + BootstrapInfo BI; + Bootstrap = &BI; + + // Bootstrap process -- here be phase-ordering dragons. + // + // The MachOPlatform class uses allocation actions to register metadata + // sections with the ORC runtime, however the runtime contains metadata + // registration functions that have their own metadata that they need to + // register (e.g. the frame-info registration functions have frame-info). + // We can't use an ordinary lookup to find these registration functions + // because their address is needed during the link of the containing graph + // itself (to build the allocation actions that will call the registration + // functions). Further complicating the situation (a) the graph containing + // the registration functions is allowed to depend on other graphs (e.g. the + // graph containing the ORC runtime RTTI support) so we need to handle with + // an unknown set of dependencies during bootstrap, and (b) these graphs may + // be linked concurrently if the user has installed a concurrent dispatcher. + // + // We satisfy these constraint by implementing a bootstrap phase during which + // allocation actions generated by MachOPlatform are appended to a list of + // deferred allocation actions, rather than to the graphs themselves. At the + // end of the bootstrap process the deferred actions are attached to a final + // "complete-bootstrap" graph that causes them to be run. + // + // The bootstrap steps are as follows: + // + // 1. Request the graph containing the mach header. This graph is guaranteed + // not to have any metadata so the fact that the registration functions + // are not available yet is not a problem. + // + // 2. Look up the registration functions and discard the results. This will + // trigger linking of the graph containing these functions, and + // consequently any graphs that it depends on. We do not use the lookup + // result to find the addresses of the functions requested (as described + // above the lookup will return too late for that), instead we capture the + // addresses in a post-allocation pass injected by the platform runtime + // during bootstrap only. + // + // 3. During bootstrap the MachOPlatformPlugin keeps a count of the number of + // graphs being linked (potentially concurrently), and we block until all + // of these graphs have completed linking. This is to avoid a race on the + // deferred-actions vector: the lookup for the runtime registration + // functions may return while some functions (those that are being + // incidentally linked in, but aren't reachable via the runtime functions) + // are still being linked, and we need to capture any allocation actions + // for this incidental code before we proceed. + // + // 4. Once all active links are complete we transfer the deferred actions to + // a newly added CompleteBootstrap graph and then request a symbol from + // the CompleteBootstrap graph to trigger materialization. This will cause + // all deferred actions to be run, and once this lookup returns we can + // proceed. + // + // 5. Finally, we associate runtime support methods in MachOPlatform with + // the corresponding jit-dispatch tag variables in the ORC runtime to make + // the support methods callable. The bootstrap is now complete. + + // Step (1) Add header materialization unit and request. + if ((Err = PlatformJD.define(std::make_unique<MachOHeaderMaterializationUnit>( + *this, MachOHeaderStartSymbol)))) return; - } - - State = BootstrapPhase2; - - // Associate wrapper function tags with JIT-side function implementations. - if (auto E2 = associateRuntimeSupportFunctions(PlatformJD)) { - Err = std::move(E2); + if ((Err = ES.lookup(&PlatformJD, MachOHeaderStartSymbol).takeError())) return; - } - // Lookup addresses of runtime functions callable by the platform, - // call the platform bootstrap function to initialize the platform-state - // object in the executor. - if (auto E2 = bootstrapMachORuntime(PlatformJD)) { - Err = std::move(E2); + // Step (2) Request runtime registration functions to trigger + // materialization.. + if ((Err = ES.lookup(makeJITDylibSearchOrder(&PlatformJD), + SymbolLookupSet( + {PlatformBootstrap.Name, PlatformShutdown.Name, + RegisterJITDylib.Name, DeregisterJITDylib.Name, + RegisterObjectPlatformSections.Name, + DeregisterObjectPlatformSections.Name, + CreatePThreadKey.Name})) + .takeError())) return; + + // Step (3) Wait for any incidental linker work to complete. + { + std::unique_lock<std::mutex> Lock(BI.Mutex); + BI.CV.wait(Lock, [&]() { return BI.ActiveGraphs == 0; }); + Bootstrap = nullptr; } - // PlatformJD hasn't been set up by the platform yet (since we're creating - // the platform now), so set it up. - if (auto E2 = setupJITDylib(PlatformJD)) { - Err = std::move(E2); + // Step (4) Add complete-bootstrap materialization unit and request. + auto BootstrapCompleteSymbol = ES.intern("__orc_rt_macho_complete_bootstrap"); + if ((Err = PlatformJD.define( + std::make_unique<MachOPlatformCompleteBootstrapMaterializationUnit>( + *this, PlatformJD.getName(), BootstrapCompleteSymbol, + std::move(BI.DeferredAAs), PlatformBootstrap.Addr, + PlatformShutdown.Addr, RegisterJITDylib.Addr, + DeregisterJITDylib.Addr, BI.MachOHeaderAddr)))) + return; + if ((Err = ES.lookup(makeJITDylibSearchOrder( + &PlatformJD, JITDylibLookupFlags::MatchAllSymbols), + std::move(BootstrapCompleteSymbol)) + .takeError())) return; - } - State = Initialized; + // (5) Associate runtime support functions. + if ((Err = associateRuntimeSupportFunctions())) + return; } -Error MachOPlatform::associateRuntimeSupportFunctions(JITDylib &PlatformJD) { +Error MachOPlatform::associateRuntimeSupportFunctions() { ExecutionSession::JITDispatchHandlerAssociationMap WFs; using PushInitializersSPSSig = @@ -440,24 +589,17 @@ void MachOPlatform::pushInitializersLoop( if (NewInitSymbols.empty()) { // To make the list intelligible to the runtime we need to convert all - // JITDylib pointers to their header addresses. + // JITDylib pointers to their header addresses. Only include JITDylibs + // that appear in the JITDylibToHeaderAddr map (i.e. those that have been + // through setupJITDylib) -- bare JITDylibs aren't managed by the platform. DenseMap<JITDylib *, ExecutorAddr> HeaderAddrs; HeaderAddrs.reserve(JDDepMap.size()); { std::lock_guard<std::mutex> Lock(PlatformMutex); for (auto &KV : JDDepMap) { auto I = JITDylibToHeaderAddr.find(KV.first); - if (I == JITDylibToHeaderAddr.end()) { - // The header address should have been materialized by the previous - // round, but we need to handle the pathalogical case where someone - // removes the symbol on another thread while we're running. - SendResult( - make_error<StringError>("JITDylib " + KV.first->getName() + - " has no registered header address", - inconvertibleErrorCode())); - return; - } - HeaderAddrs[KV.first] = I->second; + if (I != JITDylibToHeaderAddr.end()) + HeaderAddrs[KV.first] = I->second; } } @@ -465,12 +607,16 @@ void MachOPlatform::pushInitializersLoop( MachOJITDylibDepInfoMap DIM; DIM.reserve(JDDepMap.size()); for (auto &KV : JDDepMap) { - assert(HeaderAddrs.count(KV.first) && "Missing header addr"); - auto H = HeaderAddrs[KV.first]; + auto HI = HeaderAddrs.find(KV.first); + // Skip unmanaged JITDylibs. + if (HI == HeaderAddrs.end()) + continue; + auto H = HI->second; MachOJITDylibDepInfo DepInfo; for (auto &Dep : KV.second) { - assert(HeaderAddrs.count(Dep) && "Missing header addr"); - DepInfo.DepHeaders.push_back(HeaderAddrs[Dep]); + auto HJ = HeaderAddrs.find(Dep); + if (HJ != HeaderAddrs.end()) + DepInfo.DepHeaders.push_back(HJ->second); } DIM.push_back(std::make_pair(H, std::move(DepInfo))); } @@ -480,7 +626,7 @@ void MachOPlatform::pushInitializersLoop( // Otherwise issue a lookup and re-run this phase when it completes. lookupInitSymbolsAsync( - [this, SendResult = std::move(SendResult), &JD](Error Err) mutable { + [this, SendResult = std::move(SendResult), JD](Error Err) mutable { if (Err) SendResult(std::move(Err)); else @@ -571,30 +717,8 @@ void MachOPlatform::rt_lookupSymbol(SendSymbolAddressFn SendResult, RtLookupNotifyComplete(std::move(SendResult)), NoDependenciesToRegister); } -Error MachOPlatform::bootstrapMachORuntime(JITDylib &PlatformJD) { - if (auto Err = lookupAndRecordAddrs( - ES, LookupKind::Static, makeJITDylibSearchOrder(&PlatformJD), - {{ES.intern("___orc_rt_macho_platform_bootstrap"), - &orc_rt_macho_platform_bootstrap}, - {ES.intern("___orc_rt_macho_platform_shutdown"), - &orc_rt_macho_platform_shutdown}, - {ES.intern("___orc_rt_macho_register_jitdylib"), - &orc_rt_macho_register_jitdylib}, - {ES.intern("___orc_rt_macho_deregister_jitdylib"), - &orc_rt_macho_deregister_jitdylib}, - {ES.intern("___orc_rt_macho_register_object_platform_sections"), - &orc_rt_macho_register_object_platform_sections}, - {ES.intern("___orc_rt_macho_deregister_object_platform_sections"), - &orc_rt_macho_deregister_object_platform_sections}, - {ES.intern("___orc_rt_macho_create_pthread_key"), - &orc_rt_macho_create_pthread_key}})) - return Err; - - return ES.callSPSWrapper<void()>(orc_rt_macho_platform_bootstrap); -} - Expected<uint64_t> MachOPlatform::createPThreadKey() { - if (!orc_rt_macho_create_pthread_key) + if (!CreatePThreadKey.Addr) return make_error<StringError>( "Attempting to create pthread key in target, but runtime support has " "not been loaded yet", @@ -602,7 +726,7 @@ Expected<uint64_t> MachOPlatform::createPThreadKey() { Expected<uint64_t> Result(0); if (auto Err = ES.callSPSWrapper<SPSExpected<uint64_t>(void)>( - orc_rt_macho_create_pthread_key, Result)) + CreatePThreadKey.Addr, Result)) return std::move(Err); return Result; } @@ -611,7 +735,19 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( MaterializationResponsibility &MR, jitlink::LinkGraph &LG, jitlink::PassConfiguration &Config) { - auto PS = MP.State.load(); + using namespace jitlink; + + bool InBootstrapPhase = + &MR.getTargetJITDylib() == &MP.PlatformJD && MP.Bootstrap; + + // If we're in the bootstrap phase then increment the active graphs. + if (InBootstrapPhase) { + Config.PrePrunePasses.push_back( + [this](LinkGraph &G) { return bootstrapPipelineStart(G); }); + Config.PostAllocationPasses.push_back([this](LinkGraph &G) { + return bootstrapPipelineRecordRuntimeFunctions(G); + }); + } // --- Handle Initializers --- if (auto InitSymbol = MR.getInitializerSymbol()) { @@ -619,8 +755,8 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( // If the initializer symbol is the MachOHeader start symbol then just // register it and then bail out -- the header materialization unit // definitely doesn't need any other passes. - if (InitSymbol == MP.MachOHeaderStartSymbol) { - Config.PostAllocationPasses.push_back([this, &MR](jitlink::LinkGraph &G) { + if (InitSymbol == MP.MachOHeaderStartSymbol && !InBootstrapPhase) { + Config.PostAllocationPasses.push_back([this, &MR](LinkGraph &G) { return associateJITDylibHeaderSymbol(G, MR); }); return; @@ -629,34 +765,33 @@ void MachOPlatform::MachOPlatformPlugin::modifyPassConfig( // If the object contains an init symbol other than the header start symbol // then add passes to preserve, process and register the init // sections/symbols. - Config.PrePrunePasses.push_back([this, &MR](jitlink::LinkGraph &G) { + Config.PrePrunePasses.push_back([this, &MR](LinkGraph &G) { if (auto Err = preserveInitSections(G, MR)) return Err; return processObjCImageInfo(G, MR); }); } - // --- Add passes for eh-frame and TLV support --- - if (PS == MachOPlatform::BootstrapPhase1) { - Config.PostFixupPasses.push_back( - [this](jitlink::LinkGraph &G) { return registerEHSectionsPhase1(G); }); - return; - } - // Insert TLV lowering at the start of the PostPrunePasses, since we want // it to run before GOT/PLT lowering. Config.PostPrunePasses.insert( Config.PostPrunePasses.begin(), - [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { + [this, &JD = MR.getTargetJITDylib()](LinkGraph &G) { return fixTLVSectionsAndEdges(G, JD); }); // Add a pass to register the final addresses of any special sections in the // object with the runtime. Config.PostAllocationPasses.push_back( - [this, &JD = MR.getTargetJITDylib()](jitlink::LinkGraph &G) { - return registerObjectPlatformSections(G, JD); + [this, &JD = MR.getTargetJITDylib(), InBootstrapPhase](LinkGraph &G) { + return registerObjectPlatformSections(G, JD, InBootstrapPhase); }); + + // If we're in the bootstrap phase then steal allocation actions and then + // decrement the active graphs. + if (InBootstrapPhase) + Config.PostFixupPasses.push_back( + [this](LinkGraph &G) { return bootstrapPipelineEnd(G); }); } ObjectLinkingLayer::Plugin::SyntheticSymbolDependenciesMap @@ -673,6 +808,73 @@ MachOPlatform::MachOPlatformPlugin::getSyntheticSymbolDependencies( return SyntheticSymbolDependenciesMap(); } +Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineStart( + jitlink::LinkGraph &G) { + // Increment the active graphs count in BootstrapInfo. + std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex); + ++MP.Bootstrap.load()->ActiveGraphs; + return Error::success(); +} + +Error MachOPlatform::MachOPlatformPlugin:: + bootstrapPipelineRecordRuntimeFunctions(jitlink::LinkGraph &G) { + // Record bootstrap function names. + std::pair<StringRef, ExecutorAddr *> RuntimeSymbols[] = { + {*MP.MachOHeaderStartSymbol, &MP.Bootstrap.load()->MachOHeaderAddr}, + {*MP.PlatformBootstrap.Name, &MP.PlatformBootstrap.Addr}, + {*MP.PlatformShutdown.Name, &MP.PlatformShutdown.Addr}, + {*MP.RegisterJITDylib.Name, &MP.RegisterJITDylib.Addr}, + {*MP.DeregisterJITDylib.Name, &MP.DeregisterJITDylib.Addr}, + {*MP.RegisterObjectPlatformSections.Name, + &MP.RegisterObjectPlatformSections.Addr}, + {*MP.DeregisterObjectPlatformSections.Name, + &MP.DeregisterObjectPlatformSections.Addr}, + {*MP.CreatePThreadKey.Name, &MP.CreatePThreadKey.Addr}}; + + bool RegisterMachOHeader = false; + + for (auto *Sym : G.defined_symbols()) { + for (auto &RTSym : RuntimeSymbols) { + if (Sym->hasName() && Sym->getName() == RTSym.first) { + if (*RTSym.second) + return make_error<StringError>( + "Duplicate " + RTSym.first + + " detected during MachOPlatform bootstrap", + inconvertibleErrorCode()); + + if (Sym->getName() == *MP.MachOHeaderStartSymbol) + RegisterMachOHeader = true; + + *RTSym.second = Sym->getAddress(); + } + } + } + + if (RegisterMachOHeader) { + // If this graph defines the macho header symbol then create the internal + // mapping between it and PlatformJD. + std::lock_guard<std::mutex> Lock(MP.PlatformMutex); + MP.JITDylibToHeaderAddr[&MP.PlatformJD] = + MP.Bootstrap.load()->MachOHeaderAddr; + MP.HeaderAddrToJITDylib[MP.Bootstrap.load()->MachOHeaderAddr] = + &MP.PlatformJD; + } + + return Error::success(); +} + +Error MachOPlatform::MachOPlatformPlugin::bootstrapPipelineEnd( + jitlink::LinkGraph &G) { + std::lock_guard<std::mutex> Lock(MP.Bootstrap.load()->Mutex); + assert(MP.Bootstrap && "DeferredAAs reset before bootstrap completed"); + --MP.Bootstrap.load()->ActiveGraphs; + // Notify Bootstrap->CV while holding the mutex because the mutex is + // also keeping Bootstrap->CV alive. + if (MP.Bootstrap.load()->ActiveGraphs == 0) + MP.Bootstrap.load()->CV.notify_all(); + return Error::success(); +} + Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol( jitlink::LinkGraph &G, MaterializationResponsibility &MR) { auto I = llvm::find_if(G.defined_symbols(), [this](jitlink::Symbol *Sym) { @@ -685,12 +887,14 @@ Error MachOPlatform::MachOPlatformPlugin::associateJITDylibHeaderSymbol( auto HeaderAddr = (*I)->getAddress(); MP.JITDylibToHeaderAddr[&JD] = HeaderAddr; MP.HeaderAddrToJITDylib[HeaderAddr] = &JD; + // We can unconditionally add these actions to the Graph because this pass + // isn't used during bootstrap. G.allocActions().push_back( {cantFail( WrapperFunctionCall::Create<SPSArgList<SPSString, SPSExecutorAddr>>( - MP.orc_rt_macho_register_jitdylib, JD.getName(), HeaderAddr)), + MP.RegisterJITDylib.Addr, JD.getName(), HeaderAddr)), cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddr>>( - MP.orc_rt_macho_deregister_jitdylib, HeaderAddr))}); + MP.DeregisterJITDylib.Addr, HeaderAddr))}); return Error::success(); } @@ -747,7 +951,7 @@ Error MachOPlatform::MachOPlatformPlugin::processObjCImageInfo( auto ObjCImageInfoBlocks = ObjCImageInfo->blocks(); // Check that the section is not empty if present. - if (llvm::empty(ObjCImageInfoBlocks)) + if (ObjCImageInfoBlocks.empty()) return make_error<StringError>("Empty " + ObjCImageInfoSectionName + " section in " + G.getName(), inconvertibleErrorCode()); @@ -821,7 +1025,7 @@ Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( // Store key in __thread_vars struct fields. if (auto *ThreadDataSec = G.findSectionByName(ThreadVarsSectionName)) { - Optional<uint64_t> Key; + std::optional<uint64_t> Key; { std::lock_guard<std::mutex> Lock(MP.PlatformMutex); auto I = MP.JITDylibToPThreadKey.find(&JD); @@ -865,22 +1069,84 @@ Error MachOPlatform::MachOPlatformPlugin::fixTLVSectionsAndEdges( return Error::success(); } -Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( - jitlink::LinkGraph &G, JITDylib &JD) { +std::optional<MachOPlatform::MachOPlatformPlugin::UnwindSections> +MachOPlatform::MachOPlatformPlugin::findUnwindSectionInfo( + jitlink::LinkGraph &G) { + using namespace jitlink; - // Add an action to register the eh-frame. - if (auto *EHFrameSection = G.findSectionByName(EHFrameSectionName)) { - jitlink::SectionRange R(*EHFrameSection); - if (!R.empty()) - G.allocActions().push_back( - {cantFail( - WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( - MP.orc_rt_macho_register_ehframe_section, R.getRange())), - cantFail( - WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( - MP.orc_rt_macho_deregister_ehframe_section, R.getRange()))}); + UnwindSections US; + + // ScanSection records a section range and adds any executable blocks that + // that section points to to the CodeBlocks vector. + SmallVector<Block *> CodeBlocks; + auto ScanUnwindInfoSection = [&](Section &Sec, ExecutorAddrRange &SecRange) { + if (Sec.blocks().empty()) + return; + SecRange = (*Sec.blocks().begin())->getRange(); + for (auto *B : Sec.blocks()) { + auto R = B->getRange(); + SecRange.Start = std::min(SecRange.Start, R.Start); + SecRange.End = std::max(SecRange.End, R.End); + for (auto &E : B->edges()) { + if (!E.getTarget().isDefined()) + continue; + auto &TargetBlock = E.getTarget().getBlock(); + auto &TargetSection = TargetBlock.getSection(); + if ((TargetSection.getMemProt() & MemProt::Exec) == MemProt::Exec) + CodeBlocks.push_back(&TargetBlock); + } + } + }; + + if (Section *EHFrameSec = G.findSectionByName(EHFrameSectionName)) + ScanUnwindInfoSection(*EHFrameSec, US.DwarfSection); + + if (Section *CUInfoSec = G.findSectionByName(CompactUnwindInfoSectionName)) + ScanUnwindInfoSection(*CUInfoSec, US.CompactUnwindSection); + + // If we didn't find any pointed-to code-blocks then there's no need to + // register any info. + if (CodeBlocks.empty()) + return std::nullopt; + + // We have info to register. Sort the code blocks into address order and + // build a list of contiguous address ranges covering them all. + llvm::sort(CodeBlocks, [](const Block *LHS, const Block *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + for (auto *B : CodeBlocks) { + if (US.CodeRanges.empty() || US.CodeRanges.back().End != B->getAddress()) + US.CodeRanges.push_back(B->getRange()); + else + US.CodeRanges.back().End = B->getRange().End; } + LLVM_DEBUG({ + dbgs() << "MachOPlatform identified unwind info in " << G.getName() << ":\n" + << " DWARF: "; + if (US.DwarfSection.Start) + dbgs() << US.DwarfSection << "\n"; + else + dbgs() << "none\n"; + dbgs() << " Compact-unwind: "; + if (US.CompactUnwindSection.Start) + dbgs() << US.CompactUnwindSection << "\n"; + else + dbgs() << "none\n" + << "for code ranges:\n"; + for (auto &CR : US.CodeRanges) + dbgs() << " " << CR << "\n"; + if (US.CodeRanges.size() >= G.sections_size()) + dbgs() << "WARNING: High number of discontiguous code ranges! " + "Padding may be interfering with coalescing.\n"; + }); + + return US; +} + +Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( + jitlink::LinkGraph &G, JITDylib &JD, bool InBootstrapPhase) { + // Get a pointer to the thread data section if there is one. It will be used // below. jitlink::Section *ThreadDataSection = @@ -899,18 +1165,23 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( SmallVector<std::pair<StringRef, ExecutorAddrRange>, 8> MachOPlatformSecs; + // Collect data sections to register. + StringRef DataSections[] = {DataDataSectionName, DataCommonSectionName, + EHFrameSectionName}; + for (auto &SecName : DataSections) { + if (auto *Sec = G.findSectionByName(SecName)) { + jitlink::SectionRange R(*Sec); + if (!R.empty()) + MachOPlatformSecs.push_back({SecName, R.getRange()}); + } + } + // Having merged thread BSS (if present) and thread data (if present), // record the resulting section range. if (ThreadDataSection) { jitlink::SectionRange R(*ThreadDataSection); - if (!R.empty()) { - if (MP.State != MachOPlatform::Initialized) - return make_error<StringError>("__thread_data section encountered, but " - "MachOPlatform has not finished booting", - inconvertibleErrorCode()); - + if (!R.empty()) MachOPlatformSecs.push_back({ThreadDataSectionName, R.getRange()}); - } } // If any platform sections were found then add an allocation action to call @@ -933,19 +1204,23 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( MachOPlatformSecs.push_back({SecName, R.getRange()}); } - if (!MachOPlatformSecs.empty()) { - Optional<ExecutorAddr> HeaderAddr; + std::optional<std::tuple<SmallVector<ExecutorAddrRange>, ExecutorAddrRange, + ExecutorAddrRange>> + UnwindInfo; + if (auto UI = findUnwindSectionInfo(G)) + UnwindInfo = std::make_tuple(std::move(UI->CodeRanges), UI->DwarfSection, + UI->CompactUnwindSection); + + if (!MachOPlatformSecs.empty() || UnwindInfo) { + ExecutorAddr HeaderAddr; { std::lock_guard<std::mutex> Lock(MP.PlatformMutex); auto I = MP.JITDylibToHeaderAddr.find(&JD); - if (I != MP.JITDylibToHeaderAddr.end()) - HeaderAddr = I->second; + assert(I != MP.JITDylibToHeaderAddr.end() && + "Missing header for JITDylib"); + HeaderAddr = I->second; } - if (!HeaderAddr) - return make_error<StringError>("Missing header for " + JD.getName(), - inconvertibleErrorCode()); - // Dump the scraped inits. LLVM_DEBUG({ dbgs() << "MachOPlatform: Scraped " << G.getName() << " init sections:\n"; @@ -953,71 +1228,29 @@ Error MachOPlatform::MachOPlatformPlugin::registerObjectPlatformSections( dbgs() << " " << KV.first << ": " << KV.second << "\n"; }); - using SPSRegisterObjectPlatformSectionsArgs = - SPSArgList<SPSExecutorAddr, - SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>>; - G.allocActions().push_back( + using SPSRegisterObjectPlatformSectionsArgs = SPSArgList< + SPSExecutorAddr, + SPSOptional<SPSTuple<SPSSequence<SPSExecutorAddrRange>, + SPSExecutorAddrRange, SPSExecutorAddrRange>>, + SPSSequence<SPSTuple<SPSString, SPSExecutorAddrRange>>>; + + shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase) + ? G.allocActions() + : MP.Bootstrap.load()->DeferredAAs; + + allocActions.push_back( {cantFail( WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>( - MP.orc_rt_macho_register_object_platform_sections, *HeaderAddr, + MP.RegisterObjectPlatformSections.Addr, HeaderAddr, UnwindInfo, MachOPlatformSecs)), cantFail( WrapperFunctionCall::Create<SPSRegisterObjectPlatformSectionsArgs>( - MP.orc_rt_macho_deregister_object_platform_sections, - *HeaderAddr, MachOPlatformSecs))}); + MP.DeregisterObjectPlatformSections.Addr, HeaderAddr, + UnwindInfo, MachOPlatformSecs))}); } return Error::success(); } -Error MachOPlatform::MachOPlatformPlugin::registerEHSectionsPhase1( - jitlink::LinkGraph &G) { - - // If there's no eh-frame there's nothing to do. - auto *EHFrameSection = G.findSectionByName(EHFrameSectionName); - if (!EHFrameSection) - return Error::success(); - - // If the eh-frame section is empty there's nothing to do. - jitlink::SectionRange R(*EHFrameSection); - if (R.empty()) - return Error::success(); - - // Since we're linking the object containing the registration code now the - // addresses won't be ready in the platform. We'll have to find them in this - // graph instead. - ExecutorAddr orc_rt_macho_register_ehframe_section; - ExecutorAddr orc_rt_macho_deregister_ehframe_section; - for (auto *Sym : G.defined_symbols()) { - if (!Sym->hasName()) - continue; - if (Sym->getName() == "___orc_rt_macho_register_ehframe_section") - orc_rt_macho_register_ehframe_section = ExecutorAddr(Sym->getAddress()); - else if (Sym->getName() == "___orc_rt_macho_deregister_ehframe_section") - orc_rt_macho_deregister_ehframe_section = ExecutorAddr(Sym->getAddress()); - - if (orc_rt_macho_register_ehframe_section && - orc_rt_macho_deregister_ehframe_section) - break; - } - - // If we failed to find the required functions then bail out. - if (!orc_rt_macho_register_ehframe_section || - !orc_rt_macho_deregister_ehframe_section) - return make_error<StringError>("Could not find eh-frame registration " - "functions during platform bootstrap", - inconvertibleErrorCode()); - - // Otherwise, add allocation actions to the graph to register eh-frames for - // this object. - G.allocActions().push_back( - {cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( - orc_rt_macho_register_ehframe_section, R.getRange())), - cantFail(WrapperFunctionCall::Create<SPSArgList<SPSExecutorAddrRange>>( - orc_rt_macho_deregister_ehframe_section, R.getRange()))}); - - return Error::success(); -} - } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp index c2e7baabb994..d099a251232e 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MapperJITLinkMemoryManager.cpp @@ -8,11 +8,10 @@ #include "llvm/ExecutionEngine/Orc/MapperJITLinkMemoryManager.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" #include "llvm/Support/Process.h" -#include <limits> - using namespace llvm::jitlink; namespace llvm { @@ -33,7 +32,8 @@ public: std::swap(AI.Segments, Segs); std::swap(AI.Actions, G.allocActions()); - Parent.Mapper->initialize(AI, [&](Expected<ExecutorAddr> Result) { + Parent.Mapper->initialize(AI, [OnFinalize = std::move(OnFinalize)]( + Expected<ExecutorAddr> Result) mutable { if (!Result) { OnFinalize(Result.takeError()); return; @@ -55,8 +55,9 @@ private: }; MapperJITLinkMemoryManager::MapperJITLinkMemoryManager( - std::unique_ptr<MemoryMapper> Mapper) - : Mapper(std::move(Mapper)) {} + size_t ReservationGranularity, std::unique_ptr<MemoryMapper> Mapper) + : ReservationUnits(ReservationGranularity), AvailableMemory(AMAllocator), + Mapper(std::move(Mapper)) {} void MapperJITLinkMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, OnAllocatedFunction OnAllocated) { @@ -69,55 +70,78 @@ void MapperJITLinkMemoryManager::allocate(const JITLinkDylib *JD, LinkGraph &G, return; } - // Check if total size fits in address space - if (SegsSizes->total() > std::numeric_limits<size_t>::max()) { - OnAllocated(make_error<JITLinkError>( - formatv("Total requested size {:x} for graph {} exceeds address space", - SegsSizes->total(), G.getName()))); - return; - } + auto TotalSize = SegsSizes->total(); + + auto CompleteAllocation = [this, &G, BL = std::move(BL), + OnAllocated = std::move(OnAllocated)]( + Expected<ExecutorAddrRange> Result) mutable { + if (!Result) { + Mutex.unlock(); + return OnAllocated(Result.takeError()); + } + + auto NextSegAddr = Result->Start; + + std::vector<MemoryMapper::AllocInfo::SegInfo> SegInfos; + + for (auto &KV : BL.segments()) { + auto &AG = KV.first; + auto &Seg = KV.second; - Mapper->reserve( - SegsSizes->total(), - [this, &G, BL = std::move(BL), OnAllocated = std::move(OnAllocated)]( - Expected<ExecutorAddrRange> Result) mutable { - if (!Result) { - return OnAllocated(Result.takeError()); - } + auto TotalSize = Seg.ContentSize + Seg.ZeroFillSize; - auto NextSegAddr = Result->Start; + Seg.Addr = NextSegAddr; + Seg.WorkingMem = Mapper->prepare(NextSegAddr, TotalSize); - std::vector<MemoryMapper::AllocInfo::SegInfo> SegInfos; + NextSegAddr += alignTo(TotalSize, Mapper->getPageSize()); - for (auto &KV : BL.segments()) { - auto &AG = KV.first; - auto &Seg = KV.second; + MemoryMapper::AllocInfo::SegInfo SI; + SI.Offset = Seg.Addr - Result->Start; + SI.ContentSize = Seg.ContentSize; + SI.ZeroFillSize = Seg.ZeroFillSize; + SI.AG = AG; + SI.WorkingMem = Seg.WorkingMem; - auto TotalSize = Seg.ContentSize + Seg.ZeroFillSize; + SegInfos.push_back(SI); + } - Seg.Addr = NextSegAddr; - Seg.WorkingMem = Mapper->prepare(NextSegAddr, TotalSize); + UsedMemory.insert({Result->Start, NextSegAddr - Result->Start}); - NextSegAddr += alignTo(TotalSize, Mapper->getPageSize()); + if (NextSegAddr < Result->End) { + // Save the remaining memory for reuse in next allocation(s) + AvailableMemory.insert(NextSegAddr, Result->End - 1, true); + } + Mutex.unlock(); - MemoryMapper::AllocInfo::SegInfo SI; - SI.Offset = Seg.Addr - Result->Start; - SI.ContentSize = Seg.ContentSize; - SI.ZeroFillSize = Seg.ZeroFillSize; - SI.Prot = (toSysMemoryProtectionFlags(AG.getMemProt())); - SI.WorkingMem = Seg.WorkingMem; + if (auto Err = BL.apply()) { + OnAllocated(std::move(Err)); + return; + } - SegInfos.push_back(SI); - } + OnAllocated(std::make_unique<InFlightAlloc>(*this, G, Result->Start, + std::move(SegInfos))); + }; - if (auto Err = BL.apply()) { - OnAllocated(std::move(Err)); - return; - } + Mutex.lock(); - OnAllocated(std::make_unique<InFlightAlloc>(*this, G, Result->Start, - std::move(SegInfos))); - }); + // find an already reserved range that is large enough + ExecutorAddrRange SelectedRange{}; + + for (AvailableMemoryMap::iterator It = AvailableMemory.begin(); + It != AvailableMemory.end(); It++) { + if (It.stop() - It.start() + 1 >= TotalSize) { + SelectedRange = ExecutorAddrRange(It.start(), It.stop() + 1); + It.erase(); + break; + } + } + + if (SelectedRange.empty()) { // no already reserved range was found + auto TotalAllocation = alignTo(TotalSize, ReservationUnits); + Mapper->reserve(TotalAllocation, std::move(CompleteAllocation)); + } else { + CompleteAllocation(SelectedRange); + } } void MapperJITLinkMemoryManager::deallocate( @@ -125,10 +149,40 @@ void MapperJITLinkMemoryManager::deallocate( std::vector<ExecutorAddr> Bases; Bases.reserve(Allocs.size()); for (auto &FA : Allocs) { - Bases.push_back(FA.getAddress()); - FA.release(); + ExecutorAddr Addr = FA.getAddress(); + Bases.push_back(Addr); } - Mapper->release(Bases, std::move(OnDeallocated)); + + Mapper->deinitialize(Bases, [this, Allocs = std::move(Allocs), + OnDeallocated = std::move(OnDeallocated)]( + llvm::Error Err) mutable { + // TODO: How should we treat memory that we fail to deinitialize? + // We're currently bailing out and treating it as "burned" -- should we + // require that a failure to deinitialize still reset the memory so that + // we can reclaim it? + if (Err) { + for (auto &FA : Allocs) + FA.release(); + OnDeallocated(std::move(Err)); + return; + } + + { + std::lock_guard<std::mutex> Lock(Mutex); + + for (auto &FA : Allocs) { + ExecutorAddr Addr = FA.getAddress(); + ExecutorAddrDiff Size = UsedMemory[Addr]; + + UsedMemory.erase(Addr); + AvailableMemory.insert(Addr, Addr + Size - 1, true); + + FA.release(); + } + } + + OnDeallocated(Error::success()); + }); } } // end namespace orc diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp index ee92e5191b50..b457c7297bed 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/MemoryMapper.cpp @@ -11,6 +11,8 @@ #include "llvm/ExecutionEngine/Orc/Shared/OrcRTBridge.h" #include "llvm/Support/WindowsError.h" +#include <algorithm> + #if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) #include <fcntl.h> #include <sys/mman.h> @@ -60,7 +62,9 @@ char *InProcessMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, OnInitializedFunction OnInitialized) { ExecutorAddr MinAddr(~0ULL); + ExecutorAddr MaxAddr(0); + // FIXME: Release finalize lifetime segments. for (auto &Segment : AI.Segments) { auto Base = AI.MappingBase + Segment.Offset; auto Size = Segment.ContentSize + Segment.ZeroFillSize; @@ -68,14 +72,18 @@ void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, if (Base < MinAddr) MinAddr = Base; + if (Base + Size > MaxAddr) + MaxAddr = Base + Size; + std::memset((Base + Segment.ContentSize).toPtr<void *>(), 0, Segment.ZeroFillSize); - if (auto EC = sys::Memory::protectMappedMemory({Base.toPtr<void *>(), Size}, - Segment.Prot)) { + if (auto EC = sys::Memory::protectMappedMemory( + {Base.toPtr<void *>(), Size}, + toSysMemoryProtectionFlags(Segment.AG.getMemProt()))) { return OnInitialized(errorCodeToError(EC)); } - if (Segment.Prot & sys::Memory::MF_EXEC) + if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) sys::Memory::InvalidateInstructionCache(Base.toPtr<void *>(), Size); } @@ -85,6 +93,9 @@ void InProcessMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, { std::lock_guard<std::mutex> Lock(Mutex); + + // This is the maximum range whose permission have been possibly modified + Allocations[MinAddr].Size = MaxAddr - MinAddr; Allocations[MinAddr].DeinitializationActions = std::move(*DeinitializeActions); Reservations[AI.MappingBase.toPtr<void *>()].Allocations.push_back(MinAddr); @@ -101,13 +112,21 @@ void InProcessMemoryMapper::deinitialize( { std::lock_guard<std::mutex> Lock(Mutex); - for (auto Base : Bases) { + for (auto Base : llvm::reverse(Bases)) { if (Error Err = shared::runDeallocActions( Allocations[Base].DeinitializationActions)) { AllErr = joinErrors(std::move(AllErr), std::move(Err)); } + // Reset protections to read/write so the area can be reused + if (auto EC = sys::Memory::protectMappedMemory( + {Base.toPtr<void *>(), Allocations[Base].Size}, + sys::Memory::ProtectionFlags::MF_READ | + sys::Memory::ProtectionFlags::MF_WRITE)) { + AllErr = joinErrors(std::move(AllErr), errorCodeToError(EC)); + } + Allocations.erase(Base); } } @@ -275,7 +294,7 @@ void SharedMemoryMapper::reserve(size_t NumBytes, char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { auto R = Reservations.upper_bound(Addr); - assert(R != Reservations.begin() && "Attempt to prepare unknown range"); + assert(R != Reservations.begin() && "Attempt to prepare unreserved range"); R--; ExecutorAddrDiff Offset = Addr - R->first; @@ -285,9 +304,11 @@ char *SharedMemoryMapper::prepare(ExecutorAddr Addr, size_t ContentSize) { void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, OnInitializedFunction OnInitialized) { - auto Reservation = Reservations.find(AI.MappingBase); - assert(Reservation != Reservations.end() && - "Attempt to initialize unreserved range"); + auto Reservation = Reservations.upper_bound(AI.MappingBase); + assert(Reservation != Reservations.begin() && "Attempt to initialize unreserved range"); + Reservation--; + + auto AllocationOffset = AI.MappingBase - Reservation->first; tpctypes::SharedMemoryFinalizeRequest FR; @@ -296,13 +317,12 @@ void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, FR.Segments.reserve(AI.Segments.size()); for (auto Segment : AI.Segments) { - char *Base = - static_cast<char *>(Reservation->second.LocalAddr) + Segment.Offset; + char *Base = static_cast<char *>(Reservation->second.LocalAddr) + + AllocationOffset + Segment.Offset; std::memset(Base + Segment.ContentSize, 0, Segment.ZeroFillSize); tpctypes::SharedMemorySegFinalizeRequest SegReq; - SegReq.Prot = tpctypes::toWireProtectionFlags( - static_cast<sys::Memory::ProtectionFlags>(Segment.Prot)); + SegReq.AG = Segment.AG; SegReq.Addr = AI.MappingBase + Segment.Offset; SegReq.Size = Segment.ContentSize + Segment.ZeroFillSize; @@ -321,7 +341,7 @@ void SharedMemoryMapper::initialize(MemoryMapper::AllocInfo &AI, OnInitialized(std::move(Result)); }, - SAs.Instance, AI.MappingBase, std::move(FR)); + SAs.Instance, Reservation->first, std::move(FR)); } void SharedMemoryMapper::deinitialize( @@ -392,23 +412,23 @@ void SharedMemoryMapper::release(ArrayRef<ExecutorAddr> Bases, } SharedMemoryMapper::~SharedMemoryMapper() { - std::vector<ExecutorAddr> ReservationAddrs; - if (!Reservations.empty()) { - std::lock_guard<std::mutex> Lock(Mutex); - { - ReservationAddrs.reserve(Reservations.size()); - for (const auto &R : Reservations) { - ReservationAddrs.push_back(R.first); - } - } - } + std::lock_guard<std::mutex> Lock(Mutex); + for (const auto &R : Reservations) { - std::promise<MSVCPError> P; - auto F = P.get_future(); - release(ReservationAddrs, [&](Error Err) { P.set_value(std::move(Err)); }); - // FIXME: Release can actually fail. The error should be propagated. - // Meanwhile, a better option is to explicitly call release(). - cantFail(F.get()); +#if defined(LLVM_ON_UNIX) && !defined(__ANDROID__) + + munmap(R.second.LocalAddr, R.second.Size); + +#elif defined(_WIN32) + + UnmapViewOfFile(R.second.LocalAddr); + +#else + + (void)R; + +#endif + } } } // namespace orc diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp index 3de15db3f1c6..0c3beba43a35 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectFileInterface.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" +#include "llvm/ExecutionEngine/Orc/COFFPlatform.h" #include "llvm/ExecutionEngine/Orc/ELFNixPlatform.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/Object/COFF.h" @@ -14,6 +15,7 @@ #include "llvm/Object/MachO.h" #include "llvm/Object/ObjectFile.h" #include "llvm/Support/Debug.h" +#include <optional> #define DEBUG_TYPE "orc" @@ -150,7 +152,7 @@ static Expected<MaterializationUnit::Interface> getCOFFObjectFileSymbolInfo(ExecutionSession &ES, const object::COFFObjectFile &Obj) { MaterializationUnit::Interface I; - std::vector<Optional<object::coff_aux_section_definition>> ComdatDefs( + std::vector<std::optional<object::coff_aux_section_definition>> ComdatDefs( Obj.getNumberOfSections() + 1); for (auto &Sym : Obj.symbols()) { Expected<uint32_t> SymFlagsOrErr = Sym.getFlags(); @@ -177,7 +179,7 @@ getCOFFObjectFileSymbolInfo(ExecutionSession &ES, if (Def->Selection != COFF::IMAGE_COMDAT_SELECT_NODUPLICATES) { IsWeak = true; } - ComdatDefs[COFFSym.getSectionNumber()] = None; + ComdatDefs[COFFSym.getSectionNumber()] = std::nullopt; } else { // Skip symbols not defined in this object file. if (*SymFlagsOrErr & object::BasicSymbolRef::SF_Undefined) @@ -214,7 +216,16 @@ getCOFFObjectFileSymbolInfo(ExecutionSession &ES, I.SymbolFlags[ES.intern(*Name)] = std::move(*SymFlags); } - // FIXME: handle init symbols + SymbolStringPtr InitSymbol; + for (auto &Sec : Obj.sections()) { + if (auto SecName = Sec.getName()) { + if (COFFPlatform::isInitializerSection(*SecName)) { + addInitSymbol(I, ES, Obj.getFileName()); + break; + } + } else + return SecName.takeError(); + } return I; } @@ -276,5 +287,22 @@ getObjectFileInterface(ExecutionSession &ES, MemoryBufferRef ObjBuffer) { return getGenericObjectFileSymbolInfo(ES, **Obj); } +bool hasInitializerSection(jitlink::LinkGraph &G) { + bool IsMachO = G.getTargetTriple().isOSBinFormatMachO(); + bool IsElf = G.getTargetTriple().isOSBinFormatELF(); + if (!IsMachO && !IsElf) + return false; + + for (auto &Sec : G.sections()) { + if (IsMachO && std::apply(MachOPlatform::isInitializerSection, + Sec.getName().split(","))) + return true; + if (IsElf && ELFNixPlatform::isInitializerSection(Sec.getName())) + return true; + } + + return false; +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp index 5ddb35cbafd5..2b11c472e812 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -7,9 +7,9 @@ //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" -#include "llvm/ADT/Optional.h" #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" #include "llvm/ExecutionEngine/Orc/DebugObjectManagerPlugin.h" +#include "llvm/ExecutionEngine/Orc/ObjectFileInterface.h" #include "llvm/Support/MemoryBuffer.h" #include <string> #include <vector> @@ -58,35 +58,12 @@ private: LGI.SymbolFlags[ES.intern(Sym->getName())] = Flags; } - if ((G.getTargetTriple().isOSBinFormatMachO() && hasMachOInitSection(G)) || - (G.getTargetTriple().isOSBinFormatELF() && hasELFInitSection(G))) + if (hasInitializerSection(G)) LGI.InitSymbol = makeInitSymbol(ES, G); return LGI; } - static bool hasMachOInitSection(LinkGraph &G) { - for (auto &Sec : G.sections()) - if (Sec.getName() == "__DATA,__obj_selrefs" || - Sec.getName() == "__DATA,__objc_classlist" || - Sec.getName() == "__TEXT,__swift5_protos" || - Sec.getName() == "__TEXT,__swift5_proto" || - Sec.getName() == "__TEXT,__swift5_types" || - Sec.getName() == "__DATA,__mod_init_func") - return true; - return false; - } - - static bool hasELFInitSection(LinkGraph &G) { - for (auto &Sec : G.sections()) { - auto SecName = Sec.getName(); - if (SecName.consume_front(".init_array") && - (SecName.empty() || SecName[0] == '.')) - return true; - } - return false; - } - static SymbolStringPtr makeInitSymbol(ExecutionSession &ES, LinkGraph &G) { std::string InitSymString; raw_string_ostream(InitSymString) @@ -218,6 +195,8 @@ public: Flags |= JITSymbolFlags::Callable; if (Sym->getScope() == Scope::Default) Flags |= JITSymbolFlags::Exported; + if (Sym->getLinkage() == Linkage::Weak) + Flags |= JITSymbolFlags::Weak; InternedResult[InternedName] = JITEvaluatedSymbol(Sym->getAddress().getValue(), Flags); @@ -447,9 +426,15 @@ private: // claim, at which point we'll externalize that symbol. cantFail(MR->defineMaterializing(std::move(NewSymbolsToClaim))); - for (auto &KV : NameToSym) - if (!MR->getSymbols().count(KV.first)) + // Walk the list of symbols that we just tried to claim. Symbols that we're + // responsible for are marked live. Symbols that we're not responsible for + // are turned into external references. + for (auto &KV : NameToSym) { + if (MR->getSymbols().count(KV.first)) + KV.second->setLive(true); + else G.makeExternal(*KV.second); + } return Error::success(); } @@ -537,7 +522,8 @@ private: for (auto *B : G.blocks()) { auto &BI = BlockInfos[B]; for (auto &E : B->edges()) { - if (E.getTarget().getScope() == Scope::Local) { + if (E.getTarget().getScope() == Scope::Local && + !E.getTarget().isAbsolute()) { auto &TgtB = E.getTarget().getBlock(); if (&TgtB != B) { BI.Dependencies.insert(&TgtB); @@ -694,12 +680,12 @@ Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, [&](ResourceKey K) { Allocs[K].push_back(std::move(FA)); }); } -Error ObjectLinkingLayer::handleRemoveResources(ResourceKey K) { +Error ObjectLinkingLayer::handleRemoveResources(JITDylib &JD, ResourceKey K) { { Error Err = Error::success(); for (auto &P : Plugins) - Err = joinErrors(std::move(Err), P->notifyRemovingResources(K)); + Err = joinErrors(std::move(Err), P->notifyRemovingResources(JD, K)); if (Err) return Err; } @@ -719,7 +705,8 @@ Error ObjectLinkingLayer::handleRemoveResources(ResourceKey K) { return MemMgr.deallocate(std::move(AllocsToRemove)); } -void ObjectLinkingLayer::handleTransferResources(ResourceKey DstKey, +void ObjectLinkingLayer::handleTransferResources(JITDylib &JD, + ResourceKey DstKey, ResourceKey SrcKey) { auto I = Allocs.find(SrcKey); if (I != Allocs.end()) { @@ -735,7 +722,7 @@ void ObjectLinkingLayer::handleTransferResources(ResourceKey DstKey, } for (auto &P : Plugins) - P->notifyTransferringResources(DstKey, SrcKey); + P->notifyTransferringResources(JD, DstKey, SrcKey); } EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( @@ -787,7 +774,8 @@ Error EHFrameRegistrationPlugin::notifyFailed( return Error::success(); } -Error EHFrameRegistrationPlugin::notifyRemovingResources(ResourceKey K) { +Error EHFrameRegistrationPlugin::notifyRemovingResources(JITDylib &JD, + ResourceKey K) { std::vector<ExecutorAddrRange> RangesToRemove; ES.runSessionLocked([&] { @@ -811,7 +799,7 @@ Error EHFrameRegistrationPlugin::notifyRemovingResources(ResourceKey K) { } void EHFrameRegistrationPlugin::notifyTransferringResources( - ResourceKey DstKey, ResourceKey SrcKey) { + JITDylib &JD, ResourceKey DstKey, ResourceKey SrcKey) { auto SI = EHFrameRanges.find(SrcKey); if (SI == EHFrameRanges.end()) return; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp index da8aaad08cad..48dd0df80415 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -1077,5 +1077,158 @@ void OrcRiscv64::writeIndirectStubsBlock( } } +void OrcLoongArch64::writeResolverCode(char *ResolverWorkingMem, + JITTargetAddress ResolverTargetAddress, + JITTargetAddress ReentryFnAddr, + JITTargetAddress ReentryCtxAddr) { + + LLVM_DEBUG({ + dbgs() << "Writing resolver code to " + << formatv("{0:x16}", ResolverTargetAddress) << "\n"; + }); + + const uint32_t ResolverCode[] = { + 0x02fde063, // 0x0: addi.d $sp, $sp, -136(0xf78) + 0x29c00061, // 0x4: st.d $ra, $sp, 0 + 0x29c02064, // 0x8: st.d $a0, $sp, 8(0x8) + 0x29c04065, // 0xc: st.d $a1, $sp, 16(0x10) + 0x29c06066, // 0x10: st.d $a2, $sp, 24(0x18) + 0x29c08067, // 0x14: st.d $a3, $sp, 32(0x20) + 0x29c0a068, // 0x18: st.d $a4, $sp, 40(0x28) + 0x29c0c069, // 0x1c: st.d $a5, $sp, 48(0x30) + 0x29c0e06a, // 0x20: st.d $a6, $sp, 56(0x38) + 0x29c1006b, // 0x24: st.d $a7, $sp, 64(0x40) + 0x2bc12060, // 0x28: fst.d $fa0, $sp, 72(0x48) + 0x2bc14061, // 0x2c: fst.d $fa1, $sp, 80(0x50) + 0x2bc16062, // 0x30: fst.d $fa2, $sp, 88(0x58) + 0x2bc18063, // 0x34: fst.d $fa3, $sp, 96(0x60) + 0x2bc1a064, // 0x38: fst.d $fa4, $sp, 104(0x68) + 0x2bc1c065, // 0x3c: fst.d $fa5, $sp, 112(0x70) + 0x2bc1e066, // 0x40: fst.d $fa6, $sp, 120(0x78) + 0x2bc20067, // 0x44: fst.d $fa7, $sp, 128(0x80) + 0x1c000004, // 0x48: pcaddu12i $a0, 0 + 0x28c1c084, // 0x4c: ld.d $a0, $a0, 112(0x70) + 0x001501a5, // 0x50: move $a1, $t1 + 0x02ffd0a5, // 0x54: addi.d $a1, $a1, -12(0xff4) + 0x1c000006, // 0x58: pcaddu12i $a2, 0 + 0x28c1a0c6, // 0x5c: ld.d $a2, $a2, 104(0x68) + 0x4c0000c1, // 0x60: jirl $ra, $a2, 0 + 0x0015008c, // 0x64: move $t0, $a0 + 0x2b820067, // 0x68: fld.d $fa7, $sp, 128(0x80) + 0x2b81e066, // 0x6c: fld.d $fa6, $sp, 120(0x78) + 0x2b81c065, // 0x70: fld.d $fa5, $sp, 112(0x70) + 0x2b81a064, // 0x74: fld.d $fa4, $sp, 104(0x68) + 0x2b818063, // 0x78: fld.d $fa3, $sp, 96(0x60) + 0x2b816062, // 0x7c: fld.d $fa2, $sp, 88(0x58) + 0x2b814061, // 0x80: fld.d $fa1, $sp, 80(0x50) + 0x2b812060, // 0x84: fld.d $fa0, $sp, 72(0x48) + 0x28c1006b, // 0x88: ld.d $a7, $sp, 64(0x40) + 0x28c0e06a, // 0x8c: ld.d $a6, $sp, 56(0x38) + 0x28c0c069, // 0x90: ld.d $a5, $sp, 48(0x30) + 0x28c0a068, // 0x94: ld.d $a4, $sp, 40(0x28) + 0x28c08067, // 0x98: ld.d $a3, $sp, 32(0x20) + 0x28c06066, // 0x9c: ld.d $a2, $sp, 24(0x18) + 0x28c04065, // 0xa0: ld.d $a1, $sp, 16(0x10) + 0x28c02064, // 0xa4: ld.d $a0, $sp, 8(0x8) + 0x28c00061, // 0xa8: ld.d $ra, $sp, 0 + 0x02c22063, // 0xac: addi.d $sp, $sp, 136(0x88) + 0x4c000180, // 0xb0: jr $t0 + 0x00000000, // 0xb4: padding to align at 8 bytes + 0x01234567, // 0xb8: Lreentry_ctx_ptr: + 0xdeedbeef, // 0xbc: .dword 0 + 0x98765432, // 0xc0: Lreentry_fn_ptr: + 0xcafef00d, // 0xc4: .dword 0 + }; + + const unsigned ReentryCtxAddrOffset = 0xb8; + const unsigned ReentryFnAddrOffset = 0xc0; + + memcpy(ResolverWorkingMem, ResolverCode, sizeof(ResolverCode)); + memcpy(ResolverWorkingMem + ReentryFnAddrOffset, &ReentryFnAddr, + sizeof(uint64_t)); + memcpy(ResolverWorkingMem + ReentryCtxAddrOffset, &ReentryCtxAddr, + sizeof(uint64_t)); +} + +void OrcLoongArch64::writeTrampolines( + char *TrampolineBlockWorkingMem, + JITTargetAddress TrampolineBlockTargetAddress, + JITTargetAddress ResolverAddr, unsigned NumTrampolines) { + + LLVM_DEBUG({ + dbgs() << "Writing trampoline code to " + << formatv("{0:x16}", TrampolineBlockTargetAddress) << "\n"; + }); + + unsigned OffsetToPtr = alignTo(NumTrampolines * TrampolineSize, 8); + + memcpy(TrampolineBlockWorkingMem + OffsetToPtr, &ResolverAddr, + sizeof(uint64_t)); + + uint32_t *Trampolines = + reinterpret_cast<uint32_t *>(TrampolineBlockWorkingMem); + for (unsigned I = 0; I < NumTrampolines; ++I, OffsetToPtr -= TrampolineSize) { + uint32_t Hi20 = (OffsetToPtr + 0x800) & 0xfffff000; + uint32_t Lo12 = OffsetToPtr - Hi20; + Trampolines[4 * I + 0] = + 0x1c00000c | + (((Hi20 >> 12) & 0xfffff) << 5); // pcaddu12i $t0, %pc_hi20(Lptr) + Trampolines[4 * I + 1] = + 0x28c0018c | ((Lo12 & 0xfff) << 10); // ld.d $t0, $t0, %pc_lo12(Lptr) + Trampolines[4 * I + 2] = 0x4c00018d; // jirl $t1, $t0, 0 + Trampolines[4 * I + 3] = 0x0; // padding + } +} + +void OrcLoongArch64::writeIndirectStubsBlock( + char *StubsBlockWorkingMem, JITTargetAddress StubsBlockTargetAddress, + JITTargetAddress PointersBlockTargetAddress, unsigned NumStubs) { + // Stub format is: + // + // .section __orc_stubs + // stub1: + // pcaddu12i $t0, %pc_hi20(ptr1) ; PC-rel load of ptr1 + // ld.d $t0, $t0, %pc_lo12(ptr1) + // jr $t0 ; Jump to resolver + // .dword 0 ; Pad to 16 bytes + // stub2: + // pcaddu12i $t0, %pc_hi20(ptr2) ; PC-rel load of ptr2 + // ld.d $t0, $t0, %pc_lo12(ptr2) + // jr $t0 ; Jump to resolver + // .dword 0 ; Pad to 16 bytes + // ... + // + // .section __orc_ptrs + // ptr1: + // .dword 0x0 + // ptr2: + // .dword 0x0 + // ... + LLVM_DEBUG({ + dbgs() << "Writing stubs code to " + << formatv("{0:x16}", StubsBlockTargetAddress) << "\n"; + }); + assert(stubAndPointerRangesOk<OrcLoongArch64>( + StubsBlockTargetAddress, PointersBlockTargetAddress, NumStubs) && + "PointersBlock is out of range"); + + uint32_t *Stub = reinterpret_cast<uint32_t *>(StubsBlockWorkingMem); + + for (unsigned I = 0; I < NumStubs; ++I) { + uint64_t PtrDisplacement = + PointersBlockTargetAddress - StubsBlockTargetAddress; + uint32_t Hi20 = (PtrDisplacement + 0x800) & 0xfffff000; + uint32_t Lo12 = PtrDisplacement - Hi20; + Stub[4 * I + 0] = 0x1c00000c | (((Hi20 >> 12) & 0xfffff) + << 5); // pcaddu12i $t0, %pc_hi20(Lptr) + Stub[4 * I + 1] = + 0x28c0018c | ((Lo12 & 0xfff) << 10); // ld.d $t0, $t0, %pc_lo12(Lptr) + Stub[4 * I + 2] = 0x4c000180; // jr $t0 + Stub[4 * I + 3] = 0x0; // padding + PointersBlockTargetAddress += PointerSize; + StubsBlockTargetAddress += StubSize; + } +} + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp index b7eab6b85ecf..b823197b404f 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/OrcV2CBindings.cpp @@ -892,7 +892,10 @@ void LLVMOrcIRTransformLayerSetTransform( assert(!TSMRef && "TSMRef was not reset to null on error"); return unwrap(Err); } - return std::move(*unwrap(TSMRef)); + assert(TSMRef && "Transform succeeded, but TSMRef was set to null"); + ThreadSafeModule Result = std::move(*unwrap(TSMRef)); + LLVMOrcDisposeThreadSafeModule(TSMRef); + return std::move(Result); }); } @@ -1066,6 +1069,116 @@ LLVMOrcCreateRTDyldObjectLinkingLayerWithSectionMemoryManager( *unwrap(ES), [] { return std::make_unique<SectionMemoryManager>(); })); } +LLVMOrcObjectLayerRef +LLVMOrcCreateRTDyldObjectLinkingLayerWithMCJITMemoryManagerLikeCallbacks( + LLVMOrcExecutionSessionRef ES, void *CreateContextCtx, + LLVMMemoryManagerCreateContextCallback CreateContext, + LLVMMemoryManagerNotifyTerminatingCallback NotifyTerminating, + LLVMMemoryManagerAllocateCodeSectionCallback AllocateCodeSection, + LLVMMemoryManagerAllocateDataSectionCallback AllocateDataSection, + LLVMMemoryManagerFinalizeMemoryCallback FinalizeMemory, + LLVMMemoryManagerDestroyCallback Destroy) { + + struct MCJITMemoryManagerLikeCallbacks { + MCJITMemoryManagerLikeCallbacks() = default; + MCJITMemoryManagerLikeCallbacks( + void *CreateContextCtx, + LLVMMemoryManagerCreateContextCallback CreateContext, + LLVMMemoryManagerNotifyTerminatingCallback NotifyTerminating, + LLVMMemoryManagerAllocateCodeSectionCallback AllocateCodeSection, + LLVMMemoryManagerAllocateDataSectionCallback AllocateDataSection, + LLVMMemoryManagerFinalizeMemoryCallback FinalizeMemory, + LLVMMemoryManagerDestroyCallback Destroy) + : CreateContextCtx(CreateContextCtx), CreateContext(CreateContext), + NotifyTerminating(NotifyTerminating), + AllocateCodeSection(AllocateCodeSection), + AllocateDataSection(AllocateDataSection), + FinalizeMemory(FinalizeMemory), Destroy(Destroy) {} + + MCJITMemoryManagerLikeCallbacks(MCJITMemoryManagerLikeCallbacks &&Other) { + std::swap(CreateContextCtx, Other.CreateContextCtx); + std::swap(CreateContext, Other.CreateContext); + std::swap(NotifyTerminating, Other.NotifyTerminating); + std::swap(AllocateCodeSection, Other.AllocateCodeSection); + std::swap(AllocateDataSection, Other.AllocateDataSection); + std::swap(FinalizeMemory, Other.FinalizeMemory); + std::swap(Destroy, Other.Destroy); + } + + ~MCJITMemoryManagerLikeCallbacks() { + if (NotifyTerminating) + NotifyTerminating(CreateContextCtx); + } + + void *CreateContextCtx = nullptr; + LLVMMemoryManagerCreateContextCallback CreateContext = nullptr; + LLVMMemoryManagerNotifyTerminatingCallback NotifyTerminating = nullptr; + LLVMMemoryManagerAllocateCodeSectionCallback AllocateCodeSection = nullptr; + LLVMMemoryManagerAllocateDataSectionCallback AllocateDataSection = nullptr; + LLVMMemoryManagerFinalizeMemoryCallback FinalizeMemory = nullptr; + LLVMMemoryManagerDestroyCallback Destroy = nullptr; + }; + + class MCJITMemoryManagerLikeCallbacksMemMgr : public RTDyldMemoryManager { + public: + MCJITMemoryManagerLikeCallbacksMemMgr( + const MCJITMemoryManagerLikeCallbacks &CBs) + : CBs(CBs) { + Opaque = CBs.CreateContext(CBs.CreateContextCtx); + } + ~MCJITMemoryManagerLikeCallbacksMemMgr() override { CBs.Destroy(Opaque); } + + uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, + StringRef SectionName) override { + return CBs.AllocateCodeSection(Opaque, Size, Alignment, SectionID, + SectionName.str().c_str()); + } + + uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment, + unsigned SectionID, StringRef SectionName, + bool isReadOnly) override { + return CBs.AllocateDataSection(Opaque, Size, Alignment, SectionID, + SectionName.str().c_str(), isReadOnly); + } + + bool finalizeMemory(std::string *ErrMsg) override { + char *ErrMsgCString = nullptr; + bool Result = CBs.FinalizeMemory(Opaque, &ErrMsgCString); + assert((Result || !ErrMsgCString) && + "Did not expect an error message if FinalizeMemory succeeded"); + if (ErrMsgCString) { + if (ErrMsg) + *ErrMsg = ErrMsgCString; + free(ErrMsgCString); + } + return Result; + } + + private: + const MCJITMemoryManagerLikeCallbacks &CBs; + void *Opaque = nullptr; + }; + + assert(ES && "ES must not be null"); + assert(CreateContext && "CreateContext must not be null"); + assert(NotifyTerminating && "NotifyTerminating must not be null"); + assert(AllocateCodeSection && "AllocateCodeSection must not be null"); + assert(AllocateDataSection && "AllocateDataSection must not be null"); + assert(FinalizeMemory && "FinalizeMemory must not be null"); + assert(Destroy && "Destroy must not be null"); + + MCJITMemoryManagerLikeCallbacks CBs( + CreateContextCtx, CreateContext, NotifyTerminating, AllocateCodeSection, + AllocateDataSection, FinalizeMemory, Destroy); + + return wrap(new RTDyldObjectLinkingLayer(*unwrap(ES), [CBs = std::move(CBs)] { + return std::make_unique<MCJITMemoryManagerLikeCallbacksMemMgr>(CBs); + })); + + return nullptr; +} + void LLVMOrcRTDyldObjectLinkingLayerRegisterJITEventListener( LLVMOrcObjectLayerRef RTDyldObjLinkingLayer, LLVMJITEventListenerRef Listener) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 27044f66a55d..07b19b2e54f1 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -81,7 +81,7 @@ using BaseT = RTTIExtends<RTDyldObjectLinkingLayer, ObjectLayer>; RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager) - : BaseT(ES), GetMemoryManager(GetMemoryManager) { + : BaseT(ES), GetMemoryManager(std::move(GetMemoryManager)) { ES.registerResourceManager(*this); } @@ -108,6 +108,7 @@ void RTDyldObjectLinkingLayer::emit( // filter these later. auto InternalSymbols = std::make_shared<std::set<StringRef>>(); { + SymbolFlagsMap ExtraSymbolsToClaim; for (auto &Sym : (*Obj)->symbols()) { // Skip file symbols. @@ -128,6 +129,33 @@ void RTDyldObjectLinkingLayer::emit( return; } + // Try to claim responsibility of weak symbols + // if AutoClaimObjectSymbols flag is set. + if (AutoClaimObjectSymbols && + (*SymFlagsOrErr & object::BasicSymbolRef::SF_Weak)) { + auto SymName = Sym.getName(); + if (!SymName) { + ES.reportError(SymName.takeError()); + R->failMaterialization(); + return; + } + + // Already included in responsibility set, skip it + SymbolStringPtr SymbolName = ES.intern(*SymName); + if (R->getSymbols().count(SymbolName)) + continue; + + auto SymFlags = JITSymbolFlags::fromObjectSymbol(Sym); + if (!SymFlags) { + ES.reportError(SymFlags.takeError()); + R->failMaterialization(); + return; + } + + ExtraSymbolsToClaim[SymbolName] = *SymFlags; + continue; + } + // Don't include symbols that aren't global. if (!(*SymFlagsOrErr & object::BasicSymbolRef::SF_Global)) { if (auto SymName = Sym.getName()) @@ -139,6 +167,13 @@ void RTDyldObjectLinkingLayer::emit( } } } + + if (!ExtraSymbolsToClaim.empty()) { + if (auto Err = R->defineMaterializing(ExtraSymbolsToClaim)) { + ES.reportError(std::move(Err)); + R->failMaterialization(); + } + } } auto MemMgr = GetMemoryManager(); @@ -224,6 +259,46 @@ Error RTDyldObjectLinkingLayer::onObjLoad( if (COFFSec.Characteristics & COFF::IMAGE_SCN_LNK_COMDAT) I->second.setFlags(I->second.getFlags() | JITSymbolFlags::Weak); } + + // Handle any aliases. + for (auto &Sym : COFFObj->symbols()) { + uint32_t SymFlags = cantFail(Sym.getFlags()); + if (SymFlags & object::BasicSymbolRef::SF_Undefined) + continue; + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + auto I = Resolved.find(*Name); + + // Skip already-resolved symbols, and symbols that we're not responsible + // for. + if (I != Resolved.end() || !R.getSymbols().count(ES.intern(*Name))) + continue; + + // Skip anything other than weak externals. + auto COFFSym = COFFObj->getCOFFSymbol(Sym); + if (!COFFSym.isWeakExternal()) + continue; + auto *WeakExternal = COFFSym.getAux<object::coff_aux_weak_external>(); + if (WeakExternal->Characteristics != COFF::IMAGE_WEAK_EXTERN_SEARCH_ALIAS) + continue; + + // We found an alias. Reuse the resolution of the alias target for the + // alias itself. + Expected<object::COFFSymbolRef> TargetSymbol = + COFFObj->getSymbol(WeakExternal->TagIndex); + if (!TargetSymbol) + return TargetSymbol.takeError(); + Expected<StringRef> TargetName = COFFObj->getSymbolName(*TargetSymbol); + if (!TargetName) + return TargetName.takeError(); + auto J = Resolved.find(*TargetName); + if (J == Resolved.end()) + return make_error<StringError>("Could alias target " + *TargetName + + " not resolved", + inconvertibleErrorCode()); + Resolved[*Name] = J->second; + } } for (auto &KV : Resolved) { @@ -235,17 +310,21 @@ Error RTDyldObjectLinkingLayer::onObjLoad( auto InternedName = getExecutionSession().intern(KV.first); auto Flags = KV.second.getFlags(); - - // Override object flags and claim responsibility for symbols if - // requested. - if (OverrideObjectFlags || AutoClaimObjectSymbols) { - auto I = R.getSymbols().find(InternedName); - - if (OverrideObjectFlags && I != R.getSymbols().end()) + auto I = R.getSymbols().find(InternedName); + if (I != R.getSymbols().end()) { + // Override object flags and claim responsibility for symbols if + // requested. + if (OverrideObjectFlags) Flags = I->second; - else if (AutoClaimObjectSymbols && I == R.getSymbols().end()) - ExtraSymbolsToClaim[InternedName] = Flags; - } + else { + // RuntimeDyld/MCJIT's weak tracking isn't compatible with ORC's. Even + // if we're not overriding flags in general we should set the weak flag + // according to the MaterializationResponsibility object symbol table. + if (I->second.isWeak()) + Flags |= JITSymbolFlags::Weak; + } + } else if (AutoClaimObjectSymbols) + ExtraSymbolsToClaim[InternedName] = Flags; Symbols[InternedName] = JITEvaluatedSymbol(KV.second.getAddress(), Flags); } @@ -311,7 +390,8 @@ void RTDyldObjectLinkingLayer::onObjEmit( } } -Error RTDyldObjectLinkingLayer::handleRemoveResources(ResourceKey K) { +Error RTDyldObjectLinkingLayer::handleRemoveResources(JITDylib &JD, + ResourceKey K) { std::vector<MemoryManagerUP> MemMgrsToRemove; @@ -335,7 +415,8 @@ Error RTDyldObjectLinkingLayer::handleRemoveResources(ResourceKey K) { return Error::success(); } -void RTDyldObjectLinkingLayer::handleTransferResources(ResourceKey DstKey, +void RTDyldObjectLinkingLayer::handleTransferResources(JITDylib &JD, + ResourceKey DstKey, ResourceKey SrcKey) { auto I = MemMgrs.find(SrcKey); if (I != MemMgrs.end()) { diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcError.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcError.cpp index 2cc2bddeb21a..ec53338570db 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcError.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcError.cpp @@ -82,7 +82,7 @@ char DuplicateDefinition::ID = 0; char JITSymbolNotFound::ID = 0; std::error_code orcError(OrcErrorCode ErrCode) { - typedef std::underlying_type<OrcErrorCode>::type UT; + typedef std::underlying_type_t<OrcErrorCode> UT; return std::error_code(static_cast<UT>(ErrCode), getOrcErrCat()); } @@ -105,7 +105,7 @@ JITSymbolNotFound::JITSymbolNotFound(std::string SymbolName) : SymbolName(std::move(SymbolName)) {} std::error_code JITSymbolNotFound::convertToErrorCode() const { - typedef std::underlying_type<OrcErrorCode>::type UT; + typedef std::underlying_type_t<OrcErrorCode> UT; return std::error_code(static_cast<UT>(OrcErrorCode::JITSymbolNotFound), getOrcErrCat()); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp index dfdd846c46a7..86e31c52100e 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/OrcRTBridge.cpp @@ -56,6 +56,10 @@ const char *DeregisterEHFrameSectionWrapperName = "__llvm_orc_bootstrap_deregister_ehframe_section_wrapper"; const char *RunAsMainWrapperName = "__llvm_orc_bootstrap_run_as_main_wrapper"; +const char *RunAsVoidFunctionWrapperName = + "__llvm_orc_bootstrap_run_as_void_function_wrapper"; +const char *RunAsIntFunctionWrapperName = + "__llvm_orc_bootstrap_run_as_int_function_wrapper"; } // end namespace rt } // end namespace orc diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.cpp index 2bb204e688fc..921ac47d421d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/Shared/SimpleRemoteEPCUtils.cpp @@ -137,7 +137,7 @@ static Error makeUnexpectedEOFError() { Error FDSimpleRemoteEPCTransport::readBytes(char *Dst, size_t Size, bool *IsEOF) { - assert(Dst && "Attempt to read into null."); + assert((Size == 0 || Dst) && "Attempt to read into null."); ssize_t Completed = 0; while (Completed < static_cast<ssize_t>(Size)) { ssize_t Read = ::read(InFD, Dst + Completed, Size - Completed); @@ -167,7 +167,7 @@ Error FDSimpleRemoteEPCTransport::readBytes(char *Dst, size_t Size, } int FDSimpleRemoteEPCTransport::writeBytes(const char *Src, size_t Size) { - assert(Src && "Attempt to append from null."); + assert((Size == 0 || Src) && "Attempt to append from null."); ssize_t Completed = 0; while (Completed < static_cast<ssize_t>(Size)) { ssize_t Written = ::write(OutFD, Src + Completed, Size - Completed); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp index 47364a92a451..1bd10c9c6c0e 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SimpleRemoteEPC.cpp @@ -38,7 +38,7 @@ SimpleRemoteEPC::lookupSymbols(ArrayRef<LookupRequest> Request) { Result.push_back({}); Result.back().reserve(R->size()); for (auto Addr : *R) - Result.back().push_back(Addr.getValue()); + Result.back().push_back(Addr); } else return R.takeError(); } @@ -54,6 +54,23 @@ Expected<int32_t> SimpleRemoteEPC::runAsMain(ExecutorAddr MainFnAddr, return Result; } +Expected<int32_t> SimpleRemoteEPC::runAsVoidFunction(ExecutorAddr VoidFnAddr) { + int32_t Result = 0; + if (auto Err = callSPSWrapper<rt::SPSRunAsVoidFunctionSignature>( + RunAsVoidFunctionAddr, Result, ExecutorAddr(VoidFnAddr))) + return std::move(Err); + return Result; +} + +Expected<int32_t> SimpleRemoteEPC::runAsIntFunction(ExecutorAddr IntFnAddr, + int Arg) { + int32_t Result = 0; + if (auto Err = callSPSWrapper<rt::SPSRunAsIntFunctionSignature>( + RunAsIntFunctionAddr, Result, ExecutorAddr(IntFnAddr), Arg)) + return std::move(Err); + return Result; +} + void SimpleRemoteEPC::callWrapperAsync(ExecutorAddr WrapperFnAddr, IncomingWFRHandler OnComplete, ArrayRef<char> ArgBuffer) { @@ -312,7 +329,9 @@ Error SimpleRemoteEPC::setup(Setup S) { if (auto Err = getBootstrapSymbols( {{JDI.JITDispatchContext, ExecutorSessionObjectName}, {JDI.JITDispatchFunction, DispatchFnName}, - {RunAsMainAddr, rt::RunAsMainWrapperName}})) + {RunAsMainAddr, rt::RunAsMainWrapperName}, + {RunAsVoidFunctionAddr, rt::RunAsVoidFunctionWrapperName}, + {RunAsIntFunctionAddr, rt::RunAsIntFunctionWrapperName}})) return Err; if (auto DM = diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp index c2fa4466eab6..0388725dfb63 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp @@ -67,7 +67,7 @@ void SpeculateQuery::findCalles(const BasicBlock *BB, } bool SpeculateQuery::isStraightLine(const Function &F) { - return llvm::all_of(F.getBasicBlockList(), [](const BasicBlock &BB) { + return llvm::all_of(F, [](const BasicBlock &BB) { return BB.getSingleSuccessor() != nullptr; }); } @@ -97,7 +97,7 @@ BlockFreqQuery::ResultTy BlockFreqQuery::operator()(Function &F) { auto IBBs = findBBwithCalls(F); if (IBBs.empty()) - return None; + return std::nullopt; auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F); @@ -136,7 +136,7 @@ SequenceBBQuery::BlockListTy SequenceBBQuery::rearrangeBB(const Function &F, const BlockListTy &BBList) { BlockListTy RearrangedBBSet; - for (auto &Block : F.getBasicBlockList()) + for (auto &Block : F) if (llvm::is_contained(BBList, &Block)) RearrangedBBSet.push_back(&Block); @@ -288,14 +288,14 @@ SpeculateQuery::ResultTy SequenceBBQuery::operator()(Function &F) { CallerBlocks = findBBwithCalls(F); if (CallerBlocks.empty()) - return None; + return std::nullopt; if (isStraightLine(F)) SequencedBlocks = rearrangeBB(F, CallerBlocks); else SequencedBlocks = queryCFG(F, CallerBlocks); - for (auto BB : SequencedBlocks) + for (const auto *BB : SequencedBlocks) findCalles(BB, Calles); CallerAndCalles.insert({F.getName(), std::move(Calles)}); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp index caa191cea899..147f915f61d6 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/ExecutorSharedMemoryMapperService.cpp @@ -21,35 +21,30 @@ #include <unistd.h> #endif +namespace llvm { +namespace orc { +namespace rt_bootstrap { + #if defined(_WIN32) -static DWORD getWindowsProtectionFlags(unsigned Flags) { - switch (Flags & llvm::sys::Memory::MF_RWE_MASK) { - case llvm::sys::Memory::MF_READ: +static DWORD getWindowsProtectionFlags(MemProt MP) { + if (MP == MemProt::Read) return PAGE_READONLY; - case llvm::sys::Memory::MF_WRITE: + if (MP == MemProt::Write || + MP == (MemProt::Write | MemProt::Read)) { // Note: PAGE_WRITE is not supported by VirtualProtect return PAGE_READWRITE; - case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE: - return PAGE_READWRITE; - case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_EXEC: + } + if (MP == (MemProt::Read | MemProt::Exec)) return PAGE_EXECUTE_READ; - case llvm::sys::Memory::MF_READ | llvm::sys::Memory::MF_WRITE | - llvm::sys::Memory::MF_EXEC: + if (MP == (MemProt::Read | MemProt::Write | MemProt::Exec)) return PAGE_EXECUTE_READWRITE; - case llvm::sys::Memory::MF_EXEC: + if (MP == MemProt::Exec) return PAGE_EXECUTE; - default: - llvm_unreachable("Illegal memory protection flag specified!"); - } - // Provide a default return value as required by some compilers. + return PAGE_NOACCESS; } #endif -namespace llvm { -namespace orc { -namespace rt_bootstrap { - Expected<std::pair<ExecutorAddr, std::string>> ExecutorSharedMemoryMapperService::reserve(uint64_t Size) { #if (defined(LLVM_ON_UNIX) && !defined(__ANDROID__)) || defined(_WIN32) @@ -137,11 +132,11 @@ Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( #if defined(LLVM_ON_UNIX) int NativeProt = 0; - if (Segment.Prot & tpctypes::WPF_Read) + if ((Segment.AG.getMemProt() & MemProt::Read) == MemProt::Read) NativeProt |= PROT_READ; - if (Segment.Prot & tpctypes::WPF_Write) + if ((Segment.AG.getMemProt() & MemProt::Write) == MemProt::Write) NativeProt |= PROT_WRITE; - if (Segment.Prot & tpctypes::WPF_Exec) + if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) NativeProt |= PROT_EXEC; if (mprotect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt)) @@ -150,7 +145,7 @@ Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( #elif defined(_WIN32) DWORD NativeProt = - getWindowsProtectionFlags(fromWireProtectionFlags(Segment.Prot)); + getWindowsProtectionFlags(Segment.AG.getMemProt()); if (!VirtualProtect(Segment.Addr.toPtr<void *>(), Segment.Size, NativeProt, &NativeProt)) @@ -158,7 +153,7 @@ Expected<ExecutorAddr> ExecutorSharedMemoryMapperService::initialize( #endif - if (Segment.Prot & tpctypes::WPF_Exec) + if ((Segment.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) sys::Memory::InvalidateInstructionCache(Segment.Addr.toPtr<void *>(), Segment.Size); } @@ -192,12 +187,23 @@ Error ExecutorSharedMemoryMapperService::deinitialize( { std::lock_guard<std::mutex> Lock(Mutex); - for (auto Base : Bases) { + for (auto Base : llvm::reverse(Bases)) { if (Error Err = shared::runDeallocActions( Allocations[Base].DeinitializationActions)) { AllErr = joinErrors(std::move(AllErr), std::move(Err)); } + // Remove the allocation from the allocation list of its reservation + for (auto &Reservation : Reservations) { + auto AllocationIt = + std::find(Reservation.second.Allocations.begin(), + Reservation.second.Allocations.end(), Base); + if (AllocationIt != Reservation.second.Allocations.end()) { + Reservation.second.Allocations.erase(AllocationIt); + break; + } + } + Allocations.erase(Base); } } @@ -264,19 +270,15 @@ Error ExecutorSharedMemoryMapperService::release( } Error ExecutorSharedMemoryMapperService::shutdown() { + if (Reservations.empty()) + return Error::success(); + std::vector<ExecutorAddr> ReservationAddrs; - if (!Reservations.empty()) { - std::lock_guard<std::mutex> Lock(Mutex); - { - ReservationAddrs.reserve(Reservations.size()); - for (const auto &R : Reservations) { - ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst())); - } - } - } - return release(ReservationAddrs); + ReservationAddrs.reserve(Reservations.size()); + for (const auto &R : Reservations) + ReservationAddrs.push_back(ExecutorAddr::fromPtr(R.getFirst())); - return Error::success(); + return release(std::move(ReservationAddrs)); } void ExecutorSharedMemoryMapperService::addBootstrapSymbols( diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp index 909d47deef59..b38877955282 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/OrcRTBootstrap.cpp @@ -56,6 +56,27 @@ runAsMainWrapper(const char *ArgData, size_t ArgSize) { .release(); } +static llvm::orc::shared::CWrapperFunctionResult +runAsVoidFunctionWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<rt::SPSRunAsVoidFunctionSignature>::handle( + ArgData, ArgSize, + [](ExecutorAddr MainAddr) -> int32_t { + return runAsVoidFunction(MainAddr.toPtr<int32_t (*)(void)>()); + }) + .release(); +} + +static llvm::orc::shared::CWrapperFunctionResult +runAsIntFunctionWrapper(const char *ArgData, size_t ArgSize) { + return WrapperFunction<rt::SPSRunAsIntFunctionSignature>::handle( + ArgData, ArgSize, + [](ExecutorAddr MainAddr, int32_t Arg) -> int32_t { + return runAsIntFunction(MainAddr.toPtr<int32_t (*)(int32_t)>(), + Arg); + }) + .release(); +} + void addTo(StringMap<ExecutorAddr> &M) { M[rt::MemoryWriteUInt8sWrapperName] = ExecutorAddr::fromPtr( &writeUIntsWrapper<tpctypes::UInt8Write, @@ -76,6 +97,10 @@ void addTo(StringMap<ExecutorAddr> &M) { M[rt::DeregisterEHFrameSectionWrapperName] = ExecutorAddr::fromPtr(&llvm_orc_deregisterEHFrameSectionWrapper); M[rt::RunAsMainWrapperName] = ExecutorAddr::fromPtr(&runAsMainWrapper); + M[rt::RunAsVoidFunctionWrapperName] = + ExecutorAddr::fromPtr(&runAsVoidFunctionWrapper); + M[rt::RunAsIntFunctionWrapperName] = + ExecutorAddr::fromPtr(&runAsIntFunctionWrapper); } } // end namespace rt_bootstrap diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp index 3c9dd21b0832..cb11b68e2719 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorDylibManager.cpp @@ -35,24 +35,18 @@ SimpleExecutorDylibManager::open(const std::string &Path, uint64_t Mode) { return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); std::lock_guard<std::mutex> Lock(M); - Dylibs[NextId] = std::move(DL); - return NextId++; + auto H = ExecutorAddr::fromPtr(DL.getOSSpecificHandle()); + Dylibs.insert(DL.getOSSpecificHandle()); + return H; } Expected<std::vector<ExecutorAddr>> SimpleExecutorDylibManager::lookup(tpctypes::DylibHandle H, const RemoteSymbolLookupSet &L) { std::vector<ExecutorAddr> Result; - - std::lock_guard<std::mutex> Lock(M); - auto I = Dylibs.find(H); - if (I == Dylibs.end()) - return make_error<StringError>("No dylib for handle " + formatv("{0:x}", H), - inconvertibleErrorCode()); - auto &DL = I->second; + auto DL = sys::DynamicLibrary(H.toPtr<void *>()); for (const auto &E : L) { - if (E.Name.empty()) { if (E.Required) return make_error<StringError>("Required address for empty symbol \"\"", @@ -85,10 +79,10 @@ SimpleExecutorDylibManager::lookup(tpctypes::DylibHandle H, Error SimpleExecutorDylibManager::shutdown() { - DylibsMap DM; + DylibSet DS; { std::lock_guard<std::mutex> Lock(M); - std::swap(DM, Dylibs); + std::swap(DS, Dylibs); } // There is no removal of dylibs at the moment, so nothing to do here. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp index c848dd65fa7e..ce94bf1e039a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/SimpleExecutorMemoryManager.cpp @@ -132,9 +132,9 @@ Error SimpleExecutorMemoryManager::finalize(tpctypes::FinalizeRequest &FR) { assert(Seg.Size <= std::numeric_limits<size_t>::max()); if (auto EC = sys::Memory::protectMappedMemory( {Mem, static_cast<size_t>(Seg.Size)}, - tpctypes::fromWireProtectionFlags(Seg.Prot))) + toSysMemoryProtectionFlags(Seg.AG.getMemProt()))) return BailOut(errorCodeToError(EC)); - if (Seg.Prot & tpctypes::WPF_Exec) + if ((Seg.AG.getMemProt() & MemProt::Exec) == MemProt::Exec) sys::Memory::InvalidateInstructionCache(Mem, Seg.Size); } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp index a8e6c049cf4b..7546b3f8d0fa 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/TargetProcess/TargetExecutionUtils.cpp @@ -14,7 +14,7 @@ namespace llvm { namespace orc { int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args, - Optional<StringRef> ProgramName) { + std::optional<StringRef> ProgramName) { std::vector<std::unique_ptr<char[]>> ArgVStorage; std::vector<char *> ArgV; @@ -39,5 +39,9 @@ int runAsMain(int (*Main)(int, char *[]), ArrayRef<std::string> Args, return Main(Args.size() + !!ProgramName, ArgV.data()); } +int runAsVoidFunction(int (*Func)(void)) { return Func(); } + +int runAsIntFunction(int (*Func)(int), int Arg) { return Func(Arg); } + } // End namespace orc. } // End namespace llvm. diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp index bb41bac32534..b425eec5f6d6 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp @@ -205,7 +205,7 @@ PerfJITEventListener::PerfJITEventListener() Dumpstream = std::make_unique<raw_fd_ostream>(DumpFd, true); - LLVMPerfJitHeader Header = {0}; + LLVMPerfJitHeader Header = {0, 0, 0, 0, 0, 0, 0, 0}; if (!FillMachine(Header)) return; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index 54ab00732330..a9aaff42433f 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -191,11 +191,9 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { // and pass this information to the memory manager if (MemMgr.needsToReserveAllocationSpace()) { uint64_t CodeSize = 0, RODataSize = 0, RWDataSize = 0; - uint32_t CodeAlign = 1, RODataAlign = 1, RWDataAlign = 1; - if (auto Err = computeTotalAllocSize(Obj, - CodeSize, CodeAlign, - RODataSize, RODataAlign, - RWDataSize, RWDataAlign)) + Align CodeAlign, RODataAlign, RWDataAlign; + if (auto Err = computeTotalAllocSize(Obj, CodeSize, CodeAlign, RODataSize, + RODataAlign, RWDataSize, RWDataAlign)) return std::move(Err); MemMgr.reserveAllocationSpace(CodeSize, CodeAlign, RODataSize, RODataAlign, RWDataSize, RWDataAlign); @@ -310,9 +308,12 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t)Addr) << " flags: " << *FlagsOrErr << "\n"); - if (!Name.empty()) // Skip absolute symbol relocations. - GlobalSymbolTable[Name] = - SymbolTableEntry(SectionID, Addr, *JITSymFlags); + // Skip absolute symbol relocations. + if (!Name.empty()) { + auto Result = GlobalSymbolTable.insert_or_assign( + Name, SymbolTableEntry(SectionID, Addr, *JITSymFlags)); + processNewSymbol(*I, Result.first->getValue()); + } } else if (SymType == object::SymbolRef::ST_Function || SymType == object::SymbolRef::ST_Data || SymType == object::SymbolRef::ST_Unknown || @@ -344,9 +345,12 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { << " SID: " << SectionID << " Offset: " << format("%p", (uintptr_t)SectOffset) << " flags: " << *FlagsOrErr << "\n"); - if (!Name.empty()) // Skip absolute symbol relocations - GlobalSymbolTable[Name] = - SymbolTableEntry(SectionID, SectOffset, *JITSymFlags); + // Skip absolute symbol relocations. + if (!Name.empty()) { + auto Result = GlobalSymbolTable.insert_or_assign( + Name, SymbolTableEntry(SectionID, SectOffset, *JITSymFlags)); + processNewSymbol(*I, Result.first->getValue()); + } } } @@ -457,13 +461,10 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { // assuming that all sections are allocated with the given alignment static uint64_t computeAllocationSizeForSections(std::vector<uint64_t> &SectionSizes, - uint64_t Alignment) { + Align Alignment) { uint64_t TotalSize = 0; - for (uint64_t SectionSize : SectionSizes) { - uint64_t AlignedSize = - (SectionSize + Alignment - 1) / Alignment * Alignment; - TotalSize += AlignedSize; - } + for (uint64_t SectionSize : SectionSizes) + TotalSize += alignTo(SectionSize, Alignment); return TotalSize; } @@ -531,13 +532,10 @@ static bool isTLS(const SectionRef Section) { // Compute an upper bound of the memory size that is required to load all // sections -Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, - uint64_t &CodeSize, - uint32_t &CodeAlign, - uint64_t &RODataSize, - uint32_t &RODataAlign, - uint64_t &RWDataSize, - uint32_t &RWDataAlign) { +Error RuntimeDyldImpl::computeTotalAllocSize( + const ObjectFile &Obj, uint64_t &CodeSize, Align &CodeAlign, + uint64_t &RODataSize, Align &RODataAlign, uint64_t &RWDataSize, + Align &RWDataAlign) { // Compute the size of all sections required for execution std::vector<uint64_t> CodeSectionSizes; std::vector<uint64_t> ROSectionSizes; @@ -554,8 +552,7 @@ Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, // Consider only the sections that are required to be loaded for execution if (IsRequired) { uint64_t DataSize = Section.getSize(); - uint64_t Alignment64 = Section.getAlignment(); - unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; + Align Alignment = Section.getAlignment(); bool IsCode = Section.isText(); bool IsReadOnly = isReadOnlyData(Section); bool IsTLS = isTLS(Section); @@ -571,7 +568,7 @@ Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, if (Name == ".eh_frame") PaddingSize += 4; if (StubBufSize != 0) - PaddingSize += getStubAlignment() - 1; + PaddingSize += getStubAlignment().value() - 1; uint64_t SectionSize = DataSize + PaddingSize + StubBufSize; @@ -604,12 +601,12 @@ Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, // single GOT entry. if (unsigned GotSize = computeGOTSize(Obj)) { RWSectionSizes.push_back(GotSize); - RWDataAlign = std::max<uint32_t>(RWDataAlign, getGOTEntrySize()); + RWDataAlign = std::max(RWDataAlign, Align(getGOTEntrySize())); } // Compute the size of all common symbols uint64_t CommonSize = 0; - uint32_t CommonAlign = 1; + Align CommonAlign; for (symbol_iterator I = Obj.symbol_begin(), E = Obj.symbol_end(); I != E; ++I) { Expected<uint32_t> FlagsOrErr = I->getFlags(); @@ -619,12 +616,12 @@ Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, if (*FlagsOrErr & SymbolRef::SF_Common) { // Add the common symbols to a list. We'll allocate them all below. uint64_t Size = I->getCommonSize(); - uint32_t Align = I->getAlignment(); + Align Alignment = Align(I->getAlignment()); // If this is the first common symbol, use its alignment as the alignment // for the common symbols section. if (CommonSize == 0) - CommonAlign = Align; - CommonSize = alignTo(CommonSize, Align) + Size; + CommonAlign = Alignment; + CommonSize = alignTo(CommonSize, Alignment) + Size; } } if (CommonSize != 0) { @@ -632,6 +629,11 @@ Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, RWDataAlign = std::max(RWDataAlign, CommonAlign); } + if (!CodeSectionSizes.empty()) { + // Add 64 bytes for a potential IFunc resolver stub + CodeSectionSizes.push_back(64); + } + // Compute the required allocation space for each different type of sections // (code, read-only data, read-write data) assuming that all sections are // allocated with the max alignment. Note that we cannot compute with the @@ -695,14 +697,13 @@ unsigned RuntimeDyldImpl::computeSectionStubBufSize(const ObjectFile &Obj, // Get section data size and alignment uint64_t DataSize = Section.getSize(); - uint64_t Alignment64 = Section.getAlignment(); + Align Alignment = Section.getAlignment(); // Add stubbuf size alignment - unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; - unsigned StubAlignment = getStubAlignment(); - unsigned EndAlignment = (DataSize | Alignment) & -(DataSize | Alignment); + Align StubAlignment = getStubAlignment(); + Align EndAlignment = commonAlignment(Alignment, DataSize); if (StubAlignment > EndAlignment) - StubBufSize += StubAlignment - EndAlignment; + StubBufSize += StubAlignment.value() - EndAlignment.value(); return StubBufSize; } @@ -801,9 +802,8 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, const SectionRef &Section, bool IsCode) { StringRef data; - uint64_t Alignment64 = Section.getAlignment(); + Align Alignment = Section.getAlignment(); - unsigned Alignment = (unsigned)Alignment64 & 0xffffffffL; unsigned PaddingSize = 0; unsigned StubBufSize = 0; bool IsRequired = isRequiredForExecution(Section); @@ -813,11 +813,6 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, bool IsTLS = isTLS(Section); uint64_t DataSize = Section.getSize(); - // An alignment of 0 (at least with ELF) is identical to an alignment of 1, - // while being more "polite". Other formats do not support 0-aligned sections - // anyway, so we should guarantee that the alignment is always at least 1. - Alignment = std::max(1u, Alignment); - Expected<StringRef> NameOrErr = Section.getName(); if (!NameOrErr) return NameOrErr.takeError(); @@ -854,7 +849,7 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, // section is remapped. if (StubBufSize != 0) { Alignment = std::max(Alignment, getStubAlignment()); - PaddingSize += getStubAlignment() - 1; + PaddingSize += getStubAlignment().value() - 1; } // Some sections, such as debug info, don't need to be loaded for execution. @@ -864,15 +859,16 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, if (!Allocate) Allocate = 1; if (IsTLS) { - auto TLSSection = - MemMgr.allocateTLSSection(Allocate, Alignment, SectionID, Name); + auto TLSSection = MemMgr.allocateTLSSection(Allocate, Alignment.value(), + SectionID, Name); Addr = TLSSection.InitializationImage; LoadAddress = TLSSection.Offset; } else if (IsCode) { - Addr = MemMgr.allocateCodeSection(Allocate, Alignment, SectionID, Name); + Addr = MemMgr.allocateCodeSection(Allocate, Alignment.value(), SectionID, + Name); } else { - Addr = MemMgr.allocateDataSection(Allocate, Alignment, SectionID, Name, - IsReadOnly); + Addr = MemMgr.allocateDataSection(Allocate, Alignment.value(), SectionID, + Name, IsReadOnly); } if (!Addr) report_fatal_error("Unable to allocate section memory!"); @@ -892,7 +888,7 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, // Align DataSize to stub alignment if we have any stubs (PaddingSize will // have been increased above to account for this). if (StubBufSize > 0) - DataSize &= -(uint64_t)getStubAlignment(); + DataSize &= -(uint64_t)getStubAlignment().value(); } LLVM_DEBUG(dbgs() << "emitSection SectionID: " << SectionID << " Name: " diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h index ac9d4d460217..f564b0035bff 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h @@ -57,7 +57,7 @@ private: getStubOrGOTAddrFor(StringRef StubContainerName, StringRef Symbol, bool IsInsideLoad, bool IsStubAddr) const; - Optional<uint64_t> getSectionLoadAddress(void *LocalAddr) const; + std::optional<uint64_t> getSectionLoadAddress(void *LocalAddr) const; IsSymbolValidFunction IsSymbolValid; GetSymbolInfoFunction GetSymbolInfo; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index c702584b7a33..2fe49fefae2d 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -1730,10 +1730,8 @@ RuntimeDyldELF::processRelocationRef( LLVM_DEBUG(dbgs() << " Create a new stub function\n"); uintptr_t BaseAddress = uintptr_t(Section.getAddress()); - uintptr_t StubAlignment = getStubAlignment(); StubAddress = - (BaseAddress + Section.getStubOffset() + StubAlignment - 1) & - -StubAlignment; + alignTo(BaseAddress + Section.getStubOffset(), getStubAlignment()); unsigned StubOffset = StubAddress - BaseAddress; Stubs[Value] = StubOffset; @@ -1784,10 +1782,8 @@ RuntimeDyldELF::processRelocationRef( LLVM_DEBUG(dbgs() << " Create a new stub function\n"); uintptr_t BaseAddress = uintptr_t(Section->getAddress()); - uintptr_t StubAlignment = getStubAlignment(); - StubAddress = - (BaseAddress + Section->getStubOffset() + StubAlignment - 1) & - -StubAlignment; + StubAddress = alignTo(BaseAddress + Section->getStubOffset(), + getStubAlignment()); unsigned StubOffset = StubAddress - BaseAddress; Stubs[Value] = StubOffset; createStubFunction((uint8_t *)StubAddress); @@ -2027,7 +2023,7 @@ void RuntimeDyldELF::processX86_64TLSRelocation( case ELF::R_X86_64_REX_GOTPCRELX: case ELF::R_X86_64_GOTPCRELX: IsGOTPCRel = true; - LLVM_FALLTHROUGH; + [[fallthrough]]; case ELF::R_X86_64_PLT32: IsSmallCodeModel = true; break; @@ -2292,18 +2288,75 @@ RelocationEntry RuntimeDyldELF::computeGOTOffsetRE(uint64_t GOTOffset, return RelocationEntry(GOTSectionID, GOTOffset, Type, SymbolOffset); } +void RuntimeDyldELF::processNewSymbol(const SymbolRef &ObjSymbol, SymbolTableEntry& Symbol) { + // This should never return an error as `processNewSymbol` wouldn't have been + // called if getFlags() returned an error before. + auto ObjSymbolFlags = cantFail(ObjSymbol.getFlags()); + + if (ObjSymbolFlags & SymbolRef::SF_Indirect) { + if (IFuncStubSectionID == 0) { + // Create a dummy section for the ifunc stubs. It will be actually + // allocated in finalizeLoad() below. + IFuncStubSectionID = Sections.size(); + Sections.push_back( + SectionEntry(".text.__llvm_IFuncStubs", nullptr, 0, 0, 0)); + // First 64B are reserverd for the IFunc resolver + IFuncStubOffset = 64; + } + + IFuncStubs.push_back(IFuncStub{IFuncStubOffset, Symbol}); + // Modify the symbol so that it points to the ifunc stub instead of to the + // resolver function. + Symbol = SymbolTableEntry(IFuncStubSectionID, IFuncStubOffset, + Symbol.getFlags()); + IFuncStubOffset += getMaxIFuncStubSize(); + } +} + Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, ObjSectionToIDMap &SectionMap) { if (IsMipsO32ABI) if (!PendingRelocs.empty()) return make_error<RuntimeDyldError>("Can't find matching LO16 reloc"); + // Create the IFunc stubs if necessary. This must be done before processing + // the GOT entries, as the IFunc stubs may create some. + if (IFuncStubSectionID != 0) { + uint8_t *IFuncStubsAddr = MemMgr.allocateCodeSection( + IFuncStubOffset, 1, IFuncStubSectionID, ".text.__llvm_IFuncStubs"); + if (!IFuncStubsAddr) + return make_error<RuntimeDyldError>( + "Unable to allocate memory for IFunc stubs!"); + Sections[IFuncStubSectionID] = + SectionEntry(".text.__llvm_IFuncStubs", IFuncStubsAddr, IFuncStubOffset, + IFuncStubOffset, 0); + + createIFuncResolver(IFuncStubsAddr); + + LLVM_DEBUG(dbgs() << "Creating IFunc stubs SectionID: " + << IFuncStubSectionID << " Addr: " + << Sections[IFuncStubSectionID].getAddress() << '\n'); + for (auto &IFuncStub : IFuncStubs) { + auto &Symbol = IFuncStub.OriginalSymbol; + LLVM_DEBUG(dbgs() << "\tSectionID: " << Symbol.getSectionID() + << " Offset: " << format("%p", Symbol.getOffset()) + << " IFuncStubOffset: " + << format("%p\n", IFuncStub.StubOffset)); + createIFuncStub(IFuncStubSectionID, 0, IFuncStub.StubOffset, + Symbol.getSectionID(), Symbol.getOffset()); + } + + IFuncStubSectionID = 0; + IFuncStubOffset = 0; + IFuncStubs.clear(); + } + // If necessary, allocate the global offset table if (GOTSectionID != 0) { // Allocate memory for the section size_t TotalSize = CurrentGOTIndex * getGOTEntrySize(); uint8_t *Addr = MemMgr.allocateDataSection(TotalSize, getGOTEntrySize(), - GOTSectionID, ".got", false); + GOTSectionID, ".got", false); if (!Addr) return make_error<RuntimeDyldError>("Unable to allocate memory for GOT!"); @@ -2326,7 +2379,7 @@ Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, section_iterator RelocatedSection = *RelSecOrErr; ObjSectionToIDMap::iterator i = SectionMap.find(*RelocatedSection); - assert (i != SectionMap.end()); + assert(i != SectionMap.end()); SectionToGOTMap[i->second] = GOTSectionID; } } @@ -2362,6 +2415,110 @@ bool RuntimeDyldELF::isCompatibleFile(const object::ObjectFile &Obj) const { return Obj.isELF(); } +void RuntimeDyldELF::createIFuncResolver(uint8_t *Addr) const { + if (Arch == Triple::x86_64) { + // The adddres of the GOT1 entry is in %r11, the GOT2 entry is in %r11+8 + // (see createIFuncStub() for details) + // The following code first saves all registers that contain the original + // function arguments as those registers are not saved by the resolver + // function. %r11 is saved as well so that the GOT2 entry can be updated + // afterwards. Then it calls the actual IFunc resolver function whose + // address is stored in GOT2. After the resolver function returns, all + // saved registers are restored and the return value is written to GOT1. + // Finally, jump to the now resolved function. + // clang-format off + const uint8_t StubCode[] = { + 0x57, // push %rdi + 0x56, // push %rsi + 0x52, // push %rdx + 0x51, // push %rcx + 0x41, 0x50, // push %r8 + 0x41, 0x51, // push %r9 + 0x41, 0x53, // push %r11 + 0x41, 0xff, 0x53, 0x08, // call *0x8(%r11) + 0x41, 0x5b, // pop %r11 + 0x41, 0x59, // pop %r9 + 0x41, 0x58, // pop %r8 + 0x59, // pop %rcx + 0x5a, // pop %rdx + 0x5e, // pop %rsi + 0x5f, // pop %rdi + 0x49, 0x89, 0x03, // mov %rax,(%r11) + 0xff, 0xe0 // jmp *%rax + }; + // clang-format on + static_assert(sizeof(StubCode) <= 64, + "maximum size of the IFunc resolver is 64B"); + memcpy(Addr, StubCode, sizeof(StubCode)); + } else { + report_fatal_error( + "IFunc resolver is not supported for target architecture"); + } +} + +void RuntimeDyldELF::createIFuncStub(unsigned IFuncStubSectionID, + uint64_t IFuncResolverOffset, + uint64_t IFuncStubOffset, + unsigned IFuncSectionID, + uint64_t IFuncOffset) { + auto &IFuncStubSection = Sections[IFuncStubSectionID]; + auto *Addr = IFuncStubSection.getAddressWithOffset(IFuncStubOffset); + + if (Arch == Triple::x86_64) { + // The first instruction loads a PC-relative address into %r11 which is a + // GOT entry for this stub. This initially contains the address to the + // IFunc resolver. We can use %r11 here as it's caller saved but not used + // to pass any arguments. In fact, x86_64 ABI even suggests using %r11 for + // code in the PLT. The IFunc resolver will use %r11 to update the GOT + // entry. + // + // The next instruction just jumps to the address contained in the GOT + // entry. As mentioned above, we do this two-step jump by first setting + // %r11 so that the IFunc resolver has access to it. + // + // The IFunc resolver of course also needs to know the actual address of + // the actual IFunc resolver function. This will be stored in a GOT entry + // right next to the first one for this stub. So, the IFunc resolver will + // be able to call it with %r11+8. + // + // In total, two adjacent GOT entries (+relocation) and one additional + // relocation are required: + // GOT1: Address of the IFunc resolver. + // GOT2: Address of the IFunc resolver function. + // IFuncStubOffset+3: 32-bit PC-relative address of GOT1. + uint64_t GOT1 = allocateGOTEntries(2); + uint64_t GOT2 = GOT1 + getGOTEntrySize(); + + RelocationEntry RE1(GOTSectionID, GOT1, ELF::R_X86_64_64, + IFuncResolverOffset, {}); + addRelocationForSection(RE1, IFuncStubSectionID); + RelocationEntry RE2(GOTSectionID, GOT2, ELF::R_X86_64_64, IFuncOffset, {}); + addRelocationForSection(RE2, IFuncSectionID); + + const uint8_t StubCode[] = { + 0x4c, 0x8d, 0x1d, 0x00, 0x00, 0x00, 0x00, // leaq 0x0(%rip),%r11 + 0x41, 0xff, 0x23 // jmpq *(%r11) + }; + assert(sizeof(StubCode) <= getMaxIFuncStubSize() && + "IFunc stub size must not exceed getMaxIFuncStubSize()"); + memcpy(Addr, StubCode, sizeof(StubCode)); + + // The PC-relative value starts 4 bytes from the end of the leaq + // instruction, so the addend is -4. + resolveGOTOffsetRelocation(IFuncStubSectionID, IFuncStubOffset + 3, + GOT1 - 4, ELF::R_X86_64_PC32); + } else { + report_fatal_error("IFunc stub is not supported for target architecture"); + } +} + +unsigned RuntimeDyldELF::getMaxIFuncStubSize() const { + if (Arch == Triple::x86_64) { + return 10; + } + return 0; +} + bool RuntimeDyldELF::relocationNeedsGot(const RelocationRef &R) const { unsigned RelTy = R.getType(); if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be) diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h index 1251036f4caa..dfdd98cb3a34 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -79,11 +79,11 @@ class RuntimeDyldELF : public RuntimeDyldImpl { return 0; } - unsigned getStubAlignment() override { + Align getStubAlignment() override { if (Arch == Triple::systemz) - return 8; + return Align(8); else - return 1; + return Align(1); } void setMipsABI(const ObjectFile &Obj) override; @@ -158,6 +158,40 @@ private: // Map between GOT relocation value and corresponding GOT offset std::map<RelocationValueRef, uint64_t> GOTOffsetMap; + /// The ID of the current IFunc stub section + unsigned IFuncStubSectionID = 0; + /// The current offset into the IFunc stub section + uint64_t IFuncStubOffset = 0; + + /// A IFunc stub and its original symbol + struct IFuncStub { + /// The offset of this stub in the IFunc stub section + uint64_t StubOffset; + /// The symbol table entry of the original symbol + SymbolTableEntry OriginalSymbol; + }; + + /// The IFunc stubs + SmallVector<IFuncStub, 2> IFuncStubs; + + /// Create the code for the IFunc resolver at the given address. This code + /// works together with the stubs created in createIFuncStub() to call the + /// resolver function and then jump to the real function address. + /// It must not be larger than 64B. + void createIFuncResolver(uint8_t *Addr) const; + /// Create the code for an IFunc stub for the IFunc that is defined in + /// section IFuncSectionID at offset IFuncOffset. The IFunc resolver created + /// by createIFuncResolver() is defined in the section IFuncStubSectionID at + /// offset IFuncResolverOffset. The code should be written into the section + /// with the id IFuncStubSectionID at the offset IFuncStubOffset. + void createIFuncStub(unsigned IFuncStubSectionID, + uint64_t IFuncResolverOffset, uint64_t IFuncStubOffset, + unsigned IFuncSectionID, uint64_t IFuncOffset); + /// Return the maximum size of a stub created by createIFuncStub() + unsigned getMaxIFuncStubSize() const; + + void processNewSymbol(const SymbolRef &ObjSymbol, + SymbolTableEntry &Entry) override; bool relocationNeedsGot(const RelocationRef &R) const override; bool relocationNeedsStub(const RelocationRef &R) const override; diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index a5bc181f8af9..bf33a2dec18a 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -312,7 +312,7 @@ protected: NotifyStubEmittedFunction NotifyStubEmitted; virtual unsigned getMaxStubSize() const = 0; - virtual unsigned getStubAlignment() = 0; + virtual Align getStubAlignment() = 0; bool HasError; std::string ErrorStr; @@ -417,10 +417,10 @@ protected: // Compute an upper bound of the memory that is required to load all // sections - Error computeTotalAllocSize(const ObjectFile &Obj, - uint64_t &CodeSize, uint32_t &CodeAlign, - uint64_t &RODataSize, uint32_t &RODataAlign, - uint64_t &RWDataSize, uint32_t &RWDataAlign); + Error computeTotalAllocSize(const ObjectFile &Obj, uint64_t &CodeSize, + Align &CodeAlign, uint64_t &RODataSize, + Align &RODataAlign, uint64_t &RWDataSize, + Align &RWDataAlign); // Compute GOT size unsigned computeGOTSize(const ObjectFile &Obj); @@ -435,6 +435,10 @@ protected: // Return size of Global Offset Table (GOT) entry virtual size_t getGOTEntrySize() { return 0; } + // Hook for the subclasses to do further processing when a symbol is added to + // the global symbol table. This function may modify the symbol table entry. + virtual void processNewSymbol(const SymbolRef &ObjSymbol, SymbolTableEntry& Entry) {} + // Return true if the relocation R may require allocating a GOT entry. virtual bool relocationNeedsGot(const RelocationRef &R) const { return false; @@ -527,7 +531,7 @@ public: std::map<StringRef, JITEvaluatedSymbol> getSymbolTable() const { std::map<StringRef, JITEvaluatedSymbol> Result; - for (auto &KV : GlobalSymbolTable) { + for (const auto &KV : GlobalSymbolTable) { auto SectionID = KV.second.getSectionID(); uint64_t SectionAddr = getSectionLoadAddress(SectionID); Result[KV.first()] = diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h index 14510e56b35a..342c4221ff0c 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFAArch64.h @@ -92,7 +92,7 @@ public: : RuntimeDyldCOFF(MM, Resolver, 8, COFF::IMAGE_REL_ARM64_ADDR64), ImageBase(0) {} - unsigned getStubAlignment() override { return 8; } + Align getStubAlignment() override { return Align(8); } unsigned getMaxStubSize() const override { return 20; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h index 03c38260bece..2a54728fd0bf 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h @@ -31,7 +31,7 @@ public: return 8; // 2-byte jmp instruction + 32-bit relative address + 2 byte pad } - unsigned getStubAlignment() override { return 1; } + Align getStubAlignment() override { return Align(1); } Expected<object::relocation_iterator> processRelocationRef(unsigned SectionID, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h index dd66ff7ecf70..3859f36ac4bd 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h @@ -53,7 +53,7 @@ public: return 16; // 8-byte load instructions, 4-byte jump, 4-byte padding } - unsigned getStubAlignment() override { return 1; } + Align getStubAlignment() override { return Align(1); } Expected<object::relocation_iterator> processRelocationRef(unsigned SectionID, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h index 9df3e2e3c3bf..89156b992d87 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h @@ -59,7 +59,7 @@ public: : RuntimeDyldCOFF(MM, Resolver, 8, COFF::IMAGE_REL_AMD64_ADDR64), ImageBase(0) {} - unsigned getStubAlignment() override { return 1; } + Align getStubAlignment() override { return Align(1); } // 2-byte jmp instruction + 32-bit relative address + 64-bit absolute jump unsigned getMaxStubSize() const override { return 14; } diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h index f2ee1b06d494..701cc3a88149 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h @@ -28,7 +28,7 @@ public: unsigned getMaxStubSize() const override { return 8; } - unsigned getStubAlignment() override { return 8; } + Align getStubAlignment() override { return Align(8); } /// Extract the addend encoded in the instruction / memory location. Expected<int64_t> decodeAddend(const RelocationEntry &RE) const { @@ -118,7 +118,7 @@ public: (void)p; assert((*p & 0x3B000000) == 0x39000000 && "Only expected load / store instructions."); - LLVM_FALLTHROUGH; + [[fallthrough]]; } case MachO::ARM64_RELOC_PAGEOFF12: { // Verify that the relocation points to one of the expected load / store @@ -222,7 +222,7 @@ public: assert((*p & 0x3B000000) == 0x39000000 && "Only expected load / store instructions."); (void)p; - LLVM_FALLTHROUGH; + [[fallthrough]]; } case MachO::ARM64_RELOC_PAGEOFF12: { // Verify that the relocation points to one of the expected load / store @@ -453,13 +453,13 @@ private: // FIXME: There must be a better way to do this then to check and fix the // alignment every time!!! uintptr_t BaseAddress = uintptr_t(Section.getAddress()); - uintptr_t StubAlignment = getStubAlignment(); + uintptr_t StubAlignment = getStubAlignment().value(); uintptr_t StubAddress = (BaseAddress + Section.getStubOffset() + StubAlignment - 1) & -StubAlignment; unsigned StubOffset = StubAddress - BaseAddress; Stubs[Value] = StubOffset; - assert(((StubAddress % getStubAlignment()) == 0) && + assert(isAligned(getStubAlignment(), StubAddress) && "GOT entry not aligned"); RelocationEntry GOTRE(RE.SectionID, StubOffset, MachO::ARM64_RELOC_UNSIGNED, Value.Offset, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h index fcf723aaea28..79b558eb7796 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h @@ -30,7 +30,7 @@ public: unsigned getMaxStubSize() const override { return 8; } - unsigned getStubAlignment() override { return 4; } + Align getStubAlignment() override { return Align(4); } Expected<JITSymbolFlags> getJITSymbolFlags(const SymbolRef &SR) override { auto Flags = RuntimeDyldImpl::getJITSymbolFlags(SR); diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h index d029d3266f79..a983e22671b2 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h @@ -27,7 +27,7 @@ public: unsigned getMaxStubSize() const override { return 0; } - unsigned getStubAlignment() override { return 1; } + Align getStubAlignment() override { return Align(1); } Expected<relocation_iterator> processRelocationRef(unsigned SectionID, relocation_iterator RelI, diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h index a4d91cf338cb..bd0d72f9e117 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h @@ -27,7 +27,7 @@ public: unsigned getMaxStubSize() const override { return 8; } - unsigned getStubAlignment() override { return 8; } + Align getStubAlignment() override { return Align(8); } Expected<relocation_iterator> processRelocationRef(unsigned SectionID, relocation_iterator RelI, |