diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp')
| -rw-r--r-- | contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp | 486 |
1 files changed, 67 insertions, 419 deletions
diff --git a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp index c368c1e37134..2ac32293e4db 100644 --- a/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/contrib/llvm-project/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -9,12 +9,12 @@ #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" #include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/ExecutionEngine/Orc/ExecutorProcessControl.h" #include "llvm/ExecutionEngine/Orc/MachOPlatform.h" #include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/Shared/OrcError.h" -#include "llvm/ExecutionEngine/Orc/TargetProcessControl.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/IRBuilder.h" @@ -124,7 +124,6 @@ private: /// some runtime API, including __cxa_atexit, dlopen, and dlclose. class GenericLLVMIRPlatformSupport : public LLJIT::PlatformSupport { public: - // GenericLLVMIRPlatform &P) : P(P) { GenericLLVMIRPlatformSupport(LLJIT &J) : J(J), InitFunctionPrefix(J.mangle("__orc_init_func.")) { @@ -503,395 +502,26 @@ GlobalCtorDtorScraper::operator()(ThreadSafeModule TSM, return std::move(TSM); } -class MachOPlatformSupport : public LLJIT::PlatformSupport { +/// Inactive Platform Support +/// +/// Explicitly disables platform support. JITDylibs are not scanned for special +/// init/deinit symbols. No runtime API interposes are injected. +class InactivePlatformSupport : public LLJIT::PlatformSupport { public: - using DLOpenType = void *(*)(const char *Name, int Mode); - using DLCloseType = int (*)(void *Handle); - using DLSymType = void *(*)(void *Handle, const char *Name); - using DLErrorType = const char *(*)(); - - struct DlFcnValues { - Optional<void *> RTLDDefault; - DLOpenType dlopen = nullptr; - DLCloseType dlclose = nullptr; - DLSymType dlsym = nullptr; - DLErrorType dlerror = nullptr; - }; - - static Expected<std::unique_ptr<MachOPlatformSupport>> - Create(LLJIT &J, JITDylib &PlatformJITDylib) { - - // Make process symbols visible. - { - std::string ErrMsg; - auto Lib = sys::DynamicLibrary::getPermanentLibrary(nullptr, &ErrMsg); - if (!Lib.isValid()) - return make_error<StringError>(std::move(ErrMsg), - inconvertibleErrorCode()); - } - - DlFcnValues DlFcn; - - // Add support for RTLDDefault on known platforms. -#ifdef __APPLE__ - DlFcn.RTLDDefault = reinterpret_cast<void *>(-2); -#endif // __APPLE__ - - if (auto Err = hookUpFunction(DlFcn.dlopen, "dlopen")) - return std::move(Err); - if (auto Err = hookUpFunction(DlFcn.dlclose, "dlclose")) - return std::move(Err); - if (auto Err = hookUpFunction(DlFcn.dlsym, "dlsym")) - return std::move(Err); - if (auto Err = hookUpFunction(DlFcn.dlerror, "dlerror")) - return std::move(Err); - - std::unique_ptr<MachOPlatformSupport> MP( - new MachOPlatformSupport(J, PlatformJITDylib, DlFcn)); - return std::move(MP); - } + InactivePlatformSupport() = default; Error initialize(JITDylib &JD) override { - LLVM_DEBUG({ - dbgs() << "MachOPlatformSupport initializing \"" << JD.getName() - << "\"\n"; - }); - - auto InitSeq = MP.getInitializerSequence(JD); - if (!InitSeq) - return InitSeq.takeError(); - - // If ObjC is not enabled but there are JIT'd ObjC inits then return - // an error. - if (!objCRegistrationEnabled()) - for (auto &KV : *InitSeq) { - if (!KV.second.getObjCSelRefsSections().empty() || - !KV.second.getObjCClassListSections().empty()) - return make_error<StringError>("JITDylib " + KV.first->getName() + - " contains objc metadata but objc" - " is not enabled", - inconvertibleErrorCode()); - } - - // Run the initializers. - for (auto &KV : *InitSeq) { - if (objCRegistrationEnabled()) { - KV.second.registerObjCSelectors(); - if (auto Err = KV.second.registerObjCClasses()) { - // FIXME: Roll back registrations on error? - return Err; - } - } - KV.second.runModInits(); - } - + LLVM_DEBUG(dbgs() << "InactivePlatformSupport: no initializers running for " + << JD.getName() << "\n"); return Error::success(); } Error deinitialize(JITDylib &JD) override { - auto &ES = J.getExecutionSession(); - if (auto DeinitSeq = MP.getDeinitializerSequence(JD)) { - for (auto &KV : *DeinitSeq) { - auto DSOHandleName = ES.intern("___dso_handle"); - - // FIXME: Run DeInits here. - auto Result = ES.lookup( - {{KV.first, JITDylibLookupFlags::MatchAllSymbols}}, - SymbolLookupSet(DSOHandleName, - SymbolLookupFlags::WeaklyReferencedSymbol)); - if (!Result) - return Result.takeError(); - if (Result->empty()) - continue; - assert(Result->count(DSOHandleName) && - "Result does not contain __dso_handle"); - auto *DSOHandle = jitTargetAddressToPointer<void *>( - Result->begin()->second.getAddress()); - AtExitMgr.runAtExits(DSOHandle); - } - } else - return DeinitSeq.takeError(); + LLVM_DEBUG( + dbgs() << "InactivePlatformSupport: no deinitializers running for " + << JD.getName() << "\n"); return Error::success(); } - -private: - template <typename FunctionPtrTy> - static Error hookUpFunction(FunctionPtrTy &Fn, const char *Name) { - if (auto *FnAddr = sys::DynamicLibrary::SearchForAddressOfSymbol(Name)) { - Fn = reinterpret_cast<FunctionPtrTy>(Fn); - return Error::success(); - } - - return make_error<StringError>((Twine("Can not enable MachO JIT Platform: " - "missing function: ") + - Name) - .str(), - inconvertibleErrorCode()); - } - - MachOPlatformSupport(LLJIT &J, JITDylib &PlatformJITDylib, DlFcnValues DlFcn) - : J(J), MP(setupPlatform(J)), DlFcn(std::move(DlFcn)) { - - SymbolMap HelperSymbols; - - // platform and atexit helpers. - HelperSymbols[J.mangleAndIntern("__lljit.platform_support_instance")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(this), JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.cxa_atexit_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(registerAtExitHelper), - JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.run_atexits_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(runAtExitsHelper), - JITSymbolFlags()); - - // dlfcn helpers. - HelperSymbols[J.mangleAndIntern("__lljit.dlopen_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(dlopenHelper), - JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.dlclose_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(dlcloseHelper), - JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.dlsym_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(dlsymHelper), - JITSymbolFlags()); - HelperSymbols[J.mangleAndIntern("__lljit.dlerror_helper")] = - JITEvaluatedSymbol(pointerToJITTargetAddress(dlerrorHelper), - JITSymbolFlags()); - - cantFail( - PlatformJITDylib.define(absoluteSymbols(std::move(HelperSymbols)))); - cantFail(MP.setupJITDylib(J.getMainJITDylib())); - cantFail(J.addIRModule(PlatformJITDylib, createPlatformRuntimeModule())); - } - - static MachOPlatform &setupPlatform(LLJIT &J) { - auto Tmp = std::make_unique<MachOPlatform>( - J.getExecutionSession(), - static_cast<ObjectLinkingLayer &>(J.getObjLinkingLayer()), - createStandardSymbolsObject(J)); - auto &MP = *Tmp; - J.getExecutionSession().setPlatform(std::move(Tmp)); - return MP; - } - - static std::unique_ptr<MemoryBuffer> createStandardSymbolsObject(LLJIT &J) { - LLVMContext Ctx; - Module M("__standard_symbols", Ctx); - M.setDataLayout(J.getDataLayout()); - - auto *Int64Ty = Type::getInt64Ty(Ctx); - - auto *DSOHandle = - new GlobalVariable(M, Int64Ty, true, GlobalValue::ExternalLinkage, - ConstantInt::get(Int64Ty, 0), "__dso_handle"); - DSOHandle->setVisibility(GlobalValue::DefaultVisibility); - - return cantFail(J.getIRCompileLayer().getCompiler()(M)); - } - - ThreadSafeModule createPlatformRuntimeModule() { - auto Ctx = std::make_unique<LLVMContext>(); - auto M = std::make_unique<Module>("__standard_lib", *Ctx); - M->setDataLayout(J.getDataLayout()); - - auto *MachOPlatformSupportTy = - StructType::create(*Ctx, "lljit.MachOPlatformSupport"); - - auto *PlatformInstanceDecl = new GlobalVariable( - *M, MachOPlatformSupportTy, true, GlobalValue::ExternalLinkage, nullptr, - "__lljit.platform_support_instance"); - - auto *Int8Ty = Type::getInt8Ty(*Ctx); - auto *IntTy = Type::getIntNTy(*Ctx, sizeof(int) * CHAR_BIT); - auto *VoidTy = Type::getVoidTy(*Ctx); - auto *BytePtrTy = PointerType::getUnqual(Int8Ty); - auto *AtExitCallbackTy = FunctionType::get(VoidTy, {BytePtrTy}, false); - auto *AtExitCallbackPtrTy = PointerType::getUnqual(AtExitCallbackTy); - - addHelperAndWrapper( - *M, "__cxa_atexit", - FunctionType::get(IntTy, {AtExitCallbackPtrTy, BytePtrTy, BytePtrTy}, - false), - GlobalValue::DefaultVisibility, "__lljit.cxa_atexit_helper", - {PlatformInstanceDecl}); - - addHelperAndWrapper(*M, "dlopen", - FunctionType::get(BytePtrTy, {BytePtrTy, IntTy}, false), - GlobalValue::DefaultVisibility, "__lljit.dlopen_helper", - {PlatformInstanceDecl}); - - addHelperAndWrapper(*M, "dlclose", - FunctionType::get(IntTy, {BytePtrTy}, false), - GlobalValue::DefaultVisibility, - "__lljit.dlclose_helper", {PlatformInstanceDecl}); - - addHelperAndWrapper( - *M, "dlsym", - FunctionType::get(BytePtrTy, {BytePtrTy, BytePtrTy}, false), - GlobalValue::DefaultVisibility, "__lljit.dlsym_helper", - {PlatformInstanceDecl}); - - addHelperAndWrapper(*M, "dlerror", FunctionType::get(BytePtrTy, {}, false), - GlobalValue::DefaultVisibility, - "__lljit.dlerror_helper", {PlatformInstanceDecl}); - - return ThreadSafeModule(std::move(M), std::move(Ctx)); - } - - static void registerAtExitHelper(void *Self, void (*F)(void *), void *Ctx, - void *DSOHandle) { - static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.registerAtExit( - F, Ctx, DSOHandle); - } - - static void runAtExitsHelper(void *Self, void *DSOHandle) { - static_cast<MachOPlatformSupport *>(Self)->AtExitMgr.runAtExits(DSOHandle); - } - - void *jit_dlopen(const char *Path, int Mode) { - JITDylib *JDToOpen = nullptr; - // FIXME: Do the right thing with Mode flags. - { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - - // Clear any existing error messages. - dlErrorMsgs.erase(std::this_thread::get_id()); - - if (auto *JD = J.getExecutionSession().getJITDylibByName(Path)) { - auto I = JDRefCounts.find(JD); - if (I != JDRefCounts.end()) { - ++I->second; - return JD; - } - - JDRefCounts[JD] = 1; - JDToOpen = JD; - } - } - - if (JDToOpen) { - if (auto Err = initialize(*JDToOpen)) { - recordError(std::move(Err)); - return 0; - } - } - - // Fall through to dlopen if no JITDylib found for Path. - return DlFcn.dlopen(Path, Mode); - } - - static void *dlopenHelper(void *Self, const char *Path, int Mode) { - return static_cast<MachOPlatformSupport *>(Self)->jit_dlopen(Path, Mode); - } - - int jit_dlclose(void *Handle) { - JITDylib *JDToClose = nullptr; - - { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - - // Clear any existing error messages. - dlErrorMsgs.erase(std::this_thread::get_id()); - - auto I = JDRefCounts.find(Handle); - if (I != JDRefCounts.end()) { - --I->second; - if (I->second == 0) { - JDRefCounts.erase(I); - JDToClose = static_cast<JITDylib *>(Handle); - } else - return 0; - } - } - - if (JDToClose) { - if (auto Err = deinitialize(*JDToClose)) { - recordError(std::move(Err)); - return -1; - } - return 0; - } - - // Fall through to dlclose if no JITDylib found for Path. - return DlFcn.dlclose(Handle); - } - - static int dlcloseHelper(void *Self, void *Handle) { - return static_cast<MachOPlatformSupport *>(Self)->jit_dlclose(Handle); - } - - void *jit_dlsym(void *Handle, const char *Name) { - JITDylibSearchOrder JITSymSearchOrder; - - // FIXME: RTLD_NEXT, RTLD_SELF not supported. - { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - - // Clear any existing error messages. - dlErrorMsgs.erase(std::this_thread::get_id()); - - if (JDRefCounts.count(Handle)) { - JITSymSearchOrder.push_back( - {static_cast<JITDylib *>(Handle), - JITDylibLookupFlags::MatchExportedSymbolsOnly}); - } else if (Handle == DlFcn.RTLDDefault) { - for (auto &KV : JDRefCounts) - JITSymSearchOrder.push_back( - {static_cast<JITDylib *>(KV.first), - JITDylibLookupFlags::MatchExportedSymbolsOnly}); - } - } - - if (!JITSymSearchOrder.empty()) { - auto MangledName = J.mangleAndIntern(Name); - SymbolLookupSet Syms(MangledName, - SymbolLookupFlags::WeaklyReferencedSymbol); - if (auto Result = J.getExecutionSession().lookup(JITSymSearchOrder, Syms, - LookupKind::DLSym)) { - auto I = Result->find(MangledName); - if (I != Result->end()) - return jitTargetAddressToPointer<void *>(I->second.getAddress()); - } else { - recordError(Result.takeError()); - return 0; - } - } - - // Fall through to dlsym. - return DlFcn.dlsym(Handle, Name); - } - - static void *dlsymHelper(void *Self, void *Handle, const char *Name) { - return static_cast<MachOPlatformSupport *>(Self)->jit_dlsym(Handle, Name); - } - - const char *jit_dlerror() { - { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - auto I = dlErrorMsgs.find(std::this_thread::get_id()); - if (I != dlErrorMsgs.end()) - return I->second->c_str(); - } - return DlFcn.dlerror(); - } - - static const char *dlerrorHelper(void *Self) { - return static_cast<MachOPlatformSupport *>(Self)->jit_dlerror(); - } - - void recordError(Error Err) { - std::lock_guard<std::mutex> Lock(PlatformSupportMutex); - dlErrorMsgs[std::this_thread::get_id()] = - std::make_unique<std::string>(toString(std::move(Err))); - } - - std::mutex PlatformSupportMutex; - LLJIT &J; - MachOPlatform &MP; - DlFcnValues DlFcn; - ItaniumCXAAtExitSupport AtExitMgr; - DenseMap<void *, unsigned> JDRefCounts; - std::map<std::thread::id, std::unique_ptr<std::string>> dlErrorMsgs; }; } // end anonymous namespace @@ -922,7 +552,8 @@ Error LLJITBuilderState::prepareForConstruction() { } LLVM_DEBUG({ - dbgs() << " JITTargetMachineBuilder is " << JTMB << "\n" + dbgs() << " JITTargetMachineBuilder is " + << JITTargetMachineBuilderPrinter(*JTMB, " ") << " Pre-constructed ExecutionSession: " << (ES ? "Yes" : "No") << "\n" << " DataLayout: "; @@ -944,6 +575,22 @@ Error LLJITBuilderState::prepareForConstruction() { dbgs() << "\n"; }); + // If neither ES nor EPC has been set then create an EPC instance. + if (!ES && !EPC) { + LLVM_DEBUG({ + dbgs() << "ExecutorProcessControl not specified, " + "Creating SelfExecutorProcessControl instance\n"; + }); + if (auto EPCOrErr = SelfExecutorProcessControl::Create()) + EPC = std::move(*EPCOrErr); + else + return EPCOrErr.takeError(); + } else + LLVM_DEBUG({ + dbgs() << "Using explicitly specified ExecutorProcessControl instance " + << EPC.get() << "\n"; + }); + // If the client didn't configure any linker options then auto-configure the // JIT linker. if (!CreateObjectLinkingLayer) { @@ -954,16 +601,9 @@ Error LLJITBuilderState::prepareForConstruction() { JTMB->setRelocationModel(Reloc::PIC_); JTMB->setCodeModel(CodeModel::Small); CreateObjectLinkingLayer = - [TPC = this->TPC]( - ExecutionSession &ES, - const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> { - std::unique_ptr<ObjectLinkingLayer> ObjLinkingLayer; - if (TPC) - ObjLinkingLayer = - std::make_unique<ObjectLinkingLayer>(ES, TPC->getMemMgr()); - else - ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>( - ES, std::make_unique<jitlink::InProcessMemoryManager>()); + [](ExecutionSession &ES, + const Triple &) -> Expected<std::unique_ptr<ObjectLayer>> { + auto ObjLinkingLayer = std::make_unique<ObjectLinkingLayer>(ES); ObjLinkingLayer->addPlugin(std::make_unique<EHFrameRegistrationPlugin>( ES, std::make_unique<jitlink::InProcessEHFrameRegistrar>())); return std::move(ObjLinkingLayer); @@ -1022,18 +662,18 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs // a new SectionMemoryManager for each object. auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); }; - auto ObjLinkingLayer = + auto Layer = std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr)); if (S.JTMB->getTargetTriple().isOSBinFormatCOFF()) { - ObjLinkingLayer->setOverrideObjectFlagsWithResponsibilityFlags(true); - ObjLinkingLayer->setAutoClaimResponsibilityForObjectSymbols(true); + Layer->setOverrideObjectFlagsWithResponsibilityFlags(true); + Layer->setAutoClaimResponsibilityForObjectSymbols(true); } // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence // errors from some GCC / libstdc++ bots. Remove this conversion (i.e. // just return ObjLinkingLayer) once those bots are upgraded. - return std::unique_ptr<ObjectLayer>(std::move(ObjLinkingLayer)); + return std::unique_ptr<ObjectLayer>(std::move(Layer)); } Expected<std::unique_ptr<IRCompileLayer::IRCompiler>> @@ -1057,11 +697,25 @@ LLJIT::createCompileFunction(LLJITBuilderState &S, } LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) - : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()), Main(), - DL(""), TT(S.JTMB->getTargetTriple()) { + : DL(""), TT(S.JTMB->getTargetTriple()) { ErrorAsOutParameter _(&Err); + assert(!(S.EPC && S.ES) && "EPC and ES should not both be set"); + + if (S.EPC) { + ES = std::make_unique<ExecutionSession>(std::move(S.EPC)); + } else if (S.ES) + ES = std::move(S.ES); + else { + if (auto EPC = SelfExecutorProcessControl::Create()) { + ES = std::make_unique<ExecutionSession>(std::move(*EPC)); + } else { + Err = EPC.takeError(); + return; + } + } + if (auto MainOrErr = this->ES->createJITDylib("main")) Main = &*MainOrErr; else { @@ -1104,20 +758,16 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) InitHelperTransformLayer->setCloneToNewContextOnEmit(true); CompileThreads = std::make_unique<ThreadPool>(hardware_concurrency(S.NumCompileThreads)); - ES->setDispatchMaterialization( - [this](std::unique_ptr<MaterializationUnit> MU, - std::unique_ptr<MaterializationResponsibility> MR) { - // FIXME: We should be able to use move-capture here, but ThreadPool's - // AsyncTaskTys are std::functions rather than unique_functions - // (because MSVC's std::packaged_tasks don't support move-only types). - // Fix this when all the above gets sorted out. - CompileThreads->async( - [UnownedMU = MU.release(), UnownedMR = MR.release()]() mutable { - std::unique_ptr<MaterializationUnit> MU(UnownedMU); - std::unique_ptr<MaterializationResponsibility> MR(UnownedMR); - MU->materialize(std::move(MR)); - }); - }); + ES->setDispatchTask([this](std::unique_ptr<Task> T) { + // FIXME: We should be able to use move-capture here, but ThreadPool's + // AsyncTaskTys are std::functions rather than unique_functions + // (because MSVC's std::packaged_tasks don't support move-only types). + // Fix this when all the above gets sorted out. + CompileThreads->async([UnownedT = T.release()]() mutable { + std::unique_ptr<Task> T(UnownedT); + T->run(); + }); + }); } if (S.SetUpPlatform) @@ -1155,12 +805,10 @@ void setUpGenericLLVMIRPlatform(LLJIT &J) { J.setPlatformSupport(std::make_unique<GenericLLVMIRPlatformSupport>(J)); } -Error setUpMachOPlatform(LLJIT &J) { - LLVM_DEBUG({ dbgs() << "Setting up MachOPlatform support for LLJIT\n"; }); - auto MP = MachOPlatformSupport::Create(J, J.getMainJITDylib()); - if (!MP) - return MP.takeError(); - J.setPlatformSupport(std::move(*MP)); +Error setUpInactivePlatform(LLJIT &J) { + LLVM_DEBUG( + { dbgs() << "Explicitly deactivated platform support for LLJIT\n"; }); + J.setPlatformSupport(std::make_unique<InactivePlatformSupport>()); return Error::success(); } |