diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2019-10-23 17:51:42 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2019-10-23 17:51:42 +0000 |
commit | 1d5ae1026e831016fc29fd927877c86af904481f (patch) | |
tree | 2cdfd12620fcfa5d9e4a0389f85368e8e36f63f9 /include/llvm/ExecutionEngine/Orc | |
parent | e6d1592492a3a379186bfb02bd0f4eda0669c0d5 (diff) |
Notes
Diffstat (limited to 'include/llvm/ExecutionEngine/Orc')
18 files changed, 573 insertions, 162 deletions
diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 5f593a27cad6..7946b5b7b209 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -26,6 +26,7 @@ #include "llvm/ExecutionEngine/Orc/LazyReexports.h" #include "llvm/ExecutionEngine/Orc/Legacy.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/Constant.h" @@ -91,6 +92,8 @@ public: /// Sets the partition function. void setPartitionFunction(PartitionFunction Partition); + /// Sets the ImplSymbolMap + void setImplMap(ImplSymbolMap *Imp); /// Emits the given module. This should not be called by clients: it will be /// called by the JIT when a definition added via the add method is requested. void emit(MaterializationResponsibility R, ThreadSafeModule TSM) override; @@ -128,6 +131,7 @@ private: PerDylibResourcesMap DylibResources; PartitionFunction Partition = compileRequested; SymbolLinkagePromoter PromoteSymbols; + ImplSymbolMap *AliaseeImpls = nullptr; }; /// Compile-on-demand layer. @@ -187,7 +191,7 @@ private: std::unique_ptr<ResourceOwner<ResourceT>> wrapOwnership(ResourcePtrT ResourcePtr) { using RO = ResourceOwnerImpl<ResourceT, ResourcePtrT>; - return llvm::make_unique<RO>(std::move(ResourcePtr)); + return std::make_unique<RO>(std::move(ResourcePtr)); } struct LogicalDylib { @@ -440,7 +444,7 @@ private: return Error::success(); // Create the GlobalValues module. - auto GVsM = llvm::make_unique<Module>((SrcM.getName() + ".globals").str(), + auto GVsM = std::make_unique<Module>((SrcM.getName() + ".globals").str(), SrcM.getContext()); GVsM->setDataLayout(DL); @@ -633,7 +637,7 @@ private: NewName += F->getName(); } - auto M = llvm::make_unique<Module>(NewName, SrcM.getContext()); + auto M = std::make_unique<Module>(NewName, SrcM.getContext()); M->setDataLayout(SrcM.getDataLayout()); ValueToValueMapTy VMap; diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h index 94a5618233e4..4f22a4c38796 100644 --- a/include/llvm/ExecutionEngine/Orc/Core.h +++ b/include/llvm/ExecutionEngine/Orc/Core.h @@ -14,6 +14,7 @@ #define LLVM_EXECUTIONENGINE_ORC_CORE_H #include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/FunctionExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" #include "llvm/ExecutionEngine/OrcV1Deprecation.h" @@ -51,8 +52,7 @@ using SymbolMap = DenseMap<SymbolStringPtr, JITEvaluatedSymbol>; /// A map from symbol names (as SymbolStringPtrs) to JITSymbolFlags. using SymbolFlagsMap = DenseMap<SymbolStringPtr, JITSymbolFlags>; -/// A base class for materialization failures that allows the failing -/// symbols to be obtained for logging. +/// A map from JITDylibs to sets of symbols. using SymbolDependenceMap = DenseMap<JITDylib *, SymbolNameSet>; /// A list of (JITDylib*, bool) pairs. @@ -108,7 +108,7 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases); raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S); /// Callback to notify client that symbols have been resolved. -using SymbolsResolvedCallback = std::function<void(Expected<SymbolMap>)>; +using SymbolsResolvedCallback = unique_function<void(Expected<SymbolMap>)>; /// Callback to register the dependencies for a given query. using RegisterDependenciesFunction = @@ -124,13 +124,13 @@ class FailedToMaterialize : public ErrorInfo<FailedToMaterialize> { public: static char ID; - FailedToMaterialize(SymbolNameSet Symbols); + FailedToMaterialize(std::shared_ptr<SymbolDependenceMap> Symbols); std::error_code convertToErrorCode() const override; void log(raw_ostream &OS) const override; - const SymbolNameSet &getSymbols() const { return Symbols; } + const SymbolDependenceMap &getSymbols() const { return *Symbols; } private: - SymbolNameSet Symbols; + std::shared_ptr<SymbolDependenceMap> Symbols; }; /// Used to notify clients when symbols can not be found during a lookup. @@ -205,12 +205,26 @@ public: /// symbols must be ones covered by this MaterializationResponsibility /// instance. Individual calls to this method may resolve a subset of the /// symbols, but all symbols must have been resolved prior to calling emit. - void notifyResolved(const SymbolMap &Symbols); + /// + /// This method will return an error if any symbols being resolved have been + /// moved to the error state due to the failure of a dependency. If this + /// method returns an error then clients should log it and call + /// failMaterialize. If no dependencies have been registered for the + /// symbols covered by this MaterializationResponsibiility then this method + /// is guaranteed to return Error::success() and can be wrapped with cantFail. + Error notifyResolved(const SymbolMap &Symbols); /// Notifies the target JITDylib (and any pending queries on that JITDylib) /// that all symbols covered by this MaterializationResponsibility instance /// have been emitted. - void notifyEmitted(); + /// + /// This method will return an error if any symbols being resolved have been + /// moved to the error state due to the failure of a dependency. If this + /// method returns an error then clients should log it and call + /// failMaterialize. If no dependencies have been registered for the + /// symbols covered by this MaterializationResponsibiility then this method + /// is guaranteed to return Error::success() and can be wrapped with cantFail. + Error notifyEmitted(); /// Adds new symbols to the JITDylib and this responsibility instance. /// JITDylib entries start out in the materializing state. @@ -346,7 +360,7 @@ private: /// inline std::unique_ptr<AbsoluteSymbolsMaterializationUnit> absoluteSymbols(SymbolMap Symbols, VModuleKey K = VModuleKey()) { - return llvm::make_unique<AbsoluteSymbolsMaterializationUnit>( + return std::make_unique<AbsoluteSymbolsMaterializationUnit>( std::move(Symbols), std::move(K)); } @@ -390,7 +404,7 @@ private: /// \endcode inline std::unique_ptr<ReExportsMaterializationUnit> symbolAliases(SymbolAliasMap Aliases, VModuleKey K = VModuleKey()) { - return llvm::make_unique<ReExportsMaterializationUnit>( + return std::make_unique<ReExportsMaterializationUnit>( nullptr, true, std::move(Aliases), std::move(K)); } @@ -402,7 +416,7 @@ symbolAliases(SymbolAliasMap Aliases, VModuleKey K = VModuleKey()) { inline std::unique_ptr<ReExportsMaterializationUnit> reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, bool MatchNonExported = false, VModuleKey K = VModuleKey()) { - return llvm::make_unique<ReExportsMaterializationUnit>( + return std::make_unique<ReExportsMaterializationUnit>( &SourceJD, MatchNonExported, std::move(Aliases), std::move(K)); } @@ -411,32 +425,13 @@ reexports(JITDylib &SourceJD, SymbolAliasMap Aliases, Expected<SymbolAliasMap> buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols); -/// ReexportsGenerator can be used with JITDylib::setGenerator to automatically -/// re-export a subset of the source JITDylib's symbols in the target. -class ReexportsGenerator { -public: - using SymbolPredicate = std::function<bool(SymbolStringPtr)>; - - /// Create a reexports generator. If an Allow predicate is passed, only - /// symbols for which the predicate returns true will be reexported. If no - /// Allow predicate is passed, all symbols will be exported. - ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false, - SymbolPredicate Allow = SymbolPredicate()); - - Expected<SymbolNameSet> operator()(JITDylib &JD, const SymbolNameSet &Names); - -private: - JITDylib &SourceJD; - bool MatchNonExported = false; - SymbolPredicate Allow; -}; - /// Represents the state that a symbol has reached during materialization. enum class SymbolState : uint8_t { Invalid, /// No symbol should be in this state. NeverSearched, /// Added to the symbol table, never queried. Materializing, /// Queried, materialization begun. Resolved, /// Assigned address, still materializing. + Emitted, /// Emitted to memory, but waiting on transitive dependencies. Ready = 0x3f /// Ready and safe for clients to access. }; @@ -502,8 +497,12 @@ class JITDylib { friend class ExecutionSession; friend class MaterializationResponsibility; public: - using GeneratorFunction = std::function<Expected<SymbolNameSet>( - JITDylib &Parent, const SymbolNameSet &Names)>; + class DefinitionGenerator { + public: + virtual ~DefinitionGenerator(); + virtual Expected<SymbolNameSet> + tryToGenerate(JITDylib &Parent, const SymbolNameSet &Names) = 0; + }; using AsynchronousSymbolQuerySet = std::set<std::shared_ptr<AsynchronousSymbolQuery>>; @@ -519,13 +518,20 @@ public: /// Get a reference to the ExecutionSession for this JITDylib. ExecutionSession &getExecutionSession() const { return ES; } - /// Set a definition generator. If set, whenever a symbol fails to resolve - /// within this JITDylib, lookup and lookupFlags will pass the unresolved - /// symbols set to the definition generator. The generator can optionally - /// add a definition for the unresolved symbols to the dylib. - void setGenerator(GeneratorFunction DefGenerator) { - this->DefGenerator = std::move(DefGenerator); - } + /// Adds a definition generator to this JITDylib and returns a referenece to + /// it. + /// + /// When JITDylibs are searched during lookup, if no existing definition of + /// a symbol is found, then any generators that have been added are run (in + /// the order that they were added) to potentially generate a definition. + template <typename GeneratorT> + GeneratorT &addGenerator(std::unique_ptr<GeneratorT> DefGenerator); + + /// Remove a definition generator from this JITDylib. + /// + /// The given generator must exist in this JITDylib's generators list (i.e. + /// have been added and not yet removed). + void removeGenerator(DefinitionGenerator &G); /// Set the search order to be used when fixing up definitions in JITDylib. /// This will replace the previous search order, and apply to any symbol @@ -633,17 +639,17 @@ private: struct MaterializingInfo { SymbolDependenceMap Dependants; SymbolDependenceMap UnemittedDependencies; - bool IsEmitted = false; void addQuery(std::shared_ptr<AsynchronousSymbolQuery> Q); void removeQuery(const AsynchronousSymbolQuery &Q); AsynchronousSymbolQueryList takeQueriesMeeting(SymbolState RequiredState); - AsynchronousSymbolQueryList takeAllQueries(); + AsynchronousSymbolQueryList takeAllPendingQueries() { + return std::move(PendingQueries); + } bool hasQueriesPending() const { return !PendingQueries.empty(); } const AsynchronousSymbolQueryList &pendingQueries() const { return PendingQueries; } - private: AsynchronousSymbolQueryList PendingQueries; }; @@ -710,9 +716,9 @@ private: SymbolNameSet &Unresolved, bool MatchNonExported, MaterializationUnitList &MUs); - void lodgeQueryImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs); + Error lodgeQueryImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, + SymbolNameSet &Unresolved, bool MatchNonExported, + MaterializationUnitList &MUs); bool lookupImpl(std::shared_ptr<AsynchronousSymbolQuery> &Q, std::vector<std::unique_ptr<MaterializationUnit>> &MUs, @@ -734,18 +740,20 @@ private: void addDependencies(const SymbolStringPtr &Name, const SymbolDependenceMap &Dependants); - void resolve(const SymbolMap &Resolved); + Error resolve(const SymbolMap &Resolved); - void emit(const SymbolFlagsMap &Emitted); + Error emit(const SymbolFlagsMap &Emitted); - void notifyFailed(const SymbolNameSet &FailedSymbols); + using FailedSymbolsWorklist = + std::vector<std::pair<JITDylib *, SymbolStringPtr>>; + static void notifyFailed(FailedSymbolsWorklist FailedSymbols); ExecutionSession &ES; std::string JITDylibName; SymbolTable Symbols; UnmaterializedInfosMap UnmaterializedInfos; MaterializingInfosMap MaterializingInfos; - GeneratorFunction DefGenerator; + std::vector<std::unique_ptr<DefinitionGenerator>> DefGenerators; JITDylibSearchList SearchOrder; }; @@ -933,6 +941,14 @@ private: OutstandingMUs; }; +template <typename GeneratorT> +GeneratorT &JITDylib::addGenerator(std::unique_ptr<GeneratorT> DefGenerator) { + auto &G = *DefGenerator; + ES.runSessionLocked( + [&]() { DefGenerators.push_back(std::move(DefGenerator)); }); + return G; +} + template <typename Func> auto JITDylib::withSearchOrderDo(Func &&F) -> decltype(F(std::declval<const JITDylibSearchList &>())) { @@ -972,6 +988,27 @@ Error JITDylib::define(std::unique_ptr<MaterializationUnitType> &MU) { }); } +/// ReexportsGenerator can be used with JITDylib::setGenerator to automatically +/// re-export a subset of the source JITDylib's symbols in the target. +class ReexportsGenerator : public JITDylib::DefinitionGenerator { +public: + using SymbolPredicate = std::function<bool(SymbolStringPtr)>; + + /// Create a reexports generator. If an Allow predicate is passed, only + /// symbols for which the predicate returns true will be reexported. If no + /// Allow predicate is passed, all symbols will be exported. + ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false, + SymbolPredicate Allow = SymbolPredicate()); + + Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) override; + +private: + JITDylib &SourceJD; + bool MatchNonExported = false; + SymbolPredicate Allow; +}; + /// Mangles symbol names then uniques them in the context of an /// ExecutionSession. class MangleAndInterner { diff --git a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index 75865920c741..cf0a428662ef 100644 --- a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -19,6 +19,7 @@ #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" #include "llvm/ExecutionEngine/RuntimeDyld.h" +#include "llvm/Object/Archive.h" #include "llvm/Support/DynamicLibrary.h" #include <algorithm> #include <cstdint> @@ -37,6 +38,8 @@ class Value; namespace orc { +class ObjectLayer; + /// This iterator provides a convenient way to iterate over the elements /// of an llvm.global_ctors/llvm.global_dtors instance. /// @@ -237,7 +240,7 @@ public: /// If an instance of this class is attached to a JITDylib as a fallback /// definition generator, then any symbol found in the given DynamicLibrary that /// passes the 'Allow' predicate will be added to the JITDylib. -class DynamicLibrarySearchGenerator { +class DynamicLibrarySearchGenerator : public JITDylib::DefinitionGenerator { public: using SymbolPredicate = std::function<bool(SymbolStringPtr)>; @@ -253,19 +256,20 @@ public: /// Permanently loads the library at the given path and, on success, returns /// a DynamicLibrarySearchGenerator that will search it for symbol definitions /// in the library. On failure returns the reason the library failed to load. - static Expected<DynamicLibrarySearchGenerator> + static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()); /// Creates a DynamicLibrarySearchGenerator that searches for symbols in /// the current process. - static Expected<DynamicLibrarySearchGenerator> + static Expected<std::unique_ptr<DynamicLibrarySearchGenerator>> GetForCurrentProcess(char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()) { return Load(nullptr, GlobalPrefix, std::move(Allow)); } - Expected<SymbolNameSet> operator()(JITDylib &JD, const SymbolNameSet &Names); + Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) override; private: sys::DynamicLibrary Dylib; @@ -273,6 +277,40 @@ private: char GlobalPrefix; }; +/// A utility class to expose symbols from a static library. +/// +/// If an instance of this class is attached to a JITDylib as a fallback +/// definition generator, then any symbol found in the archive will result in +/// the containing object being added to the JITDylib. +class StaticLibraryDefinitionGenerator : public JITDylib::DefinitionGenerator { +public: + /// Try to create a StaticLibraryDefinitionGenerator from the given path. + /// + /// This call will succeed if the file at the given path is a static library + /// is a valid archive, otherwise it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Load(ObjectLayer &L, const char *FileName); + + /// Try to create a StaticLibrarySearchGenerator from the given memory buffer. + /// Thhis call will succeed if the buffer contains a valid archive, otherwise + /// it will return an error. + static Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>> + Create(ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer); + + Expected<SymbolNameSet> tryToGenerate(JITDylib &JD, + const SymbolNameSet &Names) override; + +private: + StaticLibraryDefinitionGenerator(ObjectLayer &L, + std::unique_ptr<MemoryBuffer> ArchiveBuffer, + Error &Err); + + ObjectLayer &L; + std::unique_ptr<MemoryBuffer> ArchiveBuffer; + object::Archive Archive; + size_t UnrealizedObjects = 0; +}; + } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h index 1b4c8b6cd95f..b71e5b339711 100644 --- a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -22,6 +22,9 @@ namespace llvm { class Module; namespace orc { +/// A layer that applies a transform to emitted modules. +/// The transform function is responsible for locking the ThreadSafeContext +/// before operating on the module. class IRTransformLayer : public IRLayer { public: using TransformFunction = std::function<Expected<ThreadSafeModule>( diff --git a/include/llvm/ExecutionEngine/Orc/LLJIT.h b/include/llvm/ExecutionEngine/Orc/LLJIT.h index 0aac1916423f..b1e47d77557c 100644 --- a/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -184,8 +184,8 @@ private: class LLJITBuilderState { public: - using ObjectLinkingLayerCreator = - std::function<std::unique_ptr<ObjectLayer>(ExecutionSession &)>; + using ObjectLinkingLayerCreator = std::function<std::unique_ptr<ObjectLayer>( + ExecutionSession &, const Triple &TT)>; using CompileFunctionCreator = std::function<Expected<IRCompileLayer::CompileFunction>( diff --git a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h index 855e31b33549..b31914f12a0d 100644 --- a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h +++ b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h @@ -16,6 +16,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/OrcV1Deprecation.h" #include <memory> namespace llvm { @@ -62,7 +63,7 @@ std::shared_ptr<LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>> createLambdaResolver(DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) { using LR = LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>; - return make_unique<LR>(std::move(DylibLookupFtor), + return std::make_unique<LR>(std::move(DylibLookupFtor), std::move(ExternalLookupFtor)); } @@ -72,7 +73,7 @@ createLambdaResolver(ORCv1DeprecationAcknowledgement, DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) { using LR = LambdaResolver<DylibLookupFtorT, ExternalLookupFtorT>; - return make_unique<LR>(AcknowledgeORCv1Deprecation, + return std::make_unique<LR>(AcknowledgeORCv1Deprecation, std::move(DylibLookupFtor), std::move(ExternalLookupFtor)); } diff --git a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h index 16202d89f861..b67a9feed523 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h @@ -49,28 +49,24 @@ private: switch (EmitState) { case NotEmitted: if (auto GV = searchGVs(Name, ExportedSymbolsOnly)) { - // Create a std::string version of Name to capture here - the argument - // (a StringRef) may go away before the lambda is executed. - // FIXME: Use capture-init when we move to C++14. - std::string PName = Name; JITSymbolFlags Flags = JITSymbolFlags::fromGlobalValue(*GV); - auto GetAddress = - [this, ExportedSymbolsOnly, PName, &B]() -> Expected<JITTargetAddress> { - if (this->EmitState == Emitting) - return 0; - else if (this->EmitState == NotEmitted) { - this->EmitState = Emitting; - if (auto Err = this->emitToBaseLayer(B)) - return std::move(Err); - this->EmitState = Emitted; - } - if (auto Sym = B.findSymbolIn(K, PName, ExportedSymbolsOnly)) - return Sym.getAddress(); - else if (auto Err = Sym.takeError()) + auto GetAddress = [this, ExportedSymbolsOnly, Name = Name.str(), + &B]() -> Expected<JITTargetAddress> { + if (this->EmitState == Emitting) + return 0; + else if (this->EmitState == NotEmitted) { + this->EmitState = Emitting; + if (auto Err = this->emitToBaseLayer(B)) return std::move(Err); - else - llvm_unreachable("Successful symbol lookup should return " - "definition address here"); + this->EmitState = Emitted; + } + if (auto Sym = B.findSymbolIn(K, Name, ExportedSymbolsOnly)) + return Sym.getAddress(); + else if (auto Err = Sym.takeError()) + return std::move(Err); + else + llvm_unreachable("Successful symbol lookup should return " + "definition address here"); }; return JITSymbol(std::move(GetAddress), Flags); } else @@ -171,7 +167,7 @@ private: bool ExportedSymbolsOnly) const { assert(!MangledSymbols && "Mangled symbols map already exists?"); - auto Symbols = llvm::make_unique<StringMap<const GlobalValue*>>(); + auto Symbols = std::make_unique<StringMap<const GlobalValue*>>(); Mangler Mang; @@ -209,7 +205,7 @@ public: Error addModule(VModuleKey K, std::unique_ptr<Module> M) { assert(!ModuleMap.count(K) && "VModuleKey K already in use"); ModuleMap[K] = - llvm::make_unique<EmissionDeferredModule>(std::move(K), std::move(M)); + std::make_unique<EmissionDeferredModule>(std::move(K), std::move(M)); return Error::success(); } diff --git a/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/include/llvm/ExecutionEngine/Orc/LazyReexports.h index 9fdd1d15f782..311ed59b1549 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyReexports.h +++ b/include/llvm/ExecutionEngine/Orc/LazyReexports.h @@ -18,6 +18,7 @@ #include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" namespace llvm { @@ -70,7 +71,7 @@ public: template <typename NotifyResolvedImpl> static std::unique_ptr<NotifyResolvedFunction> createNotifyResolvedFunction(NotifyResolvedImpl NotifyResolved) { - return llvm::make_unique<NotifyResolvedFunctionImpl<NotifyResolvedImpl>>( + return std::make_unique<NotifyResolvedFunctionImpl<NotifyResolvedImpl>>( std::move(NotifyResolved)); } @@ -159,7 +160,7 @@ public: IndirectStubsManager &ISManager, JITDylib &SourceJD, SymbolAliasMap CallableAliases, - VModuleKey K); + ImplSymbolMap *SrcJDLoc, VModuleKey K); StringRef getName() const override; @@ -174,6 +175,7 @@ private: SymbolAliasMap CallableAliases; std::shared_ptr<LazyCallThroughManager::NotifyResolvedFunction> NotifyResolved; + ImplSymbolMap *AliaseeTable; }; /// Define lazy-reexports based on the given SymbolAliasMap. Each lazy re-export @@ -182,9 +184,10 @@ private: inline std::unique_ptr<LazyReexportsMaterializationUnit> lazyReexports(LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager, JITDylib &SourceJD, - SymbolAliasMap CallableAliases, VModuleKey K = VModuleKey()) { - return llvm::make_unique<LazyReexportsMaterializationUnit>( - LCTManager, ISManager, SourceJD, std::move(CallableAliases), + SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc = nullptr, + VModuleKey K = VModuleKey()) { + return std::make_unique<LazyReexportsMaterializationUnit>( + LCTManager, ISManager, SourceJD, std::move(CallableAliases), SrcJDLoc, std::move(K)); } diff --git a/include/llvm/ExecutionEngine/Orc/Legacy.h b/include/llvm/ExecutionEngine/Orc/Legacy.h index f9cbbf6ff180..148e260c9569 100644 --- a/include/llvm/ExecutionEngine/Orc/Legacy.h +++ b/include/llvm/ExecutionEngine/Orc/Legacy.h @@ -84,7 +84,7 @@ createSymbolResolver(GetResponsibilitySetFn &&GetResponsibilitySet, typename std::remove_reference<GetResponsibilitySetFn>::type>::type, typename std::remove_cv< typename std::remove_reference<LookupFn>::type>::type>; - return llvm::make_unique<LambdaSymbolResolverImpl>( + return std::make_unique<LambdaSymbolResolverImpl>( std::forward<GetResponsibilitySetFn>(GetResponsibilitySet), std::forward<LookupFn>(Lookup)); } diff --git a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h index c1e7d27f446e..caf8e707516d 100644 --- a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -73,6 +73,9 @@ public: virtual Error notifyRemovingAllModules() { return Error::success(); } }; + using ReturnObjectBufferFunction = + std::function<void(std::unique_ptr<MemoryBuffer>)>; + /// Construct an ObjectLinkingLayer with the given NotifyLoaded, /// and NotifyEmitted functors. ObjectLinkingLayer(ExecutionSession &ES, @@ -81,6 +84,13 @@ public: /// Destruct an ObjectLinkingLayer. ~ObjectLinkingLayer(); + /// Set an object buffer return function. By default object buffers are + /// deleted once the JIT has linked them. If a return function is set then + /// it will be called to transfer ownership of the buffer instead. + void setReturnObjectBuffer(ReturnObjectBufferFunction ReturnObjectBuffer) { + this->ReturnObjectBuffer = std::move(ReturnObjectBuffer); + } + /// Add a pass-config modifier. ObjectLinkingLayer &addPlugin(std::unique_ptr<Plugin> P) { std::lock_guard<std::mutex> Lock(LayerMutex); @@ -138,6 +148,7 @@ private: jitlink::JITLinkMemoryManager &MemMgr; bool OverrideObjectFlags = false; bool AutoClaimObjectSymbols = false; + ReturnObjectBufferFunction ReturnObjectBuffer; DenseMap<VModuleKey, AllocPtr> TrackedAllocs; std::vector<AllocPtr> UntrackedAllocs; std::vector<std::unique_ptr<Plugin>> Plugins; @@ -153,10 +164,16 @@ public: Error notifyRemovingAllModules() override; private: + + struct EHFrameRange { + JITTargetAddress Addr = 0; + size_t Size; + }; + jitlink::EHFrameRegistrar &Registrar; - DenseMap<MaterializationResponsibility *, JITTargetAddress> InProcessLinks; - DenseMap<VModuleKey, JITTargetAddress> TrackedEHFrameAddrs; - std::vector<JITTargetAddress> UntrackedEHFrameAddrs; + DenseMap<MaterializationResponsibility *, EHFrameRange> InProcessLinks; + DenseMap<VModuleKey, EHFrameRange> TrackedEHFrameRanges; + std::vector<EHFrameRange> UntrackedEHFrameRanges; }; } // end namespace orc diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h index 8b875b7906e1..86e8d5df3ad9 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h @@ -493,7 +493,7 @@ public: ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) : JITCompileCallbackManager( - llvm::make_unique<RemoteTrampolinePool>(Client), ES, + std::make_unique<RemoteTrampolinePool>(Client), ES, ErrorHandlerAddress) {} }; @@ -553,7 +553,7 @@ public: auto Id = IndirectStubOwnerIds.getNext(); if (auto Err = callB<stubs::CreateIndirectStubsOwner>(Id)) return std::move(Err); - return llvm::make_unique<RemoteIndirectStubsManager>(*this, Id); + return std::make_unique<RemoteIndirectStubsManager>(*this, Id); } Expected<RemoteCompileCallbackManager &> diff --git a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h index 07c7471afc6a..752a0a34e0a1 100644 --- a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h +++ b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h @@ -359,9 +359,9 @@ public: { assert(KeyName != nullptr && "No keyname pointer"); std::lock_guard<std::recursive_mutex> Lock(SerializersMutex); - // FIXME: Move capture Serialize once we have C++14. Serializers[ErrorInfoT::classID()] = - [KeyName, Serialize](ChannelT &C, const ErrorInfoBase &EIB) -> Error { + [KeyName, Serialize = std::move(Serialize)]( + ChannelT &C, const ErrorInfoBase &EIB) -> Error { assert(EIB.dynamicClassID() == ErrorInfoT::classID() && "Serializer called for wrong error type"); if (auto Err = serializeSeq(C, *KeyName)) @@ -551,26 +551,26 @@ public: /// RPC channel serialization for std::tuple. static Error serialize(ChannelT &C, const std::tuple<ArgTs...> &V) { - return serializeTupleHelper(C, V, llvm::index_sequence_for<ArgTs...>()); + return serializeTupleHelper(C, V, std::index_sequence_for<ArgTs...>()); } /// RPC channel deserialization for std::tuple. static Error deserialize(ChannelT &C, std::tuple<ArgTs...> &V) { - return deserializeTupleHelper(C, V, llvm::index_sequence_for<ArgTs...>()); + return deserializeTupleHelper(C, V, std::index_sequence_for<ArgTs...>()); } private: // Serialization helper for std::tuple. template <size_t... Is> static Error serializeTupleHelper(ChannelT &C, const std::tuple<ArgTs...> &V, - llvm::index_sequence<Is...> _) { + std::index_sequence<Is...> _) { return serializeSeq(C, std::get<Is>(V)...); } // Serialization helper for std::tuple. template <size_t... Is> static Error deserializeTupleHelper(ChannelT &C, std::tuple<ArgTs...> &V, - llvm::index_sequence<Is...> _) { + std::index_sequence<Is...> _) { return deserializeSeq(C, std::get<Is>(V)...); } }; diff --git a/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/include/llvm/ExecutionEngine/Orc/RPCUtils.h index 3b11e1b283de..ee9c2cc69c30 100644 --- a/include/llvm/ExecutionEngine/Orc/RPCUtils.h +++ b/include/llvm/ExecutionEngine/Orc/RPCUtils.h @@ -338,7 +338,9 @@ public: return Err; // Close the response message. - return C.endSendMessage(); + if (auto Err = C.endSendMessage()) + return Err; + return C.send(); } template <typename ChannelT, typename FunctionIdT, typename SequenceNumberT> @@ -350,7 +352,9 @@ public: return Err2; if (auto Err2 = serializeSeq(C, std::move(Err))) return Err2; - return C.endSendMessage(); + if (auto Err2 = C.endSendMessage()) + return Err2; + return C.send(); } }; @@ -378,8 +382,11 @@ public: C, *ResultOrErr)) return Err; - // Close the response message. - return C.endSendMessage(); + // End the response message. + if (auto Err = C.endSendMessage()) + return Err; + + return C.send(); } template <typename ChannelT, typename FunctionIdT, typename SequenceNumberT> @@ -389,7 +396,9 @@ public: return Err; if (auto Err2 = C.startSendMessage(ResponseId, SeqNo)) return Err2; - return C.endSendMessage(); + if (auto Err2 = C.endSendMessage()) + return Err2; + return C.send(); } }; @@ -502,7 +511,7 @@ public: static typename WrappedHandlerReturn<RetT>::Type unpackAndRun(HandlerT &Handler, std::tuple<TArgTs...> &Args) { return unpackAndRunHelper(Handler, Args, - llvm::index_sequence_for<TArgTs...>()); + std::index_sequence_for<TArgTs...>()); } // Call the given handler with the given arguments. @@ -510,7 +519,7 @@ public: static Error unpackAndRunAsync(HandlerT &Handler, ResponderT &Responder, std::tuple<TArgTs...> &Args) { return unpackAndRunAsyncHelper(Handler, Responder, Args, - llvm::index_sequence_for<TArgTs...>()); + std::index_sequence_for<TArgTs...>()); } // Call the given handler with the given arguments. @@ -540,14 +549,13 @@ public: // Deserialize arguments from the channel. template <typename ChannelT, typename... CArgTs> static Error deserializeArgs(ChannelT &C, std::tuple<CArgTs...> &Args) { - return deserializeArgsHelper(C, Args, - llvm::index_sequence_for<CArgTs...>()); + return deserializeArgsHelper(C, Args, std::index_sequence_for<CArgTs...>()); } private: template <typename ChannelT, typename... CArgTs, size_t... Indexes> static Error deserializeArgsHelper(ChannelT &C, std::tuple<CArgTs...> &Args, - llvm::index_sequence<Indexes...> _) { + std::index_sequence<Indexes...> _) { return SequenceSerialization<ChannelT, ArgTs...>::deserialize( C, std::get<Indexes>(Args)...); } @@ -556,18 +564,16 @@ private: static typename WrappedHandlerReturn< typename HandlerTraits<HandlerT>::ReturnType>::Type unpackAndRunHelper(HandlerT &Handler, ArgTuple &Args, - llvm::index_sequence<Indexes...>) { + std::index_sequence<Indexes...>) { return run(Handler, std::move(std::get<Indexes>(Args))...); } - template <typename HandlerT, typename ResponderT, typename ArgTuple, size_t... Indexes> static typename WrappedHandlerReturn< typename HandlerTraits<HandlerT>::ReturnType>::Type unpackAndRunAsyncHelper(HandlerT &Handler, ResponderT &Responder, - ArgTuple &Args, - llvm::index_sequence<Indexes...>) { + ArgTuple &Args, std::index_sequence<Indexes...>) { return run(Handler, Responder, std::move(std::get<Indexes>(Args))...); } }; @@ -743,11 +749,15 @@ public: // to the user defined handler. Error handleResponse(ChannelT &C) override { Error Result = Error::success(); - if (auto Err = - SerializationTraits<ChannelT, Error, Error>::deserialize(C, Result)) + if (auto Err = SerializationTraits<ChannelT, Error, Error>::deserialize( + C, Result)) { + consumeError(std::move(Result)); return Err; - if (auto Err = C.endReceiveMessage()) + } + if (auto Err = C.endReceiveMessage()) { + consumeError(std::move(Result)); return Err; + } return Handler(std::move(Result)); } @@ -767,7 +777,7 @@ private: // Create a ResponseHandler from a given user handler. template <typename ChannelT, typename FuncRetT, typename HandlerT> std::unique_ptr<ResponseHandler<ChannelT>> createResponseHandler(HandlerT H) { - return llvm::make_unique<ResponseHandlerImpl<ChannelT, FuncRetT, HandlerT>>( + return std::make_unique<ResponseHandlerImpl<ChannelT, FuncRetT, HandlerT>>( std::move(H)); } @@ -1403,14 +1413,12 @@ public: using ErrorReturn = typename RTraits::ErrorReturnType; using ErrorReturnPromise = typename RTraits::ReturnPromiseType; - // FIXME: Stack allocate and move this into the handler once LLVM builds - // with C++14. - auto Promise = std::make_shared<ErrorReturnPromise>(); - auto FutureResult = Promise->get_future(); + ErrorReturnPromise Promise; + auto FutureResult = Promise.get_future(); if (auto Err = this->template appendCallAsync<Func>( - [Promise](ErrorReturn RetOrErr) { - Promise->set_value(std::move(RetOrErr)); + [Promise = std::move(Promise)](ErrorReturn RetOrErr) mutable { + Promise.set_value(std::move(RetOrErr)); return Error::success(); }, Args...)) { @@ -1523,6 +1531,12 @@ public: return std::move(Err); } + if (auto Err = this->C.send()) { + detail::ResultTraits<typename Func::ReturnType>::consumeAbandoned( + std::move(Result)); + return std::move(Err); + } + while (!ReceivedResponse) { if (auto Err = this->handleOne()) { detail::ResultTraits<typename Func::ReturnType>::consumeAbandoned( @@ -1582,8 +1596,7 @@ public: // outstanding calls count, then poke the condition variable. using ArgType = typename detail::ResponseHandlerArg< typename detail::HandlerTraits<HandlerT>::Type>::ArgType; - // FIXME: Move handler into wrapped handler once we have C++14. - auto WrappedHandler = [this, Handler](ArgType Arg) { + auto WrappedHandler = [this, Handler = std::move(Handler)](ArgType Arg) { auto Err = Handler(std::move(Arg)); std::unique_lock<std::mutex> Lock(M); --NumOutstandingCalls; diff --git a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index d9535ce5f21f..c5106cf09ecc 100644 --- a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -216,7 +216,7 @@ private: : K(std::move(K)), Parent(Parent), MemMgr(std::move(MemMgr)), - PFC(llvm::make_unique<PreFinalizeContents>( + PFC(std::make_unique<PreFinalizeContents>( std::move(Obj), std::move(Resolver), ProcessAllSections)) { buildInitialSymbolTable(PFC->Obj); @@ -234,7 +234,7 @@ private: JITSymbolResolverAdapter ResolverAdapter(Parent.ES, *PFC->Resolver, nullptr); - PFC->RTDyld = llvm::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter); + PFC->RTDyld = std::make_unique<RuntimeDyld>(*MemMgr, ResolverAdapter); PFC->RTDyld->setProcessAllSections(PFC->ProcessAllSections); Finalized = true; @@ -338,7 +338,7 @@ private: std::shared_ptr<SymbolResolver> Resolver, bool ProcessAllSections) { using LOS = ConcreteLinkedObject<MemoryManagerPtrT>; - return llvm::make_unique<LOS>(Parent, std::move(K), std::move(Obj), + return std::make_unique<LOS>(Parent, std::move(K), std::move(Obj), std::move(MemMgr), std::move(Resolver), ProcessAllSections); } diff --git a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h index b87cf697a81e..d7304cfcf931 100644 --- a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h @@ -137,17 +137,12 @@ protected: RemoteSymbolId Id) : C(C), Id(Id) {} - RemoteSymbolMaterializer(const RemoteSymbolMaterializer &Other) - : C(Other.C), Id(Other.Id) { - // FIXME: This is a horrible, auto_ptr-style, copy-as-move operation. - // It should be removed as soon as LLVM has C++14's generalized - // lambda capture (at which point the materializer can be moved - // into the lambda in remoteToJITSymbol below). - const_cast<RemoteSymbolMaterializer&>(Other).Id = 0; + RemoteSymbolMaterializer(RemoteSymbolMaterializer &&Other) + : C(Other.C), Id(Other.Id) { + Other.Id = 0; } - RemoteSymbolMaterializer& - operator=(const RemoteSymbolMaterializer&) = delete; + RemoteSymbolMaterializer &operator=(RemoteSymbolMaterializer &&) = delete; /// Release the remote symbol. ~RemoteSymbolMaterializer() { @@ -218,9 +213,9 @@ protected: return nullptr; // else... RemoteSymbolMaterializer RSM(*this, RemoteSym.first); - auto Sym = - JITSymbol([RSM]() mutable { return RSM.materialize(); }, - RemoteSym.second); + auto Sym = JITSymbol( + [RSM = std::move(RSM)]() mutable { return RSM.materialize(); }, + RemoteSym.second); return Sym; } else return RemoteSymOrErr.takeError(); @@ -472,7 +467,7 @@ private: } Expected<ObjHandleT> addObject(std::string ObjBuffer) { - auto Buffer = llvm::make_unique<StringMemoryBuffer>(std::move(ObjBuffer)); + auto Buffer = std::make_unique<StringMemoryBuffer>(std::move(ObjBuffer)); auto Id = HandleIdMgr.getNext(); assert(!BaseLayerHandles.count(Id) && "Id already in use?"); diff --git a/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h b/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h new file mode 100644 index 000000000000..cf57b63b6448 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/SpeculateAnalyses.h @@ -0,0 +1,84 @@ +//===-- SpeculateAnalyses.h --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// \file +/// Contains the Analyses and Result Interpretation to select likely functions +/// to Speculatively compile before they are called. [Purely Experimentation] +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H +#define LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H + +#include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Speculation.h" + +#include <vector> + +namespace llvm { + +namespace orc { + +// Provides common code. +class SpeculateQuery { +protected: + void findCalles(const BasicBlock *, DenseSet<StringRef> &); + bool isStraightLine(const Function &F); + +public: + using ResultTy = Optional<DenseMap<StringRef, DenseSet<StringRef>>>; +}; + +// Direct calls in high frequency basic blocks are extracted. +class BlockFreqQuery : public SpeculateQuery { + size_t numBBToGet(size_t); + +public: + // Find likely next executables based on IR Block Frequency + ResultTy operator()(Function &F); +}; + +// This Query generates a sequence of basic blocks which follows the order of +// execution. +// A handful of BB with higher block frequencies are taken, then path to entry +// and end BB are discovered by traversing up & down the CFG. +class SequenceBBQuery : public SpeculateQuery { + struct WalkDirection { + bool Upward = true, Downward = true; + // the block associated contain a call + bool CallerBlock = false; + }; + +public: + using VisitedBlocksInfoTy = DenseMap<const BasicBlock *, WalkDirection>; + using BlockListTy = SmallVector<const BasicBlock *, 8>; + using BackEdgesInfoTy = + SmallVector<std::pair<const BasicBlock *, const BasicBlock *>, 8>; + using BlockFreqInfoTy = + SmallVector<std::pair<const BasicBlock *, uint64_t>, 8>; + +private: + std::size_t getHottestBlocks(std::size_t TotalBlocks); + BlockListTy rearrangeBB(const Function &, const BlockListTy &); + BlockListTy queryCFG(Function &, const BlockListTy &); + void traverseToEntryBlock(const BasicBlock *, const BlockListTy &, + const BackEdgesInfoTy &, + const BranchProbabilityInfo *, + VisitedBlocksInfoTy &); + void traverseToExitBlock(const BasicBlock *, const BlockListTy &, + const BackEdgesInfoTy &, + const BranchProbabilityInfo *, + VisitedBlocksInfoTy &); + +public: + ResultTy operator()(Function &F); +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SPECULATEANALYSES_H diff --git a/include/llvm/ExecutionEngine/Orc/Speculation.h b/include/llvm/ExecutionEngine/Orc/Speculation.h new file mode 100644 index 000000000000..766a6b070f12 --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/Speculation.h @@ -0,0 +1,207 @@ +//===-- Speculation.h - Speculative Compilation --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the definition to support speculative compilation when laziness is +// enabled. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_SPECULATION_H +#define LLVM_EXECUTIONENGINE_ORC_SPECULATION_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/IRCompileLayer.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Support/Debug.h" + +#include <mutex> +#include <type_traits> +#include <utility> +#include <vector> + +namespace llvm { +namespace orc { + +class Speculator; + +// Track the Impls (JITDylib,Symbols) of Symbols while lazy call through +// trampolines are created. Operations are guarded by locks tp ensure that Imap +// stays in consistent state after read/write + +class ImplSymbolMap { + friend class Speculator; + +public: + using AliaseeDetails = std::pair<SymbolStringPtr, JITDylib *>; + using Alias = SymbolStringPtr; + using ImapTy = DenseMap<Alias, AliaseeDetails>; + void trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD); + +private: + // FIX ME: find a right way to distinguish the pre-compile Symbols, and update + // the callsite + Optional<AliaseeDetails> getImplFor(const SymbolStringPtr &StubSymbol) { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + auto Position = Maps.find(StubSymbol); + if (Position != Maps.end()) + return Position->getSecond(); + else + return None; + } + + std::mutex ConcurrentAccess; + ImapTy Maps; +}; + +// Defines Speculator Concept, +class Speculator { +public: + using TargetFAddr = JITTargetAddress; + using FunctionCandidatesMap = DenseMap<SymbolStringPtr, SymbolNameSet>; + using StubAddrLikelies = DenseMap<TargetFAddr, SymbolNameSet>; + +private: + void registerSymbolsWithAddr(TargetFAddr ImplAddr, + SymbolNameSet likelySymbols) { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + GlobalSpecMap.insert({ImplAddr, std::move(likelySymbols)}); + } + + void launchCompile(JITTargetAddress FAddr) { + SymbolNameSet CandidateSet; + // Copy CandidateSet is necessary, to avoid unsynchronized access to + // the datastructure. + { + std::lock_guard<std::mutex> Lockit(ConcurrentAccess); + auto It = GlobalSpecMap.find(FAddr); + if (It == GlobalSpecMap.end()) + return; + CandidateSet = It->getSecond(); + } + + SymbolDependenceMap SpeculativeLookUpImpls; + + for (auto &Callee : CandidateSet) { + auto ImplSymbol = AliaseeImplTable.getImplFor(Callee); + // try to distinguish already compiled & library symbols + if (!ImplSymbol.hasValue()) + continue; + const auto &ImplSymbolName = ImplSymbol.getPointer()->first; + JITDylib *ImplJD = ImplSymbol.getPointer()->second; + auto &SymbolsInJD = SpeculativeLookUpImpls[ImplJD]; + SymbolsInJD.insert(ImplSymbolName); + } + + DEBUG_WITH_TYPE("orc", for (auto &I + : SpeculativeLookUpImpls) { + llvm::dbgs() << "\n In " << I.first->getName() << " JITDylib "; + for (auto &N : I.second) + llvm::dbgs() << "\n Likely Symbol : " << N; + }); + + // for a given symbol, there may be no symbol qualified for speculatively + // compile try to fix this before jumping to this code if possible. + for (auto &LookupPair : SpeculativeLookUpImpls) + ES.lookup(JITDylibSearchList({{LookupPair.first, true}}), + LookupPair.second, SymbolState::Ready, + [this](Expected<SymbolMap> Result) { + if (auto Err = Result.takeError()) + ES.reportError(std::move(Err)); + }, + NoDependenciesToRegister); + } + +public: + Speculator(ImplSymbolMap &Impl, ExecutionSession &ref) + : AliaseeImplTable(Impl), ES(ref), GlobalSpecMap(0) {} + Speculator(const Speculator &) = delete; + Speculator(Speculator &&) = delete; + Speculator &operator=(const Speculator &) = delete; + Speculator &operator=(Speculator &&) = delete; + + /// Define symbols for this Speculator object (__orc_speculator) and the + /// speculation runtime entry point symbol (__orc_speculate_for) in the + /// given JITDylib. + Error addSpeculationRuntime(JITDylib &JD, MangleAndInterner &Mangle); + + // Speculatively compile likely functions for the given Stub Address. + // destination of __orc_speculate_for jump + void speculateFor(TargetFAddr StubAddr) { launchCompile(StubAddr); } + + // FIXME : Register with Stub Address, after JITLink Fix. + void registerSymbols(FunctionCandidatesMap Candidates, JITDylib *JD) { + for (auto &SymPair : Candidates) { + auto Target = SymPair.first; + auto Likely = SymPair.second; + + auto OnReadyFixUp = [Likely, Target, + this](Expected<SymbolMap> ReadySymbol) { + if (ReadySymbol) { + auto RAddr = (*ReadySymbol)[Target].getAddress(); + registerSymbolsWithAddr(RAddr, std::move(Likely)); + } else + this->getES().reportError(ReadySymbol.takeError()); + }; + // Include non-exported symbols also. + ES.lookup(JITDylibSearchList({{JD, true}}), SymbolNameSet({Target}), + SymbolState::Ready, OnReadyFixUp, NoDependenciesToRegister); + } + } + + ExecutionSession &getES() { return ES; } + +private: + static void speculateForEntryPoint(Speculator *Ptr, uint64_t StubId); + std::mutex ConcurrentAccess; + ImplSymbolMap &AliaseeImplTable; + ExecutionSession &ES; + StubAddrLikelies GlobalSpecMap; +}; + +class IRSpeculationLayer : public IRLayer { +public: + using IRlikiesStrRef = Optional<DenseMap<StringRef, DenseSet<StringRef>>>; + using ResultEval = std::function<IRlikiesStrRef(Function &)>; + using TargetAndLikelies = DenseMap<SymbolStringPtr, SymbolNameSet>; + + IRSpeculationLayer(ExecutionSession &ES, IRCompileLayer &BaseLayer, + Speculator &Spec, MangleAndInterner &Mangle, + ResultEval Interpreter) + : IRLayer(ES), NextLayer(BaseLayer), S(Spec), Mangle(Mangle), + QueryAnalysis(Interpreter) {} + + void emit(MaterializationResponsibility R, ThreadSafeModule TSM); + +private: + TargetAndLikelies + internToJITSymbols(DenseMap<StringRef, DenseSet<StringRef>> IRNames) { + assert(!IRNames.empty() && "No IRNames received to Intern?"); + TargetAndLikelies InternedNames; + DenseSet<SymbolStringPtr> TargetJITNames; + for (auto &NamePair : IRNames) { + for (auto &TargetNames : NamePair.second) + TargetJITNames.insert(Mangle(TargetNames)); + + InternedNames[Mangle(NamePair.first)] = std::move(TargetJITNames); + } + return InternedNames; + } + + IRCompileLayer &NextLayer; + Speculator &S; + MangleAndInterner &Mangle; + ResultEval QueryAnalysis; +}; + +} // namespace orc +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_SPECULATION_H diff --git a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h index 5787500387c4..2347faed37a2 100644 --- a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h +++ b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h @@ -38,17 +38,12 @@ private: public: // RAII based lock for ThreadSafeContext. class LLVM_NODISCARD Lock { - private: - using UnderlyingLock = std::lock_guard<std::recursive_mutex>; - public: - Lock(std::shared_ptr<State> S) - : S(std::move(S)), - L(llvm::make_unique<UnderlyingLock>(this->S->Mutex)) {} + Lock(std::shared_ptr<State> S) : S(std::move(S)), L(this->S->Mutex) {} private: std::shared_ptr<State> S; - std::unique_ptr<UnderlyingLock> L; + std::unique_lock<std::recursive_mutex> L; }; /// Construct a null context. @@ -69,7 +64,7 @@ public: /// instance, or null if the instance was default constructed. const LLVMContext *getContext() const { return S ? S->Ctx.get() : nullptr; } - Lock getLock() { + Lock getLock() const { assert(S && "Can not lock an empty ThreadSafeContext"); return Lock(S); } @@ -95,7 +90,7 @@ public: // We also need to lock the context to make sure the module tear-down // does not overlap any other work on the context. if (M) { - auto L = getContextLock(); + auto L = TSCtx.getLock(); M = nullptr; } M = std::move(Other.M); @@ -117,23 +112,14 @@ public: ~ThreadSafeModule() { // We need to lock the context while we destruct the module. if (M) { - auto L = getContextLock(); + auto L = TSCtx.getLock(); M = nullptr; } } - /// Get the module wrapped by this ThreadSafeModule. - Module *getModule() { return M.get(); } - - /// Get the module wrapped by this ThreadSafeModule. - const Module *getModule() const { return M.get(); } - - /// Take out a lock on the ThreadSafeContext for this module. - ThreadSafeContext::Lock getContextLock() { return TSCtx.getLock(); } - /// Boolean conversion: This ThreadSafeModule will evaluate to true if it /// wraps a non-null module. - explicit operator bool() { + explicit operator bool() const { if (M) { assert(TSCtx.getContext() && "Non-null module must have non-null context"); @@ -142,6 +128,33 @@ public: return false; } + /// Locks the associated ThreadSafeContext and calls the given function + /// on the contained Module. + template <typename Func> + auto withModuleDo(Func &&F) -> decltype(F(std::declval<Module &>())) { + assert(M && "Can not call on null module"); + auto Lock = TSCtx.getLock(); + return F(*M); + } + + /// Locks the associated ThreadSafeContext and calls the given function + /// on the contained Module. + template <typename Func> + auto withModuleDo(Func &&F) const + -> decltype(F(std::declval<const Module &>())) { + auto Lock = TSCtx.getLock(); + return F(*M); + } + + /// Get a raw pointer to the contained module without locking the context. + Module *getModuleUnlocked() { return M.get(); } + + /// Get a raw pointer to the contained module without locking the context. + const Module *getModuleUnlocked() const { return M.get(); } + + /// Returns the context for this ThreadSafeModule. + ThreadSafeContext getContext() const { return TSCtx; } + private: std::unique_ptr<Module> M; ThreadSafeContext TSCtx; |