aboutsummaryrefslogtreecommitdiff
path: root/lib/ExecutionEngine/Orc
diff options
context:
space:
mode:
Diffstat (limited to 'lib/ExecutionEngine/Orc')
-rw-r--r--lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp127
-rw-r--r--lib/ExecutionEngine/Orc/CompileUtils.cpp2
-rw-r--r--lib/ExecutionEngine/Orc/Core.cpp506
-rw-r--r--lib/ExecutionEngine/Orc/ExecutionUtils.cpp92
-rw-r--r--lib/ExecutionEngine/Orc/IRCompileLayer.cpp4
-rw-r--r--lib/ExecutionEngine/Orc/IRTransformLayer.cpp2
-rw-r--r--lib/ExecutionEngine/Orc/IndirectionUtils.cpp27
-rw-r--r--lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp17
-rw-r--r--lib/ExecutionEngine/Orc/LLJIT.cpp38
-rw-r--r--lib/ExecutionEngine/Orc/Layer.cpp26
-rw-r--r--lib/ExecutionEngine/Orc/LazyReexports.cpp18
-rw-r--r--lib/ExecutionEngine/Orc/Legacy.cpp5
-rw-r--r--lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp258
-rw-r--r--lib/ExecutionEngine/Orc/OrcCBindingsStack.h11
-rw-r--r--lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp24
-rw-r--r--lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp307
-rw-r--r--lib/ExecutionEngine/Orc/Speculation.cpp146
-rw-r--r--lib/ExecutionEngine/Orc/ThreadSafeModule.cpp58
18 files changed, 1249 insertions, 419 deletions
diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
index 99bf53bc3afa..75ddbc30445d 100644
--- a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
+++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp
@@ -54,11 +54,12 @@ static ThreadSafeModule extractSubModule(ThreadSafeModule &TSM,
llvm_unreachable("Unsupported global type");
};
- auto NewTSMod = cloneToNewContext(TSM, ShouldExtract, DeleteExtractedDefs);
- auto &M = *NewTSMod.getModule();
- M.setModuleIdentifier((M.getModuleIdentifier() + Suffix).str());
+ auto NewTSM = cloneToNewContext(TSM, ShouldExtract, DeleteExtractedDefs);
+ NewTSM.withModuleDo([&](Module &M) {
+ M.setModuleIdentifier((M.getModuleIdentifier() + Suffix).str());
+ });
- return NewTSMod;
+ return NewTSM;
}
namespace llvm {
@@ -117,39 +118,44 @@ void CompileOnDemandLayer::setPartitionFunction(PartitionFunction Partition) {
this->Partition = std::move(Partition);
}
+void CompileOnDemandLayer::setImplMap(ImplSymbolMap *Imp) {
+ this->AliaseeImpls = Imp;
+}
void CompileOnDemandLayer::emit(MaterializationResponsibility R,
ThreadSafeModule TSM) {
- assert(TSM.getModule() && "Null module");
+ assert(TSM && "Null module");
auto &ES = getExecutionSession();
- auto &M = *TSM.getModule();
-
- // First, do some cleanup on the module:
- cleanUpModule(M);
- // Now sort the callables and non-callables, build re-exports and lodge the
+ // Sort the callables and non-callables, build re-exports and lodge the
// actual module with the implementation dylib.
auto &PDR = getPerDylibResources(R.getTargetJITDylib());
- MangleAndInterner Mangle(ES, M.getDataLayout());
SymbolAliasMap NonCallables;
SymbolAliasMap Callables;
- for (auto &GV : M.global_values()) {
- if (GV.isDeclaration() || GV.hasLocalLinkage() || GV.hasAppendingLinkage())
- continue;
-
- auto Name = Mangle(GV.getName());
- auto Flags = JITSymbolFlags::fromGlobalValue(GV);
- if (Flags.isCallable())
- Callables[Name] = SymbolAliasMapEntry(Name, Flags);
- else
- NonCallables[Name] = SymbolAliasMapEntry(Name, Flags);
- }
+ TSM.withModuleDo([&](Module &M) {
+ // First, do some cleanup on the module:
+ cleanUpModule(M);
+
+ MangleAndInterner Mangle(ES, M.getDataLayout());
+ for (auto &GV : M.global_values()) {
+ if (GV.isDeclaration() || GV.hasLocalLinkage() ||
+ GV.hasAppendingLinkage())
+ continue;
+
+ auto Name = Mangle(GV.getName());
+ auto Flags = JITSymbolFlags::fromGlobalValue(GV);
+ if (Flags.isCallable())
+ Callables[Name] = SymbolAliasMapEntry(Name, Flags);
+ else
+ NonCallables[Name] = SymbolAliasMapEntry(Name, Flags);
+ }
+ });
// Create a partitioning materialization unit and lodge it with the
// implementation dylib.
if (auto Err = PDR.getImplDylib().define(
- llvm::make_unique<PartitioningIRMaterializationUnit>(
+ std::make_unique<PartitioningIRMaterializationUnit>(
ES, std::move(TSM), R.getVModuleKey(), *this))) {
ES.reportError(std::move(Err));
R.failMaterialization();
@@ -158,7 +164,7 @@ void CompileOnDemandLayer::emit(MaterializationResponsibility R,
R.replace(reexports(PDR.getImplDylib(), std::move(NonCallables), true));
R.replace(lazyReexports(LCTMgr, PDR.getISManager(), PDR.getImplDylib(),
- std::move(Callables)));
+ std::move(Callables), AliaseeImpls));
}
CompileOnDemandLayer::PerDylibResources &
@@ -239,14 +245,16 @@ void CompileOnDemandLayer::emitPartition(
// memory manager instance to the linking layer.
auto &ES = getExecutionSession();
-
GlobalValueSet RequestedGVs;
for (auto &Name : R.getRequestedSymbols()) {
assert(Defs.count(Name) && "No definition for symbol");
RequestedGVs.insert(Defs[Name]);
}
- auto GVsToExtract = Partition(RequestedGVs);
+ /// Perform partitioning with the context lock held, since the partition
+ /// function is allowed to access the globals to compute the partition.
+ auto GVsToExtract =
+ TSM.withModuleDo([&](Module &M) { return Partition(RequestedGVs); });
// Take a 'None' partition to mean the whole module (as opposed to an empty
// partition, which means "materialize nothing"). Emit the whole module
@@ -259,43 +267,52 @@ void CompileOnDemandLayer::emitPartition(
// If the partition is empty, return the whole module to the symbol table.
if (GVsToExtract->empty()) {
- R.replace(llvm::make_unique<PartitioningIRMaterializationUnit>(
+ R.replace(std::make_unique<PartitioningIRMaterializationUnit>(
std::move(TSM), R.getSymbols(), std::move(Defs), *this));
return;
}
// Ok -- we actually need to partition the symbols. Promote the symbol
- // linkages/names.
- // FIXME: We apply this once per partitioning. It's safe, but overkill.
- {
- auto PromotedGlobals = PromoteSymbols(*TSM.getModule());
- if (!PromotedGlobals.empty()) {
- MangleAndInterner Mangle(ES, TSM.getModule()->getDataLayout());
- SymbolFlagsMap SymbolFlags;
- for (auto &GV : PromotedGlobals)
- SymbolFlags[Mangle(GV->getName())] =
- JITSymbolFlags::fromGlobalValue(*GV);
- if (auto Err = R.defineMaterializing(SymbolFlags)) {
- ES.reportError(std::move(Err));
- R.failMaterialization();
- return;
- }
- }
+ // linkages/names, expand the partition to include any required symbols
+ // (i.e. symbols that can't be separated from our partition), and
+ // then extract the partition.
+ //
+ // FIXME: We apply this promotion once per partitioning. It's safe, but
+ // overkill.
+
+ auto ExtractedTSM =
+ TSM.withModuleDo([&](Module &M) -> Expected<ThreadSafeModule> {
+ auto PromotedGlobals = PromoteSymbols(M);
+ if (!PromotedGlobals.empty()) {
+ MangleAndInterner Mangle(ES, M.getDataLayout());
+ SymbolFlagsMap SymbolFlags;
+ for (auto &GV : PromotedGlobals)
+ SymbolFlags[Mangle(GV->getName())] =
+ JITSymbolFlags::fromGlobalValue(*GV);
+ if (auto Err = R.defineMaterializing(SymbolFlags))
+ return std::move(Err);
+ }
+
+ expandPartition(*GVsToExtract);
+
+ // Extract the requested partiton (plus any necessary aliases) and
+ // put the rest back into the impl dylib.
+ auto ShouldExtract = [&](const GlobalValue &GV) -> bool {
+ return GVsToExtract->count(&GV);
+ };
+
+ return extractSubModule(TSM, ".submodule", ShouldExtract);
+ });
+
+ if (!ExtractedTSM) {
+ ES.reportError(ExtractedTSM.takeError());
+ R.failMaterialization();
+ return;
}
- expandPartition(*GVsToExtract);
-
- // Extract the requested partiton (plus any necessary aliases) and
- // put the rest back into the impl dylib.
- auto ShouldExtract = [&](const GlobalValue &GV) -> bool {
- return GVsToExtract->count(&GV);
- };
-
- auto ExtractedTSM = extractSubModule(TSM, ".submodule", ShouldExtract);
- R.replace(llvm::make_unique<PartitioningIRMaterializationUnit>(
+ R.replace(std::make_unique<PartitioningIRMaterializationUnit>(
ES, std::move(TSM), R.getVModuleKey(), *this));
-
- BaseLayer.emit(std::move(R), std::move(ExtractedTSM));
+ BaseLayer.emit(std::move(R), std::move(*ExtractedTSM));
}
} // end namespace orc
diff --git a/lib/ExecutionEngine/Orc/CompileUtils.cpp b/lib/ExecutionEngine/Orc/CompileUtils.cpp
index d46b6fcf9a5f..f8251627a4ef 100644
--- a/lib/ExecutionEngine/Orc/CompileUtils.cpp
+++ b/lib/ExecutionEngine/Orc/CompileUtils.cpp
@@ -42,7 +42,7 @@ SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) {
PM.run(M);
}
- auto ObjBuffer = llvm::make_unique<SmallVectorMemoryBuffer>(
+ auto ObjBuffer = std::make_unique<SmallVectorMemoryBuffer>(
std::move(ObjBufferSV),
"<in memory object compiled from " + M.getModuleIdentifier() + ">");
diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp
index dac37e030e0c..5c7d888c2d6e 100644
--- a/lib/ExecutionEngine/Orc/Core.cpp
+++ b/lib/ExecutionEngine/Orc/Core.cpp
@@ -151,6 +151,8 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolNameSet &Symbols) {
}
raw_ostream &operator<<(raw_ostream &OS, const JITSymbolFlags &Flags) {
+ if (Flags.hasError())
+ OS << "[*ERROR*]";
if (Flags.isCallable())
OS << "[Callable]";
else
@@ -224,7 +226,7 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) {
for (auto &KV : Aliases)
OS << " " << *KV.first << ": " << KV.second.Aliasee << " "
<< KV.second.AliasFlags;
- OS << " }\n";
+ OS << " }";
return OS;
}
@@ -238,15 +240,18 @@ raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) {
return OS << "Materializing";
case SymbolState::Resolved:
return OS << "Resolved";
+ case SymbolState::Emitted:
+ return OS << "Emitted";
case SymbolState::Ready:
return OS << "Ready";
}
llvm_unreachable("Invalid state");
}
-FailedToMaterialize::FailedToMaterialize(SymbolNameSet Symbols)
+FailedToMaterialize::FailedToMaterialize(
+ std::shared_ptr<SymbolDependenceMap> Symbols)
: Symbols(std::move(Symbols)) {
- assert(!this->Symbols.empty() && "Can not fail to resolve an empty set");
+ assert(!this->Symbols->empty() && "Can not fail to resolve an empty set");
}
std::error_code FailedToMaterialize::convertToErrorCode() const {
@@ -254,7 +259,7 @@ std::error_code FailedToMaterialize::convertToErrorCode() const {
}
void FailedToMaterialize::log(raw_ostream &OS) const {
- OS << "Failed to materialize symbols: " << Symbols;
+ OS << "Failed to materialize symbols: " << *Symbols;
}
SymbolsNotFound::SymbolsNotFound(SymbolNameSet Symbols)
@@ -367,35 +372,35 @@ SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const {
return JD.getRequestedSymbols(SymbolFlags);
}
-void MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) {
+Error MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) {
LLVM_DEBUG({
dbgs() << "In " << JD.getName() << " resolving " << Symbols << "\n";
});
#ifndef NDEBUG
for (auto &KV : Symbols) {
+ auto WeakFlags = JITSymbolFlags::Weak | JITSymbolFlags::Common;
auto I = SymbolFlags.find(KV.first);
assert(I != SymbolFlags.end() &&
"Resolving symbol outside this responsibility set");
- if (I->second.isWeak())
- assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) &&
- "Resolving symbol with incorrect flags");
- else
- assert(I->second == KV.second.getFlags() &&
- "Resolving symbol with incorrect flags");
+ assert((KV.second.getFlags() & ~WeakFlags) == (I->second & ~WeakFlags) &&
+ "Resolving symbol with incorrect flags");
}
#endif
- JD.resolve(Symbols);
+ return JD.resolve(Symbols);
}
-void MaterializationResponsibility::notifyEmitted() {
+Error MaterializationResponsibility::notifyEmitted() {
LLVM_DEBUG({
dbgs() << "In " << JD.getName() << " emitting " << SymbolFlags << "\n";
});
- JD.emit(SymbolFlags);
+ if (auto Err = JD.emit(SymbolFlags))
+ return Err;
+
SymbolFlags.clear();
+ return Error::success();
}
Error MaterializationResponsibility::defineMaterializing(
@@ -417,12 +422,13 @@ void MaterializationResponsibility::failMaterialization() {
<< SymbolFlags << "\n";
});
- SymbolNameSet FailedSymbols;
- for (auto &KV : SymbolFlags)
- FailedSymbols.insert(KV.first);
+ JITDylib::FailedSymbolsWorklist Worklist;
- JD.notifyFailed(FailedSymbols);
+ for (auto &KV : SymbolFlags)
+ Worklist.push_back(std::make_pair(&JD, KV.first));
SymbolFlags.clear();
+
+ JD.notifyFailed(std::move(Worklist));
}
void MaterializationResponsibility::replace(
@@ -485,8 +491,9 @@ StringRef AbsoluteSymbolsMaterializationUnit::getName() const {
void AbsoluteSymbolsMaterializationUnit::materialize(
MaterializationResponsibility R) {
- R.notifyResolved(Symbols);
- R.notifyEmitted();
+ // No dependencies, so these calls can't fail.
+ cantFail(R.notifyResolved(Symbols));
+ cantFail(R.notifyEmitted());
}
void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD,
@@ -625,6 +632,7 @@ void ReExportsMaterializationUnit::materialize(
};
auto OnComplete = [QueryInfo](Expected<SymbolMap> Result) {
+ auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession();
if (Result) {
SymbolMap ResolutionMap;
for (auto &KV : QueryInfo->Aliases) {
@@ -633,10 +641,17 @@ void ReExportsMaterializationUnit::materialize(
ResolutionMap[KV.first] = JITEvaluatedSymbol(
(*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags);
}
- QueryInfo->R.notifyResolved(ResolutionMap);
- QueryInfo->R.notifyEmitted();
+ if (auto Err = QueryInfo->R.notifyResolved(ResolutionMap)) {
+ ES.reportError(std::move(Err));
+ QueryInfo->R.failMaterialization();
+ return;
+ }
+ if (auto Err = QueryInfo->R.notifyEmitted()) {
+ ES.reportError(std::move(Err));
+ QueryInfo->R.failMaterialization();
+ return;
+ }
} else {
- auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession();
ES.reportError(Result.takeError());
QueryInfo->R.failMaterialization();
}
@@ -694,7 +709,7 @@ ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD,
Allow(std::move(Allow)) {}
Expected<SymbolNameSet>
-ReexportsGenerator::operator()(JITDylib &JD, const SymbolNameSet &Names) {
+ReexportsGenerator::tryToGenerate(JITDylib &JD, const SymbolNameSet &Names) {
orc::SymbolNameSet Added;
orc::SymbolAliasMap AliasMap;
@@ -716,6 +731,19 @@ ReexportsGenerator::operator()(JITDylib &JD, const SymbolNameSet &Names) {
return Added;
}
+JITDylib::DefinitionGenerator::~DefinitionGenerator() {}
+
+void JITDylib::removeGenerator(DefinitionGenerator &G) {
+ ES.runSessionLocked([&]() {
+ auto I = std::find_if(DefGenerators.begin(), DefGenerators.end(),
+ [&](const std::unique_ptr<DefinitionGenerator> &H) {
+ return H.get() == &G;
+ });
+ assert(I != DefGenerators.end() && "Generator not found");
+ DefGenerators.erase(I);
+ });
+}
+
Error JITDylib::defineMaterializing(const SymbolFlagsMap &SymbolFlags) {
return ES.runSessionLocked([&]() -> Error {
std::vector<SymbolTable::iterator> AddedSyms;
@@ -823,26 +851,52 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name,
assert(Symbols[Name].isInMaterializationPhase() &&
"Can not add dependencies for a symbol that is not materializing");
+ // If Name is already in an error state then just bail out.
+ if (Symbols[Name].getFlags().hasError())
+ return;
+
auto &MI = MaterializingInfos[Name];
- assert(!MI.IsEmitted && "Can not add dependencies to an emitted symbol");
+ assert(Symbols[Name].getState() != SymbolState::Emitted &&
+ "Can not add dependencies to an emitted symbol");
+ bool DependsOnSymbolInErrorState = false;
+
+ // Register dependencies, record whether any depenendency is in the error
+ // state.
for (auto &KV : Dependencies) {
assert(KV.first && "Null JITDylib in dependency?");
auto &OtherJITDylib = *KV.first;
auto &DepsOnOtherJITDylib = MI.UnemittedDependencies[&OtherJITDylib];
for (auto &OtherSymbol : KV.second) {
+
+ // Check the sym entry for the dependency.
+ auto OtherSymI = OtherJITDylib.Symbols.find(OtherSymbol);
+
#ifndef NDEBUG
- // Assert that this symbol exists and has not been emitted already.
- auto SymI = OtherJITDylib.Symbols.find(OtherSymbol);
- assert(SymI != OtherJITDylib.Symbols.end() &&
- (SymI->second.getState() != SymbolState::Ready &&
- "Dependency on emitted symbol"));
+ // Assert that this symbol exists and has not reached the ready state
+ // already.
+ assert(OtherSymI != OtherJITDylib.Symbols.end() &&
+ (OtherSymI->second.getState() != SymbolState::Ready &&
+ "Dependency on emitted/ready symbol"));
#endif
+ auto &OtherSymEntry = OtherSymI->second;
+
+ // If the dependency is in an error state then note this and continue,
+ // we will move this symbol to the error state below.
+ if (OtherSymEntry.getFlags().hasError()) {
+ DependsOnSymbolInErrorState = true;
+ continue;
+ }
+
+ // If the dependency was not in the error state then add it to
+ // our list of dependencies.
+ assert(OtherJITDylib.MaterializingInfos.count(OtherSymbol) &&
+ "No MaterializingInfo for dependency");
auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol];
- if (OtherMI.IsEmitted)
+ if (OtherSymEntry.getState() == SymbolState::Emitted)
transferEmittedNodeDependencies(MI, Name, OtherMI);
else if (&OtherJITDylib != this || OtherSymbol != Name) {
OtherMI.Dependants[this].insert(Name);
@@ -853,63 +907,142 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name,
if (DepsOnOtherJITDylib.empty())
MI.UnemittedDependencies.erase(&OtherJITDylib);
}
+
+ // If this symbol dependended on any symbols in the error state then move
+ // this symbol to the error state too.
+ if (DependsOnSymbolInErrorState)
+ Symbols[Name].setFlags(Symbols[Name].getFlags() | JITSymbolFlags::HasError);
}
-void JITDylib::resolve(const SymbolMap &Resolved) {
- auto CompletedQueries = ES.runSessionLocked([&, this]() {
- AsynchronousSymbolQuerySet CompletedQueries;
+Error JITDylib::resolve(const SymbolMap &Resolved) {
+ SymbolNameSet SymbolsInErrorState;
+ AsynchronousSymbolQuerySet CompletedQueries;
+
+ ES.runSessionLocked([&, this]() {
+ struct WorklistEntry {
+ SymbolTable::iterator SymI;
+ JITEvaluatedSymbol ResolvedSym;
+ };
+
+ std::vector<WorklistEntry> Worklist;
+ Worklist.reserve(Resolved.size());
+
+ // Build worklist and check for any symbols in the error state.
for (const auto &KV : Resolved) {
- auto &Name = KV.first;
- auto Sym = KV.second;
- auto I = Symbols.find(Name);
+ assert(!KV.second.getFlags().hasError() &&
+ "Resolution result can not have error flag set");
- assert(I != Symbols.end() && "Symbol not found");
- assert(!I->second.hasMaterializerAttached() &&
+ auto SymI = Symbols.find(KV.first);
+
+ assert(SymI != Symbols.end() && "Symbol not found");
+ assert(!SymI->second.hasMaterializerAttached() &&
"Resolving symbol with materializer attached?");
- assert(I->second.getState() == SymbolState::Materializing &&
+ assert(SymI->second.getState() == SymbolState::Materializing &&
"Symbol should be materializing");
- assert(I->second.getAddress() == 0 && "Symbol has already been resolved");
+ assert(SymI->second.getAddress() == 0 &&
+ "Symbol has already been resolved");
+
+ if (SymI->second.getFlags().hasError())
+ SymbolsInErrorState.insert(KV.first);
+ else {
+ auto Flags = KV.second.getFlags();
+ Flags &= ~(JITSymbolFlags::Weak | JITSymbolFlags::Common);
+ assert(Flags == (SymI->second.getFlags() &
+ ~(JITSymbolFlags::Weak | JITSymbolFlags::Common)) &&
+ "Resolved flags should match the declared flags");
+
+ Worklist.push_back(
+ {SymI, JITEvaluatedSymbol(KV.second.getAddress(), Flags)});
+ }
+ }
+
+ // If any symbols were in the error state then bail out.
+ if (!SymbolsInErrorState.empty())
+ return;
+
+ while (!Worklist.empty()) {
+ auto SymI = Worklist.back().SymI;
+ auto ResolvedSym = Worklist.back().ResolvedSym;
+ Worklist.pop_back();
- assert((Sym.getFlags() & ~JITSymbolFlags::Weak) ==
- (I->second.getFlags() & ~JITSymbolFlags::Weak) &&
- "Resolved flags should match the declared flags");
+ auto &Name = SymI->first;
- // Once resolved, symbols can never be weak.
- JITSymbolFlags ResolvedFlags = Sym.getFlags();
- ResolvedFlags &= ~JITSymbolFlags::Weak;
- I->second.setAddress(Sym.getAddress());
- I->second.setFlags(ResolvedFlags);
- I->second.setState(SymbolState::Resolved);
+ // Resolved symbols can not be weak: discard the weak flag.
+ JITSymbolFlags ResolvedFlags = ResolvedSym.getFlags();
+ SymI->second.setAddress(ResolvedSym.getAddress());
+ SymI->second.setFlags(ResolvedFlags);
+ SymI->second.setState(SymbolState::Resolved);
auto &MI = MaterializingInfos[Name];
for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) {
- Q->notifySymbolMetRequiredState(Name, Sym);
+ Q->notifySymbolMetRequiredState(Name, ResolvedSym);
+ Q->removeQueryDependence(*this, Name);
if (Q->isComplete())
CompletedQueries.insert(std::move(Q));
}
}
-
- return CompletedQueries;
});
+ assert((SymbolsInErrorState.empty() || CompletedQueries.empty()) &&
+ "Can't fail symbols and completed queries at the same time");
+
+ // If we failed any symbols then return an error.
+ if (!SymbolsInErrorState.empty()) {
+ auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
+ (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
+ return make_error<FailedToMaterialize>(std::move(FailedSymbolsDepMap));
+ }
+
+ // Otherwise notify all the completed queries.
for (auto &Q : CompletedQueries) {
assert(Q->isComplete() && "Q not completed");
Q->handleComplete();
}
+
+ return Error::success();
}
-void JITDylib::emit(const SymbolFlagsMap &Emitted) {
- auto CompletedQueries = ES.runSessionLocked([&, this]() {
- AsynchronousSymbolQuerySet CompletedQueries;
+Error JITDylib::emit(const SymbolFlagsMap &Emitted) {
+ AsynchronousSymbolQuerySet CompletedQueries;
+ SymbolNameSet SymbolsInErrorState;
+ ES.runSessionLocked([&, this]() {
+ std::vector<SymbolTable::iterator> Worklist;
+
+ // Scan to build worklist, record any symbols in the erorr state.
for (const auto &KV : Emitted) {
- const auto &Name = KV.first;
+ auto &Name = KV.first;
+
+ auto SymI = Symbols.find(Name);
+ assert(SymI != Symbols.end() && "No symbol table entry for Name");
+
+ if (SymI->second.getFlags().hasError())
+ SymbolsInErrorState.insert(Name);
+ else
+ Worklist.push_back(SymI);
+ }
+
+ // If any symbols were in the error state then bail out.
+ if (!SymbolsInErrorState.empty())
+ return;
+
+ // Otherwise update dependencies and move to the emitted state.
+ while (!Worklist.empty()) {
+ auto SymI = Worklist.back();
+ Worklist.pop_back();
+
+ auto &Name = SymI->first;
+ auto &SymEntry = SymI->second;
+
+ // Move symbol to the emitted state.
+ assert(SymEntry.getState() == SymbolState::Resolved &&
+ "Emitting from state other than Resolved");
+ SymEntry.setState(SymbolState::Emitted);
auto MII = MaterializingInfos.find(Name);
assert(MII != MaterializingInfos.end() &&
"Missing MaterializingInfo entry");
-
auto &MI = MII->second;
// For each dependant, transfer this node's emitted dependencies to
@@ -926,8 +1059,12 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) {
auto &DependantMI = DependantMII->second;
// Remove the dependant's dependency on this node.
+ assert(DependantMI.UnemittedDependencies.count(this) &&
+ "Dependant does not have an unemitted dependencies record for "
+ "this JITDylib");
assert(DependantMI.UnemittedDependencies[this].count(Name) &&
"Dependant does not count this symbol as a dependency?");
+
DependantMI.UnemittedDependencies[this].erase(Name);
if (DependantMI.UnemittedDependencies[this].empty())
DependantMI.UnemittedDependencies.erase(this);
@@ -936,20 +1073,22 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) {
DependantJD.transferEmittedNodeDependencies(DependantMI,
DependantName, MI);
+ auto DependantSymI = DependantJD.Symbols.find(DependantName);
+ assert(DependantSymI != DependantJD.Symbols.end() &&
+ "Dependant has no entry in the Symbols table");
+ auto &DependantSymEntry = DependantSymI->second;
+
// If the dependant is emitted and this node was the last of its
// unemitted dependencies then the dependant node is now ready, so
// notify any pending queries on the dependant node.
- if (DependantMI.IsEmitted &&
+ if (DependantSymEntry.getState() == SymbolState::Emitted &&
DependantMI.UnemittedDependencies.empty()) {
assert(DependantMI.Dependants.empty() &&
"Dependants should be empty by now");
// Since this dependant is now ready, we erase its MaterializingInfo
// and update its materializing state.
- auto DependantSymI = DependantJD.Symbols.find(DependantName);
- assert(DependantSymI != DependantJD.Symbols.end() &&
- "Dependant has no entry in the Symbols table");
- DependantSymI->second.setState(SymbolState::Ready);
+ DependantSymEntry.setState(SymbolState::Ready);
for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) {
Q->notifySymbolMetRequiredState(
@@ -963,12 +1102,9 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) {
}
}
}
- MI.Dependants.clear();
- MI.IsEmitted = true;
+ MI.Dependants.clear();
if (MI.UnemittedDependencies.empty()) {
- auto SymI = Symbols.find(Name);
- assert(SymI != Symbols.end() && "Symbol has no entry in Symbols table");
SymI->second.setState(SymbolState::Ready);
for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) {
Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol());
@@ -979,80 +1115,138 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) {
MaterializingInfos.erase(MII);
}
}
-
- return CompletedQueries;
});
+ assert((SymbolsInErrorState.empty() || CompletedQueries.empty()) &&
+ "Can't fail symbols and completed queries at the same time");
+
+ // If we failed any symbols then return an error.
+ if (!SymbolsInErrorState.empty()) {
+ auto FailedSymbolsDepMap = std::make_shared<SymbolDependenceMap>();
+ (*FailedSymbolsDepMap)[this] = std::move(SymbolsInErrorState);
+ return make_error<FailedToMaterialize>(std::move(FailedSymbolsDepMap));
+ }
+
+ // Otherwise notify all the completed queries.
for (auto &Q : CompletedQueries) {
assert(Q->isComplete() && "Q is not complete");
Q->handleComplete();
}
+
+ return Error::success();
}
-void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) {
+void JITDylib::notifyFailed(FailedSymbolsWorklist Worklist) {
+ AsynchronousSymbolQuerySet FailedQueries;
+ auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
- // FIXME: This should fail any transitively dependant symbols too.
+ // Failing no symbols is a no-op.
+ if (Worklist.empty())
+ return;
- auto FailedQueriesToNotify = ES.runSessionLocked([&, this]() {
- AsynchronousSymbolQuerySet FailedQueries;
- std::vector<MaterializingInfosMap::iterator> MIIsToRemove;
+ auto &ES = Worklist.front().first->getExecutionSession();
- for (auto &Name : FailedSymbols) {
- auto I = Symbols.find(Name);
- assert(I != Symbols.end() && "Symbol not present in this JITDylib");
- Symbols.erase(I);
+ ES.runSessionLocked([&]() {
+ while (!Worklist.empty()) {
+ assert(Worklist.back().first && "Failed JITDylib can not be null");
+ auto &JD = *Worklist.back().first;
+ auto Name = std::move(Worklist.back().second);
+ Worklist.pop_back();
- auto MII = MaterializingInfos.find(Name);
+ (*FailedSymbolsMap)[&JD].insert(Name);
+
+ assert(JD.Symbols.count(Name) && "No symbol table entry for Name");
+ auto &Sym = JD.Symbols[Name];
- // If we have not created a MaterializingInfo for this symbol yet then
- // there is nobody to notify.
- if (MII == MaterializingInfos.end())
+ // Move the symbol into the error state.
+ // Note that this may be redundant: The symbol might already have been
+ // moved to this state in response to the failure of a dependence.
+ Sym.setFlags(Sym.getFlags() | JITSymbolFlags::HasError);
+
+ // FIXME: Come up with a sane mapping of state to
+ // presence-of-MaterializingInfo so that we can assert presence / absence
+ // here, rather than testing it.
+ auto MII = JD.MaterializingInfos.find(Name);
+
+ if (MII == JD.MaterializingInfos.end())
continue;
- // Remove this symbol from the dependants list of any dependencies.
- for (auto &KV : MII->second.UnemittedDependencies) {
- auto *DependencyJD = KV.first;
- auto &Dependencies = KV.second;
- for (auto &DependencyName : Dependencies) {
- auto DependencyMII =
- DependencyJD->MaterializingInfos.find(DependencyName);
- assert(DependencyMII != DependencyJD->MaterializingInfos.end() &&
- "Unemitted dependency must have a MaterializingInfo entry");
- assert(DependencyMII->second.Dependants.count(this) &&
- "Dependency's dependants list does not contain this JITDylib");
- assert(DependencyMII->second.Dependants[this].count(Name) &&
- "Dependency's dependants list does not contain dependant");
- DependencyMII->second.Dependants[this].erase(Name);
+ auto &MI = MII->second;
+
+ // Move all dependants to the error state and disconnect from them.
+ for (auto &KV : MI.Dependants) {
+ auto &DependantJD = *KV.first;
+ for (auto &DependantName : KV.second) {
+ assert(DependantJD.Symbols.count(DependantName) &&
+ "No symbol table entry for DependantName");
+ auto &DependantSym = DependantJD.Symbols[DependantName];
+ DependantSym.setFlags(DependantSym.getFlags() |
+ JITSymbolFlags::HasError);
+
+ assert(DependantJD.MaterializingInfos.count(DependantName) &&
+ "No MaterializingInfo for dependant");
+ auto &DependantMI = DependantJD.MaterializingInfos[DependantName];
+
+ auto UnemittedDepI = DependantMI.UnemittedDependencies.find(&JD);
+ assert(UnemittedDepI != DependantMI.UnemittedDependencies.end() &&
+ "No UnemittedDependencies entry for this JITDylib");
+ assert(UnemittedDepI->second.count(Name) &&
+ "No UnemittedDependencies entry for this symbol");
+ UnemittedDepI->second.erase(Name);
+ if (UnemittedDepI->second.empty())
+ DependantMI.UnemittedDependencies.erase(UnemittedDepI);
+
+ // If this symbol is already in the emitted state then we need to
+ // take responsibility for failing its queries, so add it to the
+ // worklist.
+ if (DependantSym.getState() == SymbolState::Emitted) {
+ assert(DependantMI.Dependants.empty() &&
+ "Emitted symbol should not have dependants");
+ Worklist.push_back(std::make_pair(&DependantJD, DependantName));
+ }
}
}
+ MI.Dependants.clear();
- // Copy all the queries to the FailedQueries list, then abandon them.
- // This has to be a copy, and the copy has to come before the abandon
- // operation: Each Q.detach() call will reach back into this
- // PendingQueries list to remove Q.
- for (auto &Q : MII->second.pendingQueries())
- FailedQueries.insert(Q);
-
- MIIsToRemove.push_back(std::move(MII));
- }
-
- // Detach failed queries.
- for (auto &Q : FailedQueries)
- Q->detach();
+ // Disconnect from all unemitted depenencies.
+ for (auto &KV : MI.UnemittedDependencies) {
+ auto &UnemittedDepJD = *KV.first;
+ for (auto &UnemittedDepName : KV.second) {
+ auto UnemittedDepMII =
+ UnemittedDepJD.MaterializingInfos.find(UnemittedDepName);
+ assert(UnemittedDepMII != UnemittedDepJD.MaterializingInfos.end() &&
+ "Missing MII for unemitted dependency");
+ assert(UnemittedDepMII->second.Dependants.count(&JD) &&
+ "JD not listed as a dependant of unemitted dependency");
+ assert(UnemittedDepMII->second.Dependants[&JD].count(Name) &&
+ "Name is not listed as a dependant of unemitted dependency");
+ UnemittedDepMII->second.Dependants[&JD].erase(Name);
+ if (UnemittedDepMII->second.Dependants[&JD].empty())
+ UnemittedDepMII->second.Dependants.erase(&JD);
+ }
+ }
+ MI.UnemittedDependencies.clear();
- // Remove the MaterializingInfos.
- for (auto &MII : MIIsToRemove) {
- assert(!MII->second.hasQueriesPending() &&
- "Queries remain after symbol was failed");
+ // Collect queries to be failed for this MII.
+ for (auto &Q : MII->second.pendingQueries()) {
+ // Add the query to the list to be failed and detach it.
+ FailedQueries.insert(Q);
+ Q->detach();
+ }
- MaterializingInfos.erase(MII);
+ assert(MI.Dependants.empty() &&
+ "Can not delete MaterializingInfo with dependants still attached");
+ assert(MI.UnemittedDependencies.empty() &&
+ "Can not delete MaterializingInfo with unemitted dependencies "
+ "still attached");
+ assert(!MI.hasQueriesPending() &&
+ "Can not delete MaterializingInfo with queries pending");
+ JD.MaterializingInfos.erase(MII);
}
-
- return FailedQueries;
});
- for (auto &Q : FailedQueriesToNotify)
- Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbols));
+ for (auto &Q : FailedQueries)
+ Q->handleFailed(make_error<FailedToMaterialize>(FailedSymbolsMap));
}
void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder,
@@ -1159,10 +1353,18 @@ Expected<SymbolFlagsMap> JITDylib::lookupFlags(const SymbolNameSet &Names) {
if (!Unresolved)
return Unresolved.takeError();
- if (DefGenerator && !Unresolved->empty()) {
- auto NewDefs = DefGenerator(*this, *Unresolved);
+ /// Run any definition generators.
+ for (auto &DG : DefGenerators) {
+
+ // Bail out early if we've resolved everything.
+ if (Unresolved->empty())
+ break;
+
+ // Run this generator.
+ auto NewDefs = DG->tryToGenerate(*this, *Unresolved);
if (!NewDefs)
return NewDefs.takeError();
+
if (!NewDefs->empty()) {
auto Unresolved2 = lookupFlagsImpl(Result, *NewDefs);
if (!Unresolved2)
@@ -1171,7 +1373,10 @@ Expected<SymbolFlagsMap> JITDylib::lookupFlags(const SymbolNameSet &Names) {
assert(Unresolved2->empty() &&
"All fallback defs should have been found by lookupFlagsImpl");
}
- };
+
+ for (auto &Name : *NewDefs)
+ Unresolved->erase(Name);
+ }
return Result;
});
}
@@ -1197,15 +1402,34 @@ Error JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q,
MaterializationUnitList &MUs) {
assert(Q && "Query can not be null");
- lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs);
- if (DefGenerator && !Unresolved.empty()) {
- auto NewDefs = DefGenerator(*this, Unresolved);
+ if (auto Err = lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs))
+ return Err;
+
+ // Run any definition generators.
+ for (auto &DG : DefGenerators) {
+
+ // Bail out early if we have resolved everything.
+ if (Unresolved.empty())
+ break;
+
+ // Run the generator.
+ auto NewDefs = DG->tryToGenerate(*this, Unresolved);
+
+ // If the generator returns an error then bail out.
if (!NewDefs)
return NewDefs.takeError();
+
+ // If the generator was able to generate new definitions for any of the
+ // unresolved symbols then lodge the query against them.
if (!NewDefs->empty()) {
for (auto &D : *NewDefs)
Unresolved.erase(D);
- lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs);
+
+ // Lodge query. This can not fail as any new definitions were added
+ // by the generator under the session locked. Since they can't have
+ // started materializing yet the can not have failed.
+ cantFail(lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs));
+
assert(NewDefs->empty() &&
"All fallback defs should have been found by lookupImpl");
}
@@ -1214,7 +1438,7 @@ Error JITDylib::lodgeQuery(std::shared_ptr<AsynchronousSymbolQuery> &Q,
return Error::success();
}
-void JITDylib::lodgeQueryImpl(
+Error JITDylib::lodgeQueryImpl(
std::shared_ptr<AsynchronousSymbolQuery> &Q, SymbolNameSet &Unresolved,
bool MatchNonExported,
std::vector<std::unique_ptr<MaterializationUnit>> &MUs) {
@@ -1235,6 +1459,14 @@ void JITDylib::lodgeQueryImpl(
// Unresolved set.
ToRemove.push_back(Name);
+ // If we matched against this symbol but it is in the error state then
+ // bail out and treat it as a failure to materialize.
+ if (SymI->second.getFlags().hasError()) {
+ auto FailedSymbolsMap = std::make_shared<SymbolDependenceMap>();
+ (*FailedSymbolsMap)[this] = {Name};
+ return make_error<FailedToMaterialize>(std::move(FailedSymbolsMap));
+ }
+
// If this symbol already meets the required state for then notify the
// query and continue.
if (SymI->second.getState() >= Q->getRequiredState()) {
@@ -1277,6 +1509,8 @@ void JITDylib::lodgeQueryImpl(
// Remove any symbols that we found.
for (auto &Name : ToRemove)
Unresolved.erase(Name);
+
+ return Error::success();
}
Expected<SymbolNameSet>
@@ -1292,9 +1526,16 @@ JITDylib::legacyLookup(std::shared_ptr<AsynchronousSymbolQuery> Q,
SymbolNameSet Unresolved = std::move(Names);
auto Err = ES.runSessionLocked([&, this]() -> Error {
QueryComplete = lookupImpl(Q, MUs, Unresolved);
- if (DefGenerator && !Unresolved.empty()) {
+
+ // Run any definition generators.
+ for (auto &DG : DefGenerators) {
+
+ // Bail out early if we have resolved everything.
+ if (Unresolved.empty())
+ break;
+
assert(!QueryComplete && "query complete but unresolved symbols remain?");
- auto NewDefs = DefGenerator(*this, Unresolved);
+ auto NewDefs = DG->tryToGenerate(*this, Unresolved);
if (!NewDefs)
return NewDefs.takeError();
if (!NewDefs->empty()) {
@@ -1432,8 +1673,6 @@ void JITDylib::dump(raw_ostream &OS) {
OS << " MaterializingInfos entries:\n";
for (auto &KV : MaterializingInfos) {
OS << " \"" << *KV.first << "\":\n"
- << " IsEmitted = " << (KV.second.IsEmitted ? "true" : "false")
- << "\n"
<< " " << KV.second.pendingQueries().size()
<< " pending queries: { ";
for (const auto &Q : KV.second.pendingQueries())
@@ -1486,13 +1725,6 @@ JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) {
return Result;
}
-JITDylib::AsynchronousSymbolQueryList
-JITDylib::MaterializingInfo::takeAllQueries() {
- AsynchronousSymbolQueryList Result;
- std::swap(Result, PendingQueries);
- return Result;
-}
-
JITDylib::JITDylib(ExecutionSession &ES, std::string Name)
: ES(ES), JITDylibName(std::move(Name)) {
SearchOrder.push_back({this, true});
diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
index f7fc5f8f1797..4a886ac0597c 100644
--- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
+++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp
@@ -8,6 +8,7 @@
#include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
+#include "llvm/ExecutionEngine/Orc/Layer.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalVariable.h"
@@ -67,7 +68,7 @@ CtorDtorIterator::Element CtorDtorIterator::operator*() const {
}
}
- ConstantInt *Priority = dyn_cast<ConstantInt>(CS->getOperand(0));
+ auto *Priority = cast<ConstantInt>(CS->getOperand(0));
Value *Data = CS->getNumOperands() == 3 ? CS->getOperand(2) : nullptr;
if (Data && !isa<GlobalValue>(Data))
Data = nullptr;
@@ -87,7 +88,7 @@ iterator_range<CtorDtorIterator> getDestructors(const Module &M) {
}
void CtorDtorRunner::add(iterator_range<CtorDtorIterator> CtorDtors) {
- if (empty(CtorDtors))
+ if (CtorDtors.empty())
return;
MangleAndInterner Mangle(
@@ -178,20 +179,20 @@ DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator(
: Dylib(std::move(Dylib)), Allow(std::move(Allow)),
GlobalPrefix(GlobalPrefix) {}
-Expected<DynamicLibrarySearchGenerator>
+Expected<std::unique_ptr<DynamicLibrarySearchGenerator>>
DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix,
SymbolPredicate Allow) {
std::string ErrMsg;
auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg);
if (!Lib.isValid())
return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode());
- return DynamicLibrarySearchGenerator(std::move(Lib), GlobalPrefix,
- std::move(Allow));
+ return std::make_unique<DynamicLibrarySearchGenerator>(
+ std::move(Lib), GlobalPrefix, std::move(Allow));
}
Expected<SymbolNameSet>
-DynamicLibrarySearchGenerator::operator()(JITDylib &JD,
- const SymbolNameSet &Names) {
+DynamicLibrarySearchGenerator::tryToGenerate(JITDylib &JD,
+ const SymbolNameSet &Names) {
orc::SymbolNameSet Added;
orc::SymbolMap NewSymbols;
@@ -226,5 +227,82 @@ DynamicLibrarySearchGenerator::operator()(JITDylib &JD,
return Added;
}
+Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
+StaticLibraryDefinitionGenerator::Load(ObjectLayer &L, const char *FileName) {
+ auto ArchiveBuffer = errorOrToExpected(MemoryBuffer::getFile(FileName));
+
+ if (!ArchiveBuffer)
+ return ArchiveBuffer.takeError();
+
+ return Create(L, std::move(*ArchiveBuffer));
+}
+
+Expected<std::unique_ptr<StaticLibraryDefinitionGenerator>>
+StaticLibraryDefinitionGenerator::Create(
+ ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer) {
+ Error Err = Error::success();
+
+ std::unique_ptr<StaticLibraryDefinitionGenerator> ADG(
+ new StaticLibraryDefinitionGenerator(L, std::move(ArchiveBuffer), Err));
+
+ if (Err)
+ return std::move(Err);
+
+ return std::move(ADG);
+}
+
+Expected<SymbolNameSet>
+StaticLibraryDefinitionGenerator::tryToGenerate(JITDylib &JD,
+ const SymbolNameSet &Names) {
+
+ DenseSet<std::pair<StringRef, StringRef>> ChildBufferInfos;
+ SymbolNameSet NewDefs;
+
+ for (const auto &Name : Names) {
+ auto Child = Archive.findSym(*Name);
+ if (!Child)
+ return Child.takeError();
+ if (*Child == None)
+ continue;
+ auto ChildBuffer = (*Child)->getMemoryBufferRef();
+ if (!ChildBuffer)
+ return ChildBuffer.takeError();
+ ChildBufferInfos.insert(
+ {ChildBuffer->getBuffer(), ChildBuffer->getBufferIdentifier()});
+ NewDefs.insert(Name);
+ }
+
+ for (auto ChildBufferInfo : ChildBufferInfos) {
+ MemoryBufferRef ChildBufferRef(ChildBufferInfo.first,
+ ChildBufferInfo.second);
+
+ if (auto Err =
+ L.add(JD, MemoryBuffer::getMemBuffer(ChildBufferRef), VModuleKey()))
+ return std::move(Err);
+
+ --UnrealizedObjects;
+ }
+
+ return NewDefs;
+}
+
+StaticLibraryDefinitionGenerator::StaticLibraryDefinitionGenerator(
+ ObjectLayer &L, std::unique_ptr<MemoryBuffer> ArchiveBuffer, Error &Err)
+ : L(L), ArchiveBuffer(std::move(ArchiveBuffer)),
+ Archive(*this->ArchiveBuffer, Err) {
+
+ if (Err)
+ return;
+
+ Error Err2 = Error::success();
+ for (auto _ : Archive.children(Err2)) {
+ (void)_;
+ ++UnrealizedObjects;
+ }
+
+ // No need to check this: We will leave it to the caller.
+ Err = std::move(Err2);
+}
+
} // End namespace orc.
} // End namespace llvm.
diff --git a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp
index 81dfc02f55b2..d311f34179c7 100644
--- a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp
+++ b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp
@@ -22,9 +22,9 @@ void IRCompileLayer::setNotifyCompiled(NotifyCompiledFunction NotifyCompiled) {
void IRCompileLayer::emit(MaterializationResponsibility R,
ThreadSafeModule TSM) {
- assert(TSM.getModule() && "Module must not be null");
+ assert(TSM && "Module must not be null");
- if (auto Obj = Compile(*TSM.getModule())) {
+ if (auto Obj = TSM.withModuleDo(Compile)) {
{
std::lock_guard<std::mutex> Lock(IRLayerMutex);
if (NotifyCompiled)
diff --git a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp
index e3519284613e..845ecc71eb87 100644
--- a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp
+++ b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp
@@ -19,7 +19,7 @@ IRTransformLayer::IRTransformLayer(ExecutionSession &ES,
void IRTransformLayer::emit(MaterializationResponsibility R,
ThreadSafeModule TSM) {
- assert(TSM.getModule() && "Module must not be null");
+ assert(TSM && "Module must not be null");
if (auto TransformedTSM = Transform(std::move(TSM), R))
BaseLayer.emit(std::move(R), std::move(*TransformedTSM));
diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
index cc3656fe5dc5..0295db7633dd 100644
--- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
+++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp
@@ -37,8 +37,9 @@ private:
void materialize(MaterializationResponsibility R) override {
SymbolMap Result;
Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported);
- R.notifyResolved(Result);
- R.notifyEmitted();
+ // No dependencies, so these calls cannot fail.
+ cantFail(R.notifyResolved(Result));
+ cantFail(R.notifyEmitted());
}
void discard(const JITDylib &JD, const SymbolStringPtr &Name) override {
@@ -66,7 +67,7 @@ JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) {
std::lock_guard<std::mutex> Lock(CCMgrMutex);
AddrToSymbol[*TrampolineAddr] = CallbackName;
cantFail(CallbacksJD.define(
- llvm::make_unique<CompileCallbackMaterializationUnit>(
+ std::make_unique<CompileCallbackMaterializationUnit>(
std::move(CallbackName), std::move(Compile),
ES.allocateVModule())));
return *TrampolineAddr;
@@ -119,7 +120,8 @@ createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES,
return make_error<StringError>(
std::string("No callback manager available for ") + T.str(),
inconvertibleErrorCode());
- case Triple::aarch64: {
+ case Triple::aarch64:
+ case Triple::aarch64_32: {
typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT;
return CCMgrT::Create(ES, ErrorHandlerAddress);
}
@@ -162,50 +164,51 @@ createLocalIndirectStubsManagerBuilder(const Triple &T) {
switch (T.getArch()) {
default:
return [](){
- return llvm::make_unique<
+ return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcGenericABI>>();
};
case Triple::aarch64:
+ case Triple::aarch64_32:
return [](){
- return llvm::make_unique<
+ return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcAArch64>>();
};
case Triple::x86:
return [](){
- return llvm::make_unique<
+ return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcI386>>();
};
case Triple::mips:
return [](){
- return llvm::make_unique<
+ return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcMips32Be>>();
};
case Triple::mipsel:
return [](){
- return llvm::make_unique<
+ return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcMips32Le>>();
};
case Triple::mips64:
case Triple::mips64el:
return [](){
- return llvm::make_unique<
+ return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcMips64>>();
};
case Triple::x86_64:
if (T.getOS() == Triple::OSType::Win32) {
return [](){
- return llvm::make_unique<
+ return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcX86_64_Win32>>();
};
} else {
return [](){
- return llvm::make_unique<
+ return std::make_unique<
orc::LocalIndirectStubsManager<orc::OrcX86_64_SysV>>();
};
}
diff --git a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp
index df23547a9de3..1d3e6db913e2 100644
--- a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp
+++ b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp
@@ -8,6 +8,7 @@
#include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h"
+#include "llvm/Support/Host.h"
#include "llvm/Support/TargetRegistry.h"
namespace llvm {
@@ -22,7 +23,21 @@ JITTargetMachineBuilder::JITTargetMachineBuilder(Triple TT)
Expected<JITTargetMachineBuilder> JITTargetMachineBuilder::detectHost() {
// FIXME: getProcessTriple is bogus. It returns the host LLVM was compiled on,
// rather than a valid triple for the current process.
- return JITTargetMachineBuilder(Triple(sys::getProcessTriple()));
+ JITTargetMachineBuilder TMBuilder((Triple(sys::getProcessTriple())));
+
+ // Retrieve host CPU name and sub-target features and add them to builder.
+ // Relocation model, code model and codegen opt level are kept to default
+ // values.
+ llvm::SubtargetFeatures SubtargetFeatures;
+ llvm::StringMap<bool> FeatureMap;
+ llvm::sys::getHostCPUFeatures(FeatureMap);
+ for (auto &Feature : FeatureMap)
+ SubtargetFeatures.AddFeature(Feature.first(), Feature.second);
+
+ TMBuilder.setCPU(llvm::sys::getHostCPUName());
+ TMBuilder.addFeatures(SubtargetFeatures.getFeatures());
+
+ return TMBuilder;
}
Expected<std::unique_ptr<TargetMachine>>
diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp
index b120691faf07..a80f78afe80f 100644
--- a/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -41,7 +41,8 @@ Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) {
Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) {
assert(TSM && "Can not add null module");
- if (auto Err = applyDataLayout(*TSM.getModule()))
+ if (auto Err =
+ TSM.withModuleDo([&](Module &M) { return applyDataLayout(M); }))
return Err;
return CompileLayer->add(JD, std::move(TSM), ES->allocateVModule());
@@ -63,12 +64,21 @@ LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) {
// If the config state provided an ObjectLinkingLayer factory then use it.
if (S.CreateObjectLinkingLayer)
- return S.CreateObjectLinkingLayer(ES);
+ return S.CreateObjectLinkingLayer(ES, S.JTMB->getTargetTriple());
// Otherwise default to creating an RTDyldObjectLinkingLayer that constructs
// a new SectionMemoryManager for each object.
- auto GetMemMgr = []() { return llvm::make_unique<SectionMemoryManager>(); };
- return llvm::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
+ auto GetMemMgr = []() { return std::make_unique<SectionMemoryManager>(); };
+ auto ObjLinkingLayer =
+ std::make_unique<RTDyldObjectLinkingLayer>(ES, std::move(GetMemMgr));
+
+ if (S.JTMB->getTargetTriple().isOSBinFormatCOFF())
+ ObjLinkingLayer->setOverrideObjectFlagsWithResponsibilityFlags(true);
+
+ // FIXME: Explicit conversion to std::unique_ptr<ObjectLayer> added to silence
+ // errors from some GCC / libstdc++ bots. Remove this conversion (i.e.
+ // just return ObjLinkingLayer) once those bots are upgraded.
+ return std::unique_ptr<ObjectLayer>(std::move(ObjLinkingLayer));
}
Expected<IRCompileLayer::CompileFunction>
@@ -92,7 +102,7 @@ LLJIT::createCompileFunction(LLJITBuilderState &S,
}
LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
- : ES(S.ES ? std::move(S.ES) : llvm::make_unique<ExecutionSession>()),
+ : ES(S.ES ? std::move(S.ES) : std::make_unique<ExecutionSession>()),
Main(this->ES->getMainJITDylib()), DL(""), CtorRunner(Main),
DtorRunner(Main) {
@@ -113,13 +123,13 @@ LLJIT::LLJIT(LLJITBuilderState &S, Error &Err)
Err = CompileFunction.takeError();
return;
}
- CompileLayer = llvm::make_unique<IRCompileLayer>(
+ CompileLayer = std::make_unique<IRCompileLayer>(
*ES, *ObjLinkingLayer, std::move(*CompileFunction));
}
if (S.NumCompileThreads > 0) {
CompileLayer->setCloneToNewContextOnEmit(true);
- CompileThreads = llvm::make_unique<ThreadPool>(S.NumCompileThreads);
+ CompileThreads = std::make_unique<ThreadPool>(S.NumCompileThreads);
ES->setDispatchMaterialization(
[this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) {
// FIXME: Switch to move capture once we have c++14.
@@ -166,10 +176,14 @@ Error LLLazyJITBuilderState::prepareForConstruction() {
Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
assert(TSM && "Can not add null module");
- if (auto Err = applyDataLayout(*TSM.getModule()))
- return Err;
+ if (auto Err = TSM.withModuleDo([&](Module &M) -> Error {
+ if (auto Err = applyDataLayout(M))
+ return Err;
- recordCtorDtors(*TSM.getModule());
+ recordCtorDtors(M);
+ return Error::success();
+ }))
+ return Err;
return CODLayer->add(JD, std::move(TSM), ES->allocateVModule());
}
@@ -212,10 +226,10 @@ LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) {
}
// Create the transform layer.
- TransformLayer = llvm::make_unique<IRTransformLayer>(*ES, *CompileLayer);
+ TransformLayer = std::make_unique<IRTransformLayer>(*ES, *CompileLayer);
// Create the COD layer.
- CODLayer = llvm::make_unique<CompileOnDemandLayer>(
+ CODLayer = std::make_unique<CompileOnDemandLayer>(
*ES, *TransformLayer, *LCTMgr, std::move(ISMBuilder));
if (S.NumCompileThreads > 0)
diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp
index 3ed2dabf4545..580e2682ec8c 100644
--- a/lib/ExecutionEngine/Orc/Layer.cpp
+++ b/lib/ExecutionEngine/Orc/Layer.cpp
@@ -19,7 +19,7 @@ IRLayer::IRLayer(ExecutionSession &ES) : ES(ES) {}
IRLayer::~IRLayer() {}
Error IRLayer::add(JITDylib &JD, ThreadSafeModule TSM, VModuleKey K) {
- return JD.define(llvm::make_unique<BasicIRLayerMaterializationUnit>(
+ return JD.define(std::make_unique<BasicIRLayerMaterializationUnit>(
*this, std::move(K), std::move(TSM)));
}
@@ -29,15 +29,17 @@ IRMaterializationUnit::IRMaterializationUnit(ExecutionSession &ES,
assert(this->TSM && "Module must not be null");
- MangleAndInterner Mangle(ES, this->TSM.getModule()->getDataLayout());
- for (auto &G : this->TSM.getModule()->global_values()) {
- if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() &&
- !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) {
- auto MangledName = Mangle(G.getName());
- SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G);
- SymbolToDefinition[MangledName] = &G;
+ MangleAndInterner Mangle(ES, this->TSM.getModuleUnlocked()->getDataLayout());
+ this->TSM.withModuleDo([&](Module &M) {
+ for (auto &G : M.global_values()) {
+ if (G.hasName() && !G.isDeclaration() && !G.hasLocalLinkage() &&
+ !G.hasAvailableExternallyLinkage() && !G.hasAppendingLinkage()) {
+ auto MangledName = Mangle(G.getName());
+ SymbolFlags[MangledName] = JITSymbolFlags::fromGlobalValue(G);
+ SymbolToDefinition[MangledName] = &G;
+ }
}
- }
+ });
}
IRMaterializationUnit::IRMaterializationUnit(
@@ -47,8 +49,9 @@ IRMaterializationUnit::IRMaterializationUnit(
TSM(std::move(TSM)), SymbolToDefinition(std::move(SymbolToDefinition)) {}
StringRef IRMaterializationUnit::getName() const {
- if (TSM.getModule())
- return TSM.getModule()->getModuleIdentifier();
+ if (TSM)
+ return TSM.withModuleDo(
+ [](const Module &M) -> StringRef { return M.getModuleIdentifier(); });
return "<null module>";
}
@@ -90,7 +93,6 @@ void BasicIRLayerMaterializationUnit::materialize(
auto &N = R.getTargetJITDylib().getName();
#endif // NDEBUG
- auto Lock = TSM.getContextLock();
LLVM_DEBUG(ES.runSessionLocked(
[&]() { dbgs() << "Emitting, for " << N << ", " << *this << "\n"; }););
L.emit(std::move(R), std::move(TSM));
diff --git a/lib/ExecutionEngine/Orc/LazyReexports.cpp b/lib/ExecutionEngine/Orc/LazyReexports.cpp
index fc8205845654..93aabd817d60 100644
--- a/lib/ExecutionEngine/Orc/LazyReexports.cpp
+++ b/lib/ExecutionEngine/Orc/LazyReexports.cpp
@@ -50,7 +50,6 @@ LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) {
SourceJD = I->second.first;
SymbolName = I->second.second;
}
-
auto LookupResult =
ES.lookup(JITDylibSearchList({{SourceJD, true}}), SymbolName);
@@ -91,6 +90,7 @@ createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES,
inconvertibleErrorCode());
case Triple::aarch64:
+ case Triple::aarch64_32:
return LocalLazyCallThroughManager::Create<OrcAArch64>(ES,
ErrorHandlerAddr);
@@ -121,7 +121,8 @@ createLocalLazyCallThroughManager(const Triple &T, ExecutionSession &ES,
LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit(
LazyCallThroughManager &LCTManager, IndirectStubsManager &ISManager,
- JITDylib &SourceJD, SymbolAliasMap CallableAliases, VModuleKey K)
+ JITDylib &SourceJD, SymbolAliasMap CallableAliases, ImplSymbolMap *SrcJDLoc,
+ VModuleKey K)
: MaterializationUnit(extractFlags(CallableAliases), std::move(K)),
LCTManager(LCTManager), ISManager(ISManager), SourceJD(SourceJD),
CallableAliases(std::move(CallableAliases)),
@@ -129,7 +130,8 @@ LazyReexportsMaterializationUnit::LazyReexportsMaterializationUnit(
[&ISManager](JITDylib &JD, const SymbolStringPtr &SymbolName,
JITTargetAddress ResolvedAddr) {
return ISManager.updatePointer(*SymbolName, ResolvedAddr);
- })) {}
+ })),
+ AliaseeTable(SrcJDLoc) {}
StringRef LazyReexportsMaterializationUnit::getName() const {
return "<Lazy Reexports>";
@@ -149,7 +151,7 @@ void LazyReexportsMaterializationUnit::materialize(
if (!CallableAliases.empty())
R.replace(lazyReexports(LCTManager, ISManager, SourceJD,
- std::move(CallableAliases)));
+ std::move(CallableAliases), AliaseeTable));
IndirectStubsManager::StubInitsMap StubInits;
for (auto &Alias : RequestedAliases) {
@@ -168,6 +170,9 @@ void LazyReexportsMaterializationUnit::materialize(
std::make_pair(*CallThroughTrampoline, Alias.second.AliasFlags);
}
+ if (AliaseeTable != nullptr && !RequestedAliases.empty())
+ AliaseeTable->trackImpls(RequestedAliases, &SourceJD);
+
if (auto Err = ISManager.createStubs(StubInits)) {
SourceJD.getExecutionSession().reportError(std::move(Err));
R.failMaterialization();
@@ -178,8 +183,9 @@ void LazyReexportsMaterializationUnit::materialize(
for (auto &Alias : RequestedAliases)
Stubs[Alias.first] = ISManager.findStub(*Alias.first, false);
- R.notifyResolved(Stubs);
- R.notifyEmitted();
+ // No registered dependencies, so these calls cannot fail.
+ cantFail(R.notifyResolved(Stubs));
+ cantFail(R.notifyEmitted());
}
void LazyReexportsMaterializationUnit::discard(const JITDylib &JD,
diff --git a/lib/ExecutionEngine/Orc/Legacy.cpp b/lib/ExecutionEngine/Orc/Legacy.cpp
index ce6368b57a89..9f9a6730b2c3 100644
--- a/lib/ExecutionEngine/Orc/Legacy.cpp
+++ b/lib/ExecutionEngine/Orc/Legacy.cpp
@@ -23,7 +23,8 @@ void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols,
for (auto &S : Symbols)
InternedSymbols.insert(ES.intern(S));
- auto OnResolvedWithUnwrap = [OnResolved](Expected<SymbolMap> InternedResult) {
+ auto OnResolvedWithUnwrap = [OnResolved = std::move(OnResolved)](
+ Expected<SymbolMap> InternedResult) mutable {
if (!InternedResult) {
OnResolved(InternedResult.takeError());
return;
@@ -36,7 +37,7 @@ void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols,
};
auto Q = std::make_shared<AsynchronousSymbolQuery>(
- InternedSymbols, SymbolState::Resolved, OnResolvedWithUnwrap);
+ InternedSymbols, SymbolState::Resolved, std::move(OnResolvedWithUnwrap));
auto Unresolved = R.lookup(Q, InternedSymbols);
if (Unresolved.empty()) {
diff --git a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
index def0b300eca1..874decb2ade0 100644
--- a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
+++ b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp
@@ -29,6 +29,13 @@ public:
std::unique_ptr<MemoryBuffer> ObjBuffer)
: Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {}
+ ~ObjectLinkingLayerJITLinkContext() {
+ // If there is an object buffer return function then use it to
+ // return ownership of the buffer.
+ if (Layer.ReturnObjectBuffer)
+ Layer.ReturnObjectBuffer(std::move(ObjBuffer));
+ }
+
JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; }
MemoryBufferRef getObjectBuffer() const override {
@@ -41,7 +48,7 @@ public:
}
void lookup(const DenseSet<StringRef> &Symbols,
- JITLinkAsyncLookupContinuation LookupContinuation) override {
+ std::unique_ptr<JITLinkAsyncLookupContinuation> LC) override {
JITDylibSearchList SearchOrder;
MR.getTargetJITDylib().withSearchOrderDo(
@@ -54,18 +61,16 @@ public:
InternedSymbols.insert(ES.intern(S));
// OnResolve -- De-intern the symbols and pass the result to the linker.
- // FIXME: Capture LookupContinuation by move once we have c++14.
- auto SharedLookupContinuation =
- std::make_shared<JITLinkAsyncLookupContinuation>(
- std::move(LookupContinuation));
- auto OnResolve = [SharedLookupContinuation](Expected<SymbolMap> Result) {
+ auto OnResolve = [this, LookupContinuation = std::move(LC)](
+ Expected<SymbolMap> Result) mutable {
+ auto Main = Layer.getExecutionSession().intern("_main");
if (!Result)
- (*SharedLookupContinuation)(Result.takeError());
+ LookupContinuation->run(Result.takeError());
else {
AsyncLookupResult LR;
for (auto &KV : *Result)
LR[*KV.first] = KV.second;
- (*SharedLookupContinuation)(std::move(LR));
+ LookupContinuation->run(std::move(LR));
}
};
@@ -75,29 +80,25 @@ public:
});
}
- void notifyResolved(AtomGraph &G) override {
+ void notifyResolved(LinkGraph &G) override {
auto &ES = Layer.getExecutionSession();
SymbolFlagsMap ExtraSymbolsToClaim;
bool AutoClaim = Layer.AutoClaimObjectSymbols;
SymbolMap InternedResult;
- for (auto *DA : G.defined_atoms())
- if (DA->hasName() && DA->isGlobal()) {
- auto InternedName = ES.intern(DA->getName());
+ for (auto *Sym : G.defined_symbols())
+ if (Sym->hasName() && Sym->getScope() != Scope::Local) {
+ auto InternedName = ES.intern(Sym->getName());
JITSymbolFlags Flags;
- if (DA->isExported())
- Flags |= JITSymbolFlags::Exported;
- if (DA->isWeak())
- Flags |= JITSymbolFlags::Weak;
- if (DA->isCallable())
+ if (Sym->isCallable())
Flags |= JITSymbolFlags::Callable;
- if (DA->isCommon())
- Flags |= JITSymbolFlags::Common;
+ if (Sym->getScope() == Scope::Default)
+ Flags |= JITSymbolFlags::Exported;
InternedResult[InternedName] =
- JITEvaluatedSymbol(DA->getAddress(), Flags);
+ JITEvaluatedSymbol(Sym->getAddress(), Flags);
if (AutoClaim && !MR.getSymbols().count(InternedName)) {
assert(!ExtraSymbolsToClaim.count(InternedName) &&
"Duplicate symbol to claim?");
@@ -105,17 +106,17 @@ public:
}
}
- for (auto *A : G.absolute_atoms())
- if (A->hasName()) {
- auto InternedName = ES.intern(A->getName());
+ for (auto *Sym : G.absolute_symbols())
+ if (Sym->hasName()) {
+ auto InternedName = ES.intern(Sym->getName());
JITSymbolFlags Flags;
Flags |= JITSymbolFlags::Absolute;
- if (A->isWeak())
- Flags |= JITSymbolFlags::Weak;
- if (A->isCallable())
+ if (Sym->isCallable())
Flags |= JITSymbolFlags::Callable;
+ if (Sym->getLinkage() == Linkage::Weak)
+ Flags |= JITSymbolFlags::Weak;
InternedResult[InternedName] =
- JITEvaluatedSymbol(A->getAddress(), Flags);
+ JITEvaluatedSymbol(Sym->getAddress(), Flags);
if (AutoClaim && !MR.getSymbols().count(InternedName)) {
assert(!ExtraSymbolsToClaim.count(InternedName) &&
"Duplicate symbol to claim?");
@@ -126,35 +127,38 @@ public:
if (!ExtraSymbolsToClaim.empty())
if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim))
return notifyFailed(std::move(Err));
-
- MR.notifyResolved(InternedResult);
-
+ if (auto Err = MR.notifyResolved(InternedResult)) {
+ Layer.getExecutionSession().reportError(std::move(Err));
+ MR.failMaterialization();
+ return;
+ }
Layer.notifyLoaded(MR);
}
void notifyFinalized(
std::unique_ptr<JITLinkMemoryManager::Allocation> A) override {
-
if (auto Err = Layer.notifyEmitted(MR, std::move(A))) {
Layer.getExecutionSession().reportError(std::move(Err));
MR.failMaterialization();
-
return;
}
- MR.notifyEmitted();
+ if (auto Err = MR.notifyEmitted()) {
+ Layer.getExecutionSession().reportError(std::move(Err));
+ MR.failMaterialization();
+ }
}
- AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override {
- return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); };
+ LinkGraphPassFunction getMarkLivePass(const Triple &TT) const override {
+ return [this](LinkGraph &G) { return markResponsibilitySymbolsLive(G); };
}
Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override {
// Add passes to mark duplicate defs as should-discard, and to walk the
- // atom graph to build the symbol dependence graph.
+ // link graph to build the symbol dependence graph.
Config.PrePrunePasses.push_back(
- [this](AtomGraph &G) { return markSymbolsToDiscard(G); });
+ [this](LinkGraph &G) { return externalizeWeakAndCommonSymbols(G); });
Config.PostPrunePasses.push_back(
- [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); });
+ [this](LinkGraph &G) { return computeNamedSymbolDependencies(G); });
Layer.modifyPassConfig(MR, TT, Config);
@@ -162,65 +166,59 @@ public:
}
private:
- using AnonAtomNamedDependenciesMap =
- DenseMap<const DefinedAtom *, SymbolNameSet>;
+ using AnonToNamedDependenciesMap = DenseMap<const Symbol *, SymbolNameSet>;
- Error markSymbolsToDiscard(AtomGraph &G) {
+ Error externalizeWeakAndCommonSymbols(LinkGraph &G) {
auto &ES = Layer.getExecutionSession();
- for (auto *DA : G.defined_atoms())
- if (DA->isWeak() && DA->hasName()) {
- auto S = ES.intern(DA->getName());
- auto I = MR.getSymbols().find(S);
- if (I == MR.getSymbols().end())
- DA->setShouldDiscard(true);
+ for (auto *Sym : G.defined_symbols())
+ if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) {
+ if (!MR.getSymbols().count(ES.intern(Sym->getName())))
+ G.makeExternal(*Sym);
}
- for (auto *A : G.absolute_atoms())
- if (A->isWeak() && A->hasName()) {
- auto S = ES.intern(A->getName());
- auto I = MR.getSymbols().find(S);
- if (I == MR.getSymbols().end())
- A->setShouldDiscard(true);
+ for (auto *Sym : G.absolute_symbols())
+ if (Sym->hasName() && Sym->getLinkage() == Linkage::Weak) {
+ if (!MR.getSymbols().count(ES.intern(Sym->getName())))
+ G.makeExternal(*Sym);
}
return Error::success();
}
- Error markResponsibilitySymbolsLive(AtomGraph &G) const {
+ Error markResponsibilitySymbolsLive(LinkGraph &G) const {
auto &ES = Layer.getExecutionSession();
- for (auto *DA : G.defined_atoms())
- if (DA->hasName() &&
- MR.getSymbols().count(ES.intern(DA->getName())))
- DA->setLive(true);
+ for (auto *Sym : G.defined_symbols())
+ if (Sym->hasName() && MR.getSymbols().count(ES.intern(Sym->getName())))
+ Sym->setLive(true);
return Error::success();
}
- Error computeNamedSymbolDependencies(AtomGraph &G) {
+ Error computeNamedSymbolDependencies(LinkGraph &G) {
auto &ES = MR.getTargetJITDylib().getExecutionSession();
auto AnonDeps = computeAnonDeps(G);
- for (auto *DA : G.defined_atoms()) {
+ for (auto *Sym : G.defined_symbols()) {
// Skip anonymous and non-global atoms: we do not need dependencies for
// these.
- if (!DA->hasName() || !DA->isGlobal())
+ if (Sym->getScope() == Scope::Local)
continue;
- auto DAName = ES.intern(DA->getName());
- SymbolNameSet &DADeps = NamedSymbolDeps[DAName];
+ auto SymName = ES.intern(Sym->getName());
+ SymbolNameSet &SymDeps = NamedSymbolDeps[SymName];
- for (auto &E : DA->edges()) {
- auto &TA = E.getTarget();
+ for (auto &E : Sym->getBlock().edges()) {
+ auto &TargetSym = E.getTarget();
- if (TA.hasName())
- DADeps.insert(ES.intern(TA.getName()));
+ if (TargetSym.getScope() != Scope::Local)
+ SymDeps.insert(ES.intern(TargetSym.getName()));
else {
- assert(TA.isDefined() && "Anonymous atoms must be defined");
- auto &DTA = static_cast<DefinedAtom &>(TA);
- auto I = AnonDeps.find(&DTA);
+ assert(TargetSym.isDefined() &&
+ "Anonymous/local symbols must be defined");
+ auto I = AnonDeps.find(&TargetSym);
if (I != AnonDeps.end())
for (auto &S : I->second)
- DADeps.insert(S);
+ SymDeps.insert(S);
}
}
}
@@ -228,58 +226,59 @@ private:
return Error::success();
}
- AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) {
+ AnonToNamedDependenciesMap computeAnonDeps(LinkGraph &G) {
auto &ES = MR.getTargetJITDylib().getExecutionSession();
- AnonAtomNamedDependenciesMap DepMap;
+ AnonToNamedDependenciesMap DepMap;
- // For all anonymous atoms:
+ // For all anonymous symbols:
// (1) Add their named dependencies.
// (2) Add them to the worklist for further iteration if they have any
- // depend on any other anonymous atoms.
+ // depend on any other anonymous symbols.
struct WorklistEntry {
- WorklistEntry(DefinedAtom *DA, DenseSet<DefinedAtom *> DAAnonDeps)
- : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {}
+ WorklistEntry(Symbol *Sym, DenseSet<Symbol *> SymAnonDeps)
+ : Sym(Sym), SymAnonDeps(std::move(SymAnonDeps)) {}
- DefinedAtom *DA = nullptr;
- DenseSet<DefinedAtom *> DAAnonDeps;
+ Symbol *Sym = nullptr;
+ DenseSet<Symbol *> SymAnonDeps;
};
std::vector<WorklistEntry> Worklist;
- for (auto *DA : G.defined_atoms())
- if (!DA->hasName()) {
- auto &DANamedDeps = DepMap[DA];
- DenseSet<DefinedAtom *> DAAnonDeps;
-
- for (auto &E : DA->edges()) {
- auto &TA = E.getTarget();
- if (TA.hasName())
- DANamedDeps.insert(ES.intern(TA.getName()));
+ for (auto *Sym : G.defined_symbols())
+ if (!Sym->hasName()) {
+ auto &SymNamedDeps = DepMap[Sym];
+ DenseSet<Symbol *> SymAnonDeps;
+
+ for (auto &E : Sym->getBlock().edges()) {
+ auto &TargetSym = E.getTarget();
+ if (TargetSym.hasName())
+ SymNamedDeps.insert(ES.intern(TargetSym.getName()));
else {
- assert(TA.isDefined() && "Anonymous atoms must be defined");
- DAAnonDeps.insert(static_cast<DefinedAtom *>(&TA));
+ assert(TargetSym.isDefined() &&
+ "Anonymous symbols must be defined");
+ SymAnonDeps.insert(&TargetSym);
}
}
- if (!DAAnonDeps.empty())
- Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps)));
+ if (!SymAnonDeps.empty())
+ Worklist.push_back(WorklistEntry(Sym, std::move(SymAnonDeps)));
}
- // Loop over all anonymous atoms with anonymous dependencies, propagating
+ // Loop over all anonymous symbols with anonymous dependencies, propagating
// their respective *named* dependencies. Iterate until we hit a stable
// state.
bool Changed;
do {
Changed = false;
for (auto &WLEntry : Worklist) {
- auto *DA = WLEntry.DA;
- auto &DANamedDeps = DepMap[DA];
- auto &DAAnonDeps = WLEntry.DAAnonDeps;
+ auto *Sym = WLEntry.Sym;
+ auto &SymNamedDeps = DepMap[Sym];
+ auto &SymAnonDeps = WLEntry.SymAnonDeps;
- for (auto *TA : DAAnonDeps) {
- auto I = DepMap.find(TA);
+ for (auto *TargetSym : SymAnonDeps) {
+ auto I = DepMap.find(TargetSym);
if (I != DepMap.end())
for (const auto &S : I->second)
- Changed |= DANamedDeps.insert(S).second;
+ Changed |= SymNamedDeps.insert(S).second;
}
}
} while (Changed);
@@ -330,7 +329,7 @@ ObjectLinkingLayer::~ObjectLinkingLayer() {
void ObjectLinkingLayer::emit(MaterializationResponsibility R,
std::unique_ptr<MemoryBuffer> O) {
assert(O && "Object must not be null");
- jitLink(llvm::make_unique<ObjectLinkingLayerJITLinkContext>(
+ jitLink(std::make_unique<ObjectLinkingLayerJITLinkContext>(
*this, std::move(R), std::move(O)));
}
@@ -410,7 +409,7 @@ Error ObjectLinkingLayer::removeAllModules() {
}
EHFrameRegistrationPlugin::EHFrameRegistrationPlugin(
- jitlink::EHFrameRegistrar &Registrar)
+ EHFrameRegistrar &Registrar)
: Registrar(Registrar) {}
void EHFrameRegistrationPlugin::modifyPassConfig(
@@ -419,61 +418,66 @@ void EHFrameRegistrationPlugin::modifyPassConfig(
assert(!InProcessLinks.count(&MR) && "Link for MR already being tracked?");
PassConfig.PostFixupPasses.push_back(
- createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr) {
+ createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr,
+ size_t Size) {
if (Addr)
- InProcessLinks[&MR] = Addr;
+ InProcessLinks[&MR] = { Addr, Size };
}));
}
Error EHFrameRegistrationPlugin::notifyEmitted(
MaterializationResponsibility &MR) {
- auto EHFrameAddrItr = InProcessLinks.find(&MR);
- if (EHFrameAddrItr == InProcessLinks.end())
+ auto EHFrameRangeItr = InProcessLinks.find(&MR);
+ if (EHFrameRangeItr == InProcessLinks.end())
return Error::success();
- auto EHFrameAddr = EHFrameAddrItr->second;
- assert(EHFrameAddr && "eh-frame addr to register can not be null");
+ auto EHFrameRange = EHFrameRangeItr->second;
+ assert(EHFrameRange.Addr &&
+ "eh-frame addr to register can not be null");
- InProcessLinks.erase(EHFrameAddrItr);
+ InProcessLinks.erase(EHFrameRangeItr);
if (auto Key = MR.getVModuleKey())
- TrackedEHFrameAddrs[Key] = EHFrameAddr;
+ TrackedEHFrameRanges[Key] = EHFrameRange;
else
- UntrackedEHFrameAddrs.push_back(EHFrameAddr);
+ UntrackedEHFrameRanges.push_back(EHFrameRange);
- return Registrar.registerEHFrames(EHFrameAddr);
+ return Registrar.registerEHFrames(EHFrameRange.Addr, EHFrameRange.Size);
}
Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) {
- auto EHFrameAddrItr = TrackedEHFrameAddrs.find(K);
- if (EHFrameAddrItr == TrackedEHFrameAddrs.end())
+ auto EHFrameRangeItr = TrackedEHFrameRanges.find(K);
+ if (EHFrameRangeItr == TrackedEHFrameRanges.end())
return Error::success();
- auto EHFrameAddr = EHFrameAddrItr->second;
- assert(EHFrameAddr && "Tracked eh-frame addr must not be null");
+ auto EHFrameRange = EHFrameRangeItr->second;
+ assert(EHFrameRange.Addr && "Tracked eh-frame range must not be null");
- TrackedEHFrameAddrs.erase(EHFrameAddrItr);
+ TrackedEHFrameRanges.erase(EHFrameRangeItr);
- return Registrar.deregisterEHFrames(EHFrameAddr);
+ return Registrar.deregisterEHFrames(EHFrameRange.Addr, EHFrameRange.Size);
}
Error EHFrameRegistrationPlugin::notifyRemovingAllModules() {
- std::vector<JITTargetAddress> EHFrameAddrs = std::move(UntrackedEHFrameAddrs);
- EHFrameAddrs.reserve(EHFrameAddrs.size() + TrackedEHFrameAddrs.size());
+ std::vector<EHFrameRange> EHFrameRanges =
+ std::move(UntrackedEHFrameRanges);
+ EHFrameRanges.reserve(EHFrameRanges.size() + TrackedEHFrameRanges.size());
- for (auto &KV : TrackedEHFrameAddrs)
- EHFrameAddrs.push_back(KV.second);
+ for (auto &KV : TrackedEHFrameRanges)
+ EHFrameRanges.push_back(KV.second);
- TrackedEHFrameAddrs.clear();
+ TrackedEHFrameRanges.clear();
Error Err = Error::success();
- while (!EHFrameAddrs.empty()) {
- auto EHFrameAddr = EHFrameAddrs.back();
- assert(EHFrameAddr && "Untracked eh-frame addr must not be null");
- EHFrameAddrs.pop_back();
- Err = joinErrors(std::move(Err), Registrar.deregisterEHFrames(EHFrameAddr));
+ while (!EHFrameRanges.empty()) {
+ auto EHFrameRange = EHFrameRanges.back();
+ assert(EHFrameRange.Addr && "Untracked eh-frame range must not be null");
+ EHFrameRanges.pop_back();
+ Err = joinErrors(std::move(Err),
+ Registrar.deregisterEHFrames(EHFrameRange.Addr,
+ EHFrameRange.Size));
}
return Err;
diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h
index 98129e1690d2..e0af3df9d010 100644
--- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h
+++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h
@@ -97,7 +97,7 @@ public:
template <typename LayerT>
std::unique_ptr<GenericLayerImpl<LayerT>> createGenericLayer(LayerT &Layer) {
- return llvm::make_unique<GenericLayerImpl<LayerT>>(Layer);
+ return std::make_unique<GenericLayerImpl<LayerT>>(Layer);
}
} // end namespace detail
@@ -316,7 +316,8 @@ public:
if (auto Err = CtorRunner.runViaLayer(*this))
return std::move(Err);
- IRStaticDestructorRunners.emplace_back(std::move(DtorNames), K);
+ IRStaticDestructorRunners.emplace_back(AcknowledgeORCv1Deprecation,
+ std::move(DtorNames), K);
return K;
}
@@ -326,7 +327,7 @@ public:
LLVMOrcSymbolResolverFn ExternalResolver,
void *ExternalResolverCtx) {
return addIRModule(CompileLayer, std::move(M),
- llvm::make_unique<SectionMemoryManager>(),
+ std::make_unique<SectionMemoryManager>(),
std::move(ExternalResolver), ExternalResolverCtx);
}
@@ -340,7 +341,7 @@ public:
inconvertibleErrorCode());
return addIRModule(*CODLayer, std::move(M),
- llvm::make_unique<SectionMemoryManager>(),
+ std::make_unique<SectionMemoryManager>(),
std::move(ExternalResolver), ExternalResolverCtx);
}
@@ -468,7 +469,7 @@ private:
if (!CCMgr)
return nullptr;
- return llvm::make_unique<CODLayerT>(
+ return std::make_unique<CODLayerT>(
AcknowledgeORCv1Deprecation, ES, CompileLayer,
[&Resolvers](orc::VModuleKey K) {
auto ResolverI = Resolvers.find(K);
diff --git a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
index b22ecd5f80a1..939cd539d1fb 100644
--- a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
+++ b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp
@@ -27,9 +27,9 @@ public:
// Build an OnResolve callback to unwrap the interned strings and pass them
// to the OnResolved callback.
- // FIXME: Switch to move capture of OnResolved once we have c++14.
auto OnResolvedWithUnwrap =
- [OnResolved](Expected<SymbolMap> InternedResult) {
+ [OnResolved = std::move(OnResolved)](
+ Expected<SymbolMap> InternedResult) mutable {
if (!InternedResult) {
OnResolved(InternedResult.takeError());
return;
@@ -50,7 +50,7 @@ public:
MR.getTargetJITDylib().withSearchOrderDo(
[&](const JITDylibSearchList &JDs) { SearchOrder = JDs; });
ES.lookup(SearchOrder, InternedSymbols, SymbolState::Resolved,
- OnResolvedWithUnwrap, RegisterDependencies);
+ std::move(OnResolvedWithUnwrap), RegisterDependencies);
}
Expected<LookupSet> getResponsibilitySet(const LookupSet &Symbols) {
@@ -133,8 +133,6 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R,
JITDylibSearchOrderResolver Resolver(*SharedR);
- // FIXME: Switch to move-capture for the 'O' buffer once we have c++14.
- MemoryBuffer *UnownedObjBuffer = O.release();
jitLinkForORC(
**Obj, std::move(O), *MemMgr, Resolver, ProcessAllSections,
[this, K, SharedR, &Obj, InternalSymbols](
@@ -143,9 +141,8 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R,
return onObjLoad(K, *SharedR, **Obj, std::move(LoadedObjInfo),
ResolvedSymbols, *InternalSymbols);
},
- [this, K, SharedR, UnownedObjBuffer](Error Err) {
- std::unique_ptr<MemoryBuffer> ObjBuffer(UnownedObjBuffer);
- onObjEmit(K, std::move(ObjBuffer), *SharedR, std::move(Err));
+ [this, K, SharedR, O = std::move(O)](Error Err) mutable {
+ onObjEmit(K, std::move(O), *SharedR, std::move(Err));
});
}
@@ -184,7 +181,10 @@ Error RTDyldObjectLinkingLayer::onObjLoad(
if (auto Err = R.defineMaterializing(ExtraSymbolsToClaim))
return Err;
- R.notifyResolved(Symbols);
+ if (auto Err = R.notifyResolved(Symbols)) {
+ R.failMaterialization();
+ return Err;
+ }
if (NotifyLoaded)
NotifyLoaded(K, Obj, *LoadedObjInfo);
@@ -201,7 +201,11 @@ void RTDyldObjectLinkingLayer::onObjEmit(
return;
}
- R.notifyEmitted();
+ if (auto Err = R.notifyEmitted()) {
+ getExecutionSession().reportError(std::move(Err));
+ R.failMaterialization();
+ return;
+ }
if (NotifyEmitted)
NotifyEmitted(K, std::move(ObjBuffer));
diff --git a/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp b/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp
new file mode 100644
index 000000000000..f22acf50419d
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/SpeculateAnalyses.cpp
@@ -0,0 +1,307 @@
+//===-- SpeculateAnalyses.cpp --*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/SpeculateAnalyses.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
+#include "llvm/Analysis/CFG.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/ErrorHandling.h"
+
+#include <algorithm>
+
+namespace {
+using namespace llvm;
+SmallVector<const BasicBlock *, 8> findBBwithCalls(const Function &F,
+ bool IndirectCall = false) {
+ SmallVector<const BasicBlock *, 8> BBs;
+
+ auto findCallInst = [&IndirectCall](const Instruction &I) {
+ if (auto Call = dyn_cast<CallBase>(&I))
+ return Call->isIndirectCall() ? IndirectCall : true;
+ else
+ return false;
+ };
+ for (auto &BB : F)
+ if (findCallInst(*BB.getTerminator()) ||
+ llvm::any_of(BB.instructionsWithoutDebug(), findCallInst))
+ BBs.emplace_back(&BB);
+
+ return BBs;
+}
+} // namespace
+
+// Implementations of Queries shouldn't need to lock the resources
+// such as LLVMContext, each argument (function) has a non-shared LLVMContext
+// Plus, if Queries contain states necessary locking scheme should be provided.
+namespace llvm {
+namespace orc {
+
+// Collect direct calls only
+void SpeculateQuery::findCalles(const BasicBlock *BB,
+ DenseSet<StringRef> &CallesNames) {
+ assert(BB != nullptr && "Traversing Null BB to find calls?");
+
+ auto getCalledFunction = [&CallesNames](const CallBase *Call) {
+ auto CalledValue = Call->getCalledOperand()->stripPointerCasts();
+ if (auto DirectCall = dyn_cast<Function>(CalledValue))
+ CallesNames.insert(DirectCall->getName());
+ };
+ for (auto &I : BB->instructionsWithoutDebug())
+ if (auto CI = dyn_cast<CallInst>(&I))
+ getCalledFunction(CI);
+
+ if (auto II = dyn_cast<InvokeInst>(BB->getTerminator()))
+ getCalledFunction(II);
+}
+
+bool SpeculateQuery::isStraightLine(const Function &F) {
+ return llvm::all_of(F.getBasicBlockList(), [](const BasicBlock &BB) {
+ return BB.getSingleSuccessor() != nullptr;
+ });
+}
+
+// BlockFreqQuery Implementations
+
+size_t BlockFreqQuery::numBBToGet(size_t numBB) {
+ // small CFG
+ if (numBB < 4)
+ return numBB;
+ // mid-size CFG
+ else if (numBB < 20)
+ return (numBB / 2);
+ else
+ return (numBB / 2) + (numBB / 4);
+}
+
+BlockFreqQuery::ResultTy BlockFreqQuery::operator()(Function &F) {
+ DenseMap<StringRef, DenseSet<StringRef>> CallerAndCalles;
+ DenseSet<StringRef> Calles;
+ SmallVector<std::pair<const BasicBlock *, uint64_t>, 8> BBFreqs;
+
+ PassBuilder PB;
+ FunctionAnalysisManager FAM;
+ PB.registerFunctionAnalyses(FAM);
+
+ auto IBBs = findBBwithCalls(F);
+
+ if (IBBs.empty())
+ return None;
+
+ auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
+
+ for (const auto I : IBBs)
+ BBFreqs.push_back({I, BFI.getBlockFreq(I).getFrequency()});
+
+ assert(IBBs.size() == BBFreqs.size() && "BB Count Mismatch");
+
+ llvm::sort(BBFreqs.begin(), BBFreqs.end(),
+ [](decltype(BBFreqs)::const_reference BBF,
+ decltype(BBFreqs)::const_reference BBS) {
+ return BBF.second > BBS.second ? true : false;
+ });
+
+ // ignoring number of direct calls in a BB
+ auto Topk = numBBToGet(BBFreqs.size());
+
+ for (size_t i = 0; i < Topk; i++)
+ findCalles(BBFreqs[i].first, Calles);
+
+ assert(!Calles.empty() && "Running Analysis on Function with no calls?");
+
+ CallerAndCalles.insert({F.getName(), std::move(Calles)});
+
+ return CallerAndCalles;
+}
+
+// SequenceBBQuery Implementation
+std::size_t SequenceBBQuery::getHottestBlocks(std::size_t TotalBlocks) {
+ if (TotalBlocks == 1)
+ return TotalBlocks;
+ return TotalBlocks / 2;
+}
+
+// FIXME : find good implementation.
+SequenceBBQuery::BlockListTy
+SequenceBBQuery::rearrangeBB(const Function &F, const BlockListTy &BBList) {
+ BlockListTy RearrangedBBSet;
+
+ for (auto &Block : F.getBasicBlockList())
+ if (llvm::is_contained(BBList, &Block))
+ RearrangedBBSet.push_back(&Block);
+
+ assert(RearrangedBBSet.size() == BBList.size() &&
+ "BasicBlock missing while rearranging?");
+ return RearrangedBBSet;
+}
+
+void SequenceBBQuery::traverseToEntryBlock(const BasicBlock *AtBB,
+ const BlockListTy &CallerBlocks,
+ const BackEdgesInfoTy &BackEdgesInfo,
+ const BranchProbabilityInfo *BPI,
+ VisitedBlocksInfoTy &VisitedBlocks) {
+ auto Itr = VisitedBlocks.find(AtBB);
+ if (Itr != VisitedBlocks.end()) { // already visited.
+ if (!Itr->second.Upward)
+ return;
+ Itr->second.Upward = false;
+ } else {
+ // Create hint for newly discoverd blocks.
+ WalkDirection BlockHint;
+ BlockHint.Upward = false;
+ // FIXME: Expensive Check
+ if (llvm::is_contained(CallerBlocks, AtBB))
+ BlockHint.CallerBlock = true;
+ VisitedBlocks.insert(std::make_pair(AtBB, BlockHint));
+ }
+
+ const_pred_iterator PIt = pred_begin(AtBB), EIt = pred_end(AtBB);
+ // Move this check to top, when we have code setup to launch speculative
+ // compiles for function in entry BB, this triggers the speculative compiles
+ // before running the program.
+ if (PIt == EIt) // No Preds.
+ return;
+
+ DenseSet<const BasicBlock *> PredSkipNodes;
+
+ // Since we are checking for predecessor's backedges, this Block
+ // occurs in second position.
+ for (auto &I : BackEdgesInfo)
+ if (I.second == AtBB)
+ PredSkipNodes.insert(I.first);
+
+ // Skip predecessors which source of back-edges.
+ for (; PIt != EIt; ++PIt)
+ // checking EdgeHotness is cheaper
+ if (BPI->isEdgeHot(*PIt, AtBB) && !PredSkipNodes.count(*PIt))
+ traverseToEntryBlock(*PIt, CallerBlocks, BackEdgesInfo, BPI,
+ VisitedBlocks);
+}
+
+void SequenceBBQuery::traverseToExitBlock(const BasicBlock *AtBB,
+ const BlockListTy &CallerBlocks,
+ const BackEdgesInfoTy &BackEdgesInfo,
+ const BranchProbabilityInfo *BPI,
+ VisitedBlocksInfoTy &VisitedBlocks) {
+ auto Itr = VisitedBlocks.find(AtBB);
+ if (Itr != VisitedBlocks.end()) { // already visited.
+ if (!Itr->second.Downward)
+ return;
+ Itr->second.Downward = false;
+ } else {
+ // Create hint for newly discoverd blocks.
+ WalkDirection BlockHint;
+ BlockHint.Downward = false;
+ // FIXME: Expensive Check
+ if (llvm::is_contained(CallerBlocks, AtBB))
+ BlockHint.CallerBlock = true;
+ VisitedBlocks.insert(std::make_pair(AtBB, BlockHint));
+ }
+
+ succ_const_iterator PIt = succ_begin(AtBB), EIt = succ_end(AtBB);
+ if (PIt == EIt) // No succs.
+ return;
+
+ // If there are hot edges, then compute SuccSkipNodes.
+ DenseSet<const BasicBlock *> SuccSkipNodes;
+
+ // Since we are checking for successor's backedges, this Block
+ // occurs in first position.
+ for (auto &I : BackEdgesInfo)
+ if (I.first == AtBB)
+ SuccSkipNodes.insert(I.second);
+
+ for (; PIt != EIt; ++PIt)
+ if (BPI->isEdgeHot(AtBB, *PIt) && !SuccSkipNodes.count(*PIt))
+ traverseToExitBlock(*PIt, CallerBlocks, BackEdgesInfo, BPI,
+ VisitedBlocks);
+}
+
+// Get Block frequencies for blocks and take most frquently executed block,
+// walk towards the entry block from those blocks and discover the basic blocks
+// with call.
+SequenceBBQuery::BlockListTy
+SequenceBBQuery::queryCFG(Function &F, const BlockListTy &CallerBlocks) {
+
+ BlockFreqInfoTy BBFreqs;
+ VisitedBlocksInfoTy VisitedBlocks;
+ BackEdgesInfoTy BackEdgesInfo;
+
+ PassBuilder PB;
+ FunctionAnalysisManager FAM;
+ PB.registerFunctionAnalyses(FAM);
+
+ auto &BFI = FAM.getResult<BlockFrequencyAnalysis>(F);
+
+ llvm::FindFunctionBackedges(F, BackEdgesInfo);
+
+ for (const auto I : CallerBlocks)
+ BBFreqs.push_back({I, BFI.getBlockFreq(I).getFrequency()});
+
+ llvm::sort(BBFreqs, [](decltype(BBFreqs)::const_reference Bbf,
+ decltype(BBFreqs)::const_reference Bbs) {
+ return Bbf.second > Bbs.second;
+ });
+
+ ArrayRef<std::pair<const BasicBlock *, uint64_t>> HotBlocksRef(BBFreqs);
+ HotBlocksRef =
+ HotBlocksRef.drop_back(BBFreqs.size() - getHottestBlocks(BBFreqs.size()));
+
+ BranchProbabilityInfo *BPI =
+ FAM.getCachedResult<BranchProbabilityAnalysis>(F);
+
+ // visit NHotBlocks,
+ // traverse upwards to entry
+ // traverse downwards to end.
+
+ for (auto I : HotBlocksRef) {
+ traverseToEntryBlock(I.first, CallerBlocks, BackEdgesInfo, BPI,
+ VisitedBlocks);
+ traverseToExitBlock(I.first, CallerBlocks, BackEdgesInfo, BPI,
+ VisitedBlocks);
+ }
+
+ BlockListTy MinCallerBlocks;
+ for (auto &I : VisitedBlocks)
+ if (I.second.CallerBlock)
+ MinCallerBlocks.push_back(std::move(I.first));
+
+ return rearrangeBB(F, MinCallerBlocks);
+}
+
+SpeculateQuery::ResultTy SequenceBBQuery::operator()(Function &F) {
+ // reduce the number of lists!
+ DenseMap<StringRef, DenseSet<StringRef>> CallerAndCalles;
+ DenseSet<StringRef> Calles;
+ BlockListTy SequencedBlocks;
+ BlockListTy CallerBlocks;
+
+ CallerBlocks = findBBwithCalls(F);
+ if (CallerBlocks.empty())
+ return None;
+
+ if (isStraightLine(F))
+ SequencedBlocks = rearrangeBB(F, CallerBlocks);
+ else
+ SequencedBlocks = queryCFG(F, CallerBlocks);
+
+ for (auto BB : SequencedBlocks)
+ findCalles(BB, Calles);
+
+ CallerAndCalles.insert({F.getName(), std::move(Calles)});
+ return CallerAndCalles;
+}
+
+} // namespace orc
+} // namespace llvm
diff --git a/lib/ExecutionEngine/Orc/Speculation.cpp b/lib/ExecutionEngine/Orc/Speculation.cpp
new file mode 100644
index 000000000000..f29201c147a1
--- /dev/null
+++ b/lib/ExecutionEngine/Orc/Speculation.cpp
@@ -0,0 +1,146 @@
+//===---------- speculation.cpp - Utilities for Speculation ----------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/Speculation.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/Support/Debug.h"
+
+#include <vector>
+
+namespace llvm {
+
+namespace orc {
+
+// ImplSymbolMap methods
+void ImplSymbolMap::trackImpls(SymbolAliasMap ImplMaps, JITDylib *SrcJD) {
+ assert(SrcJD && "Tracking on Null Source .impl dylib");
+ std::lock_guard<std::mutex> Lockit(ConcurrentAccess);
+ for (auto &I : ImplMaps) {
+ auto It = Maps.insert({I.first, {I.second.Aliasee, SrcJD}});
+ // check rationale when independent dylibs have same symbol name?
+ assert(It.second && "ImplSymbols are already tracked for this Symbol?");
+ (void)(It);
+ }
+}
+
+// Trigger Speculative Compiles.
+void Speculator::speculateForEntryPoint(Speculator *Ptr, uint64_t StubId) {
+ assert(Ptr && " Null Address Received in orc_speculate_for ");
+ Ptr->speculateFor(StubId);
+}
+
+Error Speculator::addSpeculationRuntime(JITDylib &JD,
+ MangleAndInterner &Mangle) {
+ JITEvaluatedSymbol ThisPtr(pointerToJITTargetAddress(this),
+ JITSymbolFlags::Exported);
+ JITEvaluatedSymbol SpeculateForEntryPtr(
+ pointerToJITTargetAddress(&speculateForEntryPoint),
+ JITSymbolFlags::Exported);
+ return JD.define(absoluteSymbols({
+ {Mangle("__orc_speculator"), ThisPtr}, // Data Symbol
+ {Mangle("__orc_speculate_for"), SpeculateForEntryPtr} // Callable Symbol
+ }));
+}
+
+// If two modules, share the same LLVMContext, different threads must
+// not access them concurrently without locking the associated LLVMContext
+// this implementation follows this contract.
+void IRSpeculationLayer::emit(MaterializationResponsibility R,
+ ThreadSafeModule TSM) {
+
+ assert(TSM && "Speculation Layer received Null Module ?");
+ assert(TSM.getContext().getContext() != nullptr &&
+ "Module with null LLVMContext?");
+
+ // Instrumentation of runtime calls, lock the Module
+ TSM.withModuleDo([this, &R](Module &M) {
+ auto &MContext = M.getContext();
+ auto SpeculatorVTy = StructType::create(MContext, "Class.Speculator");
+ auto RuntimeCallTy = FunctionType::get(
+ Type::getVoidTy(MContext),
+ {SpeculatorVTy->getPointerTo(), Type::getInt64Ty(MContext)}, false);
+ auto RuntimeCall =
+ Function::Create(RuntimeCallTy, Function::LinkageTypes::ExternalLinkage,
+ "__orc_speculate_for", &M);
+ auto SpeclAddr = new GlobalVariable(
+ M, SpeculatorVTy, false, GlobalValue::LinkageTypes::ExternalLinkage,
+ nullptr, "__orc_speculator");
+
+ IRBuilder<> Mutator(MContext);
+
+ // QueryAnalysis allowed to transform the IR source, one such example is
+ // Simplify CFG helps the static branch prediction heuristics!
+ for (auto &Fn : M.getFunctionList()) {
+ if (!Fn.isDeclaration()) {
+
+ auto IRNames = QueryAnalysis(Fn);
+ // Instrument and register if Query has result
+ if (IRNames.hasValue()) {
+
+ // Emit globals for each function.
+ auto LoadValueTy = Type::getInt8Ty(MContext);
+ auto SpeculatorGuard = new GlobalVariable(
+ M, LoadValueTy, false, GlobalValue::LinkageTypes::InternalLinkage,
+ ConstantInt::get(LoadValueTy, 0),
+ "__orc_speculate.guard.for." + Fn.getName());
+ SpeculatorGuard->setAlignment(Align::None());
+ SpeculatorGuard->setUnnamedAddr(GlobalValue::UnnamedAddr::Local);
+
+ BasicBlock &ProgramEntry = Fn.getEntryBlock();
+ // Create BasicBlocks before the program's entry basicblock
+ BasicBlock *SpeculateBlock = BasicBlock::Create(
+ MContext, "__orc_speculate.block", &Fn, &ProgramEntry);
+ BasicBlock *SpeculateDecisionBlock = BasicBlock::Create(
+ MContext, "__orc_speculate.decision.block", &Fn, SpeculateBlock);
+
+ assert(SpeculateDecisionBlock == &Fn.getEntryBlock() &&
+ "SpeculateDecisionBlock not updated?");
+ Mutator.SetInsertPoint(SpeculateDecisionBlock);
+
+ auto LoadGuard =
+ Mutator.CreateLoad(LoadValueTy, SpeculatorGuard, "guard.value");
+ // if just loaded value equal to 0,return true.
+ auto CanSpeculate =
+ Mutator.CreateICmpEQ(LoadGuard, ConstantInt::get(LoadValueTy, 0),
+ "compare.to.speculate");
+ Mutator.CreateCondBr(CanSpeculate, SpeculateBlock, &ProgramEntry);
+
+ Mutator.SetInsertPoint(SpeculateBlock);
+ auto ImplAddrToUint =
+ Mutator.CreatePtrToInt(&Fn, Type::getInt64Ty(MContext));
+ Mutator.CreateCall(RuntimeCallTy, RuntimeCall,
+ {SpeclAddr, ImplAddrToUint});
+ Mutator.CreateStore(ConstantInt::get(LoadValueTy, 1),
+ SpeculatorGuard);
+ Mutator.CreateBr(&ProgramEntry);
+
+ assert(Mutator.GetInsertBlock()->getParent() == &Fn &&
+ "IR builder association mismatch?");
+ S.registerSymbols(internToJITSymbols(IRNames.getValue()),
+ &R.getTargetJITDylib());
+ }
+ }
+ }
+ });
+
+ assert(!TSM.withModuleDo([](const Module &M) { return verifyModule(M); }) &&
+ "Speculation Instrumentation breaks IR?");
+
+ NextLayer.emit(std::move(R), std::move(TSM));
+}
+
+} // namespace orc
+} // namespace llvm
diff --git a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
index 4cb7376758a7..1f4e6f132115 100644
--- a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
+++ b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp
@@ -23,41 +23,41 @@ ThreadSafeModule cloneToNewContext(ThreadSafeModule &TSM,
if (!ShouldCloneDef)
ShouldCloneDef = [](const GlobalValue &) { return true; };
- auto Lock = TSM.getContextLock();
+ return TSM.withModuleDo([&](Module &M) {
+ SmallVector<char, 1> ClonedModuleBuffer;
- SmallVector<char, 1> ClonedModuleBuffer;
+ {
+ std::set<GlobalValue *> ClonedDefsInSrc;
+ ValueToValueMapTy VMap;
+ auto Tmp = CloneModule(M, VMap, [&](const GlobalValue *GV) {
+ if (ShouldCloneDef(*GV)) {
+ ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV));
+ return true;
+ }
+ return false;
+ });
- {
- std::set<GlobalValue *> ClonedDefsInSrc;
- ValueToValueMapTy VMap;
- auto Tmp = CloneModule(*TSM.getModule(), VMap, [&](const GlobalValue *GV) {
- if (ShouldCloneDef(*GV)) {
- ClonedDefsInSrc.insert(const_cast<GlobalValue *>(GV));
- return true;
- }
- return false;
- });
+ if (UpdateClonedDefSource)
+ for (auto *GV : ClonedDefsInSrc)
+ UpdateClonedDefSource(*GV);
- if (UpdateClonedDefSource)
- for (auto *GV : ClonedDefsInSrc)
- UpdateClonedDefSource(*GV);
+ BitcodeWriter BCWriter(ClonedModuleBuffer);
- BitcodeWriter BCWriter(ClonedModuleBuffer);
+ BCWriter.writeModule(*Tmp);
+ BCWriter.writeSymtab();
+ BCWriter.writeStrtab();
+ }
- BCWriter.writeModule(*Tmp);
- BCWriter.writeSymtab();
- BCWriter.writeStrtab();
- }
+ MemoryBufferRef ClonedModuleBufferRef(
+ StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()),
+ "cloned module buffer");
+ ThreadSafeContext NewTSCtx(std::make_unique<LLVMContext>());
- MemoryBufferRef ClonedModuleBufferRef(
- StringRef(ClonedModuleBuffer.data(), ClonedModuleBuffer.size()),
- "cloned module buffer");
- ThreadSafeContext NewTSCtx(llvm::make_unique<LLVMContext>());
-
- auto ClonedModule =
- cantFail(parseBitcodeFile(ClonedModuleBufferRef, *NewTSCtx.getContext()));
- ClonedModule->setModuleIdentifier(TSM.getModule()->getName());
- return ThreadSafeModule(std::move(ClonedModule), std::move(NewTSCtx));
+ auto ClonedModule = cantFail(
+ parseBitcodeFile(ClonedModuleBufferRef, *NewTSCtx.getContext()));
+ ClonedModule->setModuleIdentifier(M.getName());
+ return ThreadSafeModule(std::move(ClonedModule), std::move(NewTSCtx));
+ });
}
} // end namespace orc
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-19 09:22:04 +0000