summaryrefslogtreecommitdiff
path: root/lib/LTO/LTO.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/LTO/LTO.cpp')
-rw-r--r--lib/LTO/LTO.cpp210
1 files changed, 134 insertions, 76 deletions
diff --git a/lib/LTO/LTO.cpp b/lib/LTO/LTO.cpp
index 19973946ac5a..64e5186255bd 100644
--- a/lib/LTO/LTO.cpp
+++ b/lib/LTO/LTO.cpp
@@ -65,7 +65,9 @@ static void computeCacheKey(
const FunctionImporter::ExportSetTy &ExportList,
const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
const GVSummaryMapTy &DefinedGlobals,
- const TypeIdSummariesByGuidTy &TypeIdSummariesByGuid) {
+ const TypeIdSummariesByGuidTy &TypeIdSummariesByGuid,
+ const std::set<GlobalValue::GUID> &CfiFunctionDefs,
+ const std::set<GlobalValue::GUID> &CfiFunctionDecls) {
// Compute the unique hash for this entry.
// This is based on the current compiler version, the module itself, the
// export list, the hash for every single module in the import list, the
@@ -118,7 +120,10 @@ static void computeCacheKey(
AddUnsigned(*Conf.RelocModel);
else
AddUnsigned(-1);
- AddUnsigned(Conf.CodeModel);
+ if (Conf.CodeModel)
+ AddUnsigned(*Conf.CodeModel);
+ else
+ AddUnsigned(-1);
AddUnsigned(Conf.CGOptLevel);
AddUnsigned(Conf.CGFileType);
AddUnsigned(Conf.OptLevel);
@@ -155,22 +160,39 @@ static void computeCacheKey(
sizeof(GlobalValue::LinkageTypes)));
}
+ // Members of CfiFunctionDefs and CfiFunctionDecls that are referenced or
+ // defined in this module.
+ std::set<GlobalValue::GUID> UsedCfiDefs;
+ std::set<GlobalValue::GUID> UsedCfiDecls;
+
+ // Typeids used in this module.
std::set<GlobalValue::GUID> UsedTypeIds;
- auto AddUsedTypeIds = [&](GlobalValueSummary *GS) {
- auto *FS = dyn_cast_or_null<FunctionSummary>(GS);
- if (!FS)
- return;
- for (auto &TT : FS->type_tests())
- UsedTypeIds.insert(TT);
- for (auto &TT : FS->type_test_assume_vcalls())
- UsedTypeIds.insert(TT.GUID);
- for (auto &TT : FS->type_checked_load_vcalls())
- UsedTypeIds.insert(TT.GUID);
- for (auto &TT : FS->type_test_assume_const_vcalls())
- UsedTypeIds.insert(TT.VFunc.GUID);
- for (auto &TT : FS->type_checked_load_const_vcalls())
- UsedTypeIds.insert(TT.VFunc.GUID);
+ auto AddUsedCfiGlobal = [&](GlobalValue::GUID ValueGUID) {
+ if (CfiFunctionDefs.count(ValueGUID))
+ UsedCfiDefs.insert(ValueGUID);
+ if (CfiFunctionDecls.count(ValueGUID))
+ UsedCfiDecls.insert(ValueGUID);
+ };
+
+ auto AddUsedThings = [&](GlobalValueSummary *GS) {
+ if (!GS) return;
+ for (const ValueInfo &VI : GS->refs())
+ AddUsedCfiGlobal(VI.getGUID());
+ if (auto *FS = dyn_cast<FunctionSummary>(GS)) {
+ for (auto &TT : FS->type_tests())
+ UsedTypeIds.insert(TT);
+ for (auto &TT : FS->type_test_assume_vcalls())
+ UsedTypeIds.insert(TT.GUID);
+ for (auto &TT : FS->type_checked_load_vcalls())
+ UsedTypeIds.insert(TT.GUID);
+ for (auto &TT : FS->type_test_assume_const_vcalls())
+ UsedTypeIds.insert(TT.VFunc.GUID);
+ for (auto &TT : FS->type_checked_load_const_vcalls())
+ UsedTypeIds.insert(TT.VFunc.GUID);
+ for (auto &ET : FS->calls())
+ AddUsedCfiGlobal(ET.first.getGUID());
+ }
};
// Include the hash for the linkage type to reflect internalization and weak
@@ -179,14 +201,15 @@ static void computeCacheKey(
GlobalValue::LinkageTypes Linkage = GS.second->linkage();
Hasher.update(
ArrayRef<uint8_t>((const uint8_t *)&Linkage, sizeof(Linkage)));
- AddUsedTypeIds(GS.second);
+ AddUsedCfiGlobal(GS.first);
+ AddUsedThings(GS.second);
}
// Imported functions may introduce new uses of type identifier resolutions,
// so we need to collect their used resolutions as well.
for (auto &ImpM : ImportList)
for (auto &ImpF : ImpM.second)
- AddUsedTypeIds(Index.findSummaryInModule(ImpF.first, ImpM.first()));
+ AddUsedThings(Index.findSummaryInModule(ImpF.first, ImpM.first()));
auto AddTypeIdSummary = [&](StringRef TId, const TypeIdSummary &S) {
AddString(TId);
@@ -194,6 +217,11 @@ static void computeCacheKey(
AddUnsigned(S.TTRes.TheKind);
AddUnsigned(S.TTRes.SizeM1BitWidth);
+ AddUint64(S.TTRes.AlignLog2);
+ AddUint64(S.TTRes.SizeM1);
+ AddUint64(S.TTRes.BitMask);
+ AddUint64(S.TTRes.InlineBits);
+
AddUint64(S.WPDRes.size());
for (auto &WPD : S.WPDRes) {
AddUnsigned(WPD.first);
@@ -207,6 +235,8 @@ static void computeCacheKey(
AddUint64(Arg);
AddUnsigned(ByArg.second.TheKind);
AddUint64(ByArg.second.Info);
+ AddUnsigned(ByArg.second.Byte);
+ AddUnsigned(ByArg.second.Bit);
}
}
};
@@ -219,6 +249,14 @@ static void computeCacheKey(
AddTypeIdSummary(Summary->first, Summary->second);
}
+ AddUnsigned(UsedCfiDefs.size());
+ for (auto &V : UsedCfiDefs)
+ AddUint64(V);
+
+ AddUnsigned(UsedCfiDecls.size());
+ for (auto &V : UsedCfiDecls)
+ AddUint64(V);
+
if (!Conf.SampleProfile.empty()) {
auto FileOrErr = MemoryBuffer::getFile(Conf.SampleProfile);
if (FileOrErr)
@@ -347,7 +385,8 @@ StringRef InputFile::getName() const {
LTO::RegularLTOState::RegularLTOState(unsigned ParallelCodeGenParallelismLevel,
Config &Conf)
: ParallelCodeGenParallelismLevel(ParallelCodeGenParallelismLevel),
- Ctx(Conf) {}
+ Ctx(Conf), CombinedModule(llvm::make_unique<Module>("ld-temp.o", Ctx)),
+ Mover(llvm::make_unique<IRMover>(*CombinedModule)) {}
LTO::ThinLTOState::ThinLTOState(ThinBackend Backend) : Backend(Backend) {
if (!Backend)
@@ -378,8 +417,11 @@ void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms,
auto &GlobalRes = GlobalResolutions[Sym.getName()];
GlobalRes.UnnamedAddr &= Sym.isUnnamedAddr();
- if (Res.Prevailing)
+ if (Res.Prevailing) {
+ assert(GlobalRes.IRName.empty() &&
+ "Multiple prevailing defs are not allowed");
GlobalRes.IRName = Sym.getIRName();
+ }
// Set the partition to external if we know it is re-defined by the linker
// with -defsym or -wrap options, used elsewhere, e.g. it is visible to a
@@ -431,6 +473,9 @@ Error LTO::add(std::unique_ptr<InputFile> Input,
if (Conf.ResolutionFile)
writeToResolutionFile(*Conf.ResolutionFile, Input.get(), Res);
+ if (RegularLTO.CombinedModule->getTargetTriple().empty())
+ RegularLTO.CombinedModule->setTargetTriple(Input->getTargetTriple());
+
const SymbolResolution *ResI = Res.begin();
for (unsigned I = 0; I != Input->Mods.size(); ++I)
if (Error Err = addModule(*Input, I, ResI, Res.end()))
@@ -592,6 +637,9 @@ LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
NonPrevailingComdats.insert(GV->getComdat());
cast<GlobalObject>(GV)->setComdat(nullptr);
}
+
+ // Set the 'local' flag based on the linker resolution for this symbol.
+ GV->setDSOLocal(Res.FinalDefinitionInLinkageUnit);
}
// Common resolution: collect the maximum size/alignment over all commons.
// We also record if we see an instance of a common as prevailing, so that
@@ -605,7 +653,6 @@ LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
CommonRes.Prevailing |= Res.Prevailing;
}
- // FIXME: use proposed local attribute for FinalDefinitionInLinkageUnit.
}
if (!M.getComdatSymbolTable().empty())
for (GlobalValue &GV : M.global_values())
@@ -616,12 +663,6 @@ LTO::addRegularLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod,
bool LivenessFromIndex) {
- if (!RegularLTO.CombinedModule) {
- RegularLTO.CombinedModule =
- llvm::make_unique<Module>("ld-temp.o", RegularLTO.Ctx);
- RegularLTO.Mover = llvm::make_unique<IRMover>(*RegularLTO.CombinedModule);
- }
-
std::vector<GlobalValue *> Keep;
for (GlobalValue *GV : Mod.Keep) {
if (LivenessFromIndex && !ThinLTO.CombinedIndex.isGUIDLive(GV->getGUID()))
@@ -660,10 +701,10 @@ Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
assert(ResI != ResE);
SymbolResolution Res = *ResI++;
- if (Res.Prevailing) {
- if (!Sym.getIRName().empty()) {
- auto GUID = GlobalValue::getGUID(GlobalValue::getGlobalIdentifier(
- Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
+ if (!Sym.getIRName().empty()) {
+ auto GUID = GlobalValue::getGUID(GlobalValue::getGlobalIdentifier(
+ Sym.getIRName(), GlobalValue::ExternalLinkage, ""));
+ if (Res.Prevailing) {
ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier();
// For linker redefined symbols (via --wrap or --defsym) we want to
@@ -675,6 +716,15 @@ Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms,
GUID, BM.getModuleIdentifier()))
S->setLinkage(GlobalValue::WeakAnyLinkage);
}
+
+ // If the linker resolved the symbol to a local definition then mark it
+ // as local in the summary for the module we are adding.
+ if (Res.FinalDefinitionInLinkageUnit) {
+ if (auto S = ThinLTO.CombinedIndex.findSummaryInModule(
+ GUID, BM.getModuleIdentifier())) {
+ S->setDSOLocal(true);
+ }
+ }
}
}
@@ -705,16 +755,9 @@ Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {
computeDeadSymbols(ThinLTO.CombinedIndex, GUIDPreservedSymbols);
- // Save the status of having a regularLTO combined module, as
- // this is needed for generating the ThinLTO Task ID, and
- // the CombinedModule will be moved at the end of runRegularLTO.
- bool HasRegularLTO = RegularLTO.CombinedModule != nullptr ||
- !RegularLTO.ModsWithSummaries.empty();
- // Invoke regular LTO if there was a regular LTO module to start with.
- if (HasRegularLTO)
- if (auto E = runRegularLTO(AddStream))
- return E;
- return runThinLTO(AddStream, Cache, HasRegularLTO);
+ if (auto E = runRegularLTO(AddStream))
+ return E;
+ return runThinLTO(AddStream, Cache);
}
Error LTO::runRegularLTO(AddStreamFn AddStream) {
@@ -812,6 +855,8 @@ class InProcessThinBackend : public ThinBackendProc {
AddStreamFn AddStream;
NativeObjectCache Cache;
TypeIdSummariesByGuidTy TypeIdSummariesByGuid;
+ std::set<GlobalValue::GUID> CfiFunctionDefs;
+ std::set<GlobalValue::GUID> CfiFunctionDecls;
Optional<Error> Err;
std::mutex ErrMu;
@@ -831,6 +876,12 @@ public:
// each function without needing to compute GUIDs in each backend.
for (auto &TId : CombinedIndex.typeIds())
TypeIdSummariesByGuid[GlobalValue::getGUID(TId.first)].push_back(&TId);
+ for (auto &Name : CombinedIndex.cfiFunctionDefs())
+ CfiFunctionDefs.insert(
+ GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
+ for (auto &Name : CombinedIndex.cfiFunctionDecls())
+ CfiFunctionDecls.insert(
+ GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Name)));
}
Error runThinLTOBackendThread(
@@ -864,7 +915,8 @@ public:
SmallString<40> Key;
// The module may be cached, this helps handling it.
computeCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList, ExportList,
- ResolvedODR, DefinedGlobals, TypeIdSummariesByGuid);
+ ResolvedODR, DefinedGlobals, TypeIdSummariesByGuid,
+ CfiFunctionDefs, CfiFunctionDecls);
if (AddStreamFn CacheAddStream = Cache(Task, Key))
return RunThinBackend(CacheAddStream);
@@ -1020,8 +1072,7 @@ ThinBackend lto::createWriteIndexesThinBackend(std::string OldPrefix,
};
}
-Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
- bool HasRegularLTO) {
+Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache) {
if (ThinLTO.ModuleMap.empty())
return Error::success();
@@ -1051,36 +1102,45 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
ThinLTO.ModuleMap.size());
StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
- if (Conf.OptLevel > 0) {
+ if (Conf.OptLevel > 0)
ComputeCrossModuleImport(ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
ImportLists, ExportLists);
- std::set<GlobalValue::GUID> ExportedGUIDs;
- for (auto &Res : GlobalResolutions) {
- // First check if the symbol was flagged as having external references.
- if (Res.second.Partition != GlobalResolution::External)
- continue;
- // IRName will be defined if we have seen the prevailing copy of
- // this value. If not, no need to mark as exported from a ThinLTO
- // partition (and we can't get the GUID).
- if (Res.second.IRName.empty())
- continue;
- auto GUID = GlobalValue::getGUID(
- GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
- // Mark exported unless index-based analysis determined it to be dead.
- if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
- ExportedGUIDs.insert(GUID);
- }
-
- auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
- const auto &ExportList = ExportLists.find(ModuleIdentifier);
- return (ExportList != ExportLists.end() &&
- ExportList->second.count(GUID)) ||
- ExportedGUIDs.count(GUID);
- };
- thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported);
+ // Figure out which symbols need to be internalized. This also needs to happen
+ // at -O0 because summary-based DCE is implemented using internalization, and
+ // we must apply DCE consistently with the full LTO module in order to avoid
+ // undefined references during the final link.
+ std::set<GlobalValue::GUID> ExportedGUIDs;
+ for (auto &Res : GlobalResolutions) {
+ // First check if the symbol was flagged as having external references.
+ if (Res.second.Partition != GlobalResolution::External)
+ continue;
+ // IRName will be defined if we have seen the prevailing copy of
+ // this value. If not, no need to mark as exported from a ThinLTO
+ // partition (and we can't get the GUID).
+ if (Res.second.IRName.empty())
+ continue;
+ auto GUID = GlobalValue::getGUID(
+ GlobalValue::dropLLVMManglingEscape(Res.second.IRName));
+ // Mark exported unless index-based analysis determined it to be dead.
+ if (ThinLTO.CombinedIndex.isGUIDLive(GUID))
+ ExportedGUIDs.insert(GUID);
}
+ // Any functions referenced by the jump table in the regular LTO object must
+ // be exported.
+ for (auto &Def : ThinLTO.CombinedIndex.cfiFunctionDefs())
+ ExportedGUIDs.insert(
+ GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(Def)));
+
+ auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
+ const auto &ExportList = ExportLists.find(ModuleIdentifier);
+ return (ExportList != ExportLists.end() &&
+ ExportList->second.count(GUID)) ||
+ ExportedGUIDs.count(GUID);
+ };
+ thinLTOInternalizeAndPromoteInIndex(ThinLTO.CombinedIndex, isExported);
+
auto isPrevailing = [&](GlobalValue::GUID GUID,
const GlobalValueSummary *S) {
return ThinLTO.PrevailingModuleForGUID[GUID] == S->modulePath();
@@ -1097,11 +1157,9 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
ThinLTO.Backend(Conf, ThinLTO.CombinedIndex, ModuleToDefinedGVSummaries,
AddStream, Cache);
- // Task numbers start at ParallelCodeGenParallelismLevel if an LTO
- // module is present, as tasks 0 through ParallelCodeGenParallelismLevel-1
- // are reserved for parallel code generation partitions.
- unsigned Task =
- HasRegularLTO ? RegularLTO.ParallelCodeGenParallelismLevel : 0;
+ // Tasks 0 through ParallelCodeGenParallelismLevel-1 are reserved for combined
+ // module and parallel code generation partitions.
+ unsigned Task = RegularLTO.ParallelCodeGenParallelismLevel;
for (auto &Mod : ThinLTO.ModuleMap) {
if (Error E = BackendProc->start(Task, Mod.second, ImportLists[Mod.first],
ExportLists[Mod.first],
@@ -1113,7 +1171,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
return BackendProc->wait();
}
-Expected<std::unique_ptr<tool_output_file>>
+Expected<std::unique_ptr<ToolOutputFile>>
lto::setupOptimizationRemarks(LLVMContext &Context,
StringRef LTORemarksFilename,
bool LTOPassRemarksWithHotness, int Count) {
@@ -1126,7 +1184,7 @@ lto::setupOptimizationRemarks(LLVMContext &Context,
std::error_code EC;
auto DiagnosticFile =
- llvm::make_unique<tool_output_file>(Filename, EC, sys::fs::F_None);
+ llvm::make_unique<ToolOutputFile>(Filename, EC, sys::fs::F_None);
if (EC)
return errorCodeToError(EC);
Context.setDiagnosticsOutputFile(
generated by cgit v1.2.3 (git 2.25.1) at 2025-11-03 07:01:42 +0000