aboutsummaryrefslogtreecommitdiff
path: root/lib/LTO/ThinLTOCodeGenerator.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/LTO/ThinLTOCodeGenerator.cpp')
-rw-r--r--lib/LTO/ThinLTOCodeGenerator.cpp139
1 files changed, 97 insertions, 42 deletions
diff --git a/lib/LTO/ThinLTOCodeGenerator.cpp b/lib/LTO/ThinLTOCodeGenerator.cpp
index 1c52218836ca..d151de17896f 100644
--- a/lib/LTO/ThinLTOCodeGenerator.cpp
+++ b/lib/LTO/ThinLTOCodeGenerator.cpp
@@ -39,6 +39,7 @@
#include "llvm/Support/CachePruning.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/Error.h"
+#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/SHA1.h"
#include "llvm/Support/SmallVectorMemoryBuffer.h"
@@ -52,6 +53,7 @@
#include "llvm/Transforms/IPO/FunctionImport.h"
#include "llvm/Transforms/IPO/Internalize.h"
#include "llvm/Transforms/IPO/PassManagerBuilder.h"
+#include "llvm/Transforms/IPO/WholeProgramDevirt.h"
#include "llvm/Transforms/ObjCARC.h"
#include "llvm/Transforms/Utils/FunctionImportUtils.h"
@@ -89,7 +91,7 @@ static void saveTempBitcode(const Module &TheModule, StringRef TempDir,
// User asked to save temps, let dump the bitcode file after import.
std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str();
std::error_code EC;
- raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None);
+ raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
if (EC)
report_fatal_error(Twine("Failed to open ") + SaveTempPath +
" to save optimized bitcode\n");
@@ -224,7 +226,8 @@ crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index,
}
static void optimizeModule(Module &TheModule, TargetMachine &TM,
- unsigned OptLevel, bool Freestanding) {
+ unsigned OptLevel, bool Freestanding,
+ ModuleSummaryIndex *Index) {
// Populate the PassManager
PassManagerBuilder PMB;
PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple());
@@ -238,6 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM,
// Already did this in verifyLoadedModule().
PMB.VerifyInput = false;
PMB.VerifyOutput = false;
+ PMB.ImportSummary = Index;
legacy::PassManager PM;
@@ -295,7 +299,7 @@ std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule,
// Run codegen now. resulting binary is in OutputBuffer.
PM.run(TheModule);
}
- return make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
+ return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
}
/// Manage caching for a single Module.
@@ -368,23 +372,26 @@ public:
// Write to a temporary to avoid race condition
SmallString<128> TempFilename;
SmallString<128> CachePath(EntryPath);
- int TempFD;
llvm::sys::path::remove_filename(CachePath);
sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o");
- std::error_code EC =
- sys::fs::createUniqueFile(TempFilename, TempFD, TempFilename);
- if (EC) {
- errs() << "Error: " << EC.message() << "\n";
- report_fatal_error("ThinLTO: Can't get a temporary file");
- }
- {
- raw_fd_ostream OS(TempFD, /* ShouldClose */ true);
- OS << OutputBuffer.getBuffer();
+
+ if (auto Err = handleErrors(
+ llvm::writeFileAtomically(TempFilename, EntryPath,
+ OutputBuffer.getBuffer()),
+ [](const llvm::AtomicFileWriteError &E) {
+ std::string ErrorMsgBuffer;
+ llvm::raw_string_ostream S(ErrorMsgBuffer);
+ E.log(S);
+
+ if (E.Error ==
+ llvm::atomic_write_error::failed_to_create_uniq_file) {
+ errs() << "Error: " << ErrorMsgBuffer << "\n";
+ report_fatal_error("ThinLTO: Can't get a temporary file");
+ }
+ })) {
+ // FIXME
+ consumeError(std::move(Err));
}
- // Rename temp file to final destination; rename is atomic
- EC = sys::fs::rename(TempFilename, EntryPath);
- if (EC)
- sys::fs::remove(TempFilename);
}
};
@@ -429,7 +436,7 @@ ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc");
}
- optimizeModule(TheModule, TM, OptLevel, Freestanding);
+ optimizeModule(TheModule, TM, OptLevel, Freestanding, &Index);
saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc");
@@ -442,7 +449,7 @@ ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index,
auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI);
WriteBitcodeToFile(TheModule, OS, true, &Index);
}
- return make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
+ return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer));
}
return codegenModule(TheModule, TM);
@@ -457,10 +464,9 @@ static void resolvePrevailingInIndex(
ModuleSummaryIndex &Index,
StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>>
&ResolvedODR,
- const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
-
- DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
- computePrevailingCopies(Index, PrevailingCopy);
+ const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
+ const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
+ &PrevailingCopy) {
auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) {
const auto &Prevailing = PrevailingCopy.find(GUID);
@@ -490,7 +496,8 @@ static void initTMBuilder(TargetMachineBuilder &TMBuilder,
TMBuilder.MCpu = "core2";
else if (TheTriple.getArch() == llvm::Triple::x86)
TMBuilder.MCpu = "yonah";
- else if (TheTriple.getArch() == llvm::Triple::aarch64)
+ else if (TheTriple.getArch() == llvm::Triple::aarch64 ||
+ TheTriple.getArch() == llvm::Triple::aarch64_32)
TMBuilder.MCpu = "cyclone";
}
TMBuilder.TheTriple = std::move(TheTriple);
@@ -557,7 +564,7 @@ std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const {
*/
std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
std::unique_ptr<ModuleSummaryIndex> CombinedIndex =
- llvm::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
+ std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false);
uint64_t NextModuleId = 0;
for (auto &Mod : Modules) {
auto &M = Mod->getSingleBitcodeModule();
@@ -573,19 +580,36 @@ std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() {
return CombinedIndex;
}
-static void internalizeAndPromoteInIndex(
- const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
- const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols,
- ModuleSummaryIndex &Index) {
- auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) {
+struct IsExported {
+ const StringMap<FunctionImporter::ExportSetTy> &ExportLists;
+ const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols;
+
+ IsExported(const StringMap<FunctionImporter::ExportSetTy> &ExportLists,
+ const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols)
+ : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {}
+
+ bool operator()(StringRef ModuleIdentifier, GlobalValue::GUID GUID) const {
const auto &ExportList = ExportLists.find(ModuleIdentifier);
return (ExportList != ExportLists.end() &&
ExportList->second.count(GUID)) ||
GUIDPreservedSymbols.count(GUID);
- };
+ }
+};
- thinLTOInternalizeAndPromoteInIndex(Index, isExported);
-}
+struct IsPrevailing {
+ const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy;
+ IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *>
+ &PrevailingCopy)
+ : PrevailingCopy(PrevailingCopy) {}
+
+ bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const {
+ const auto &Prevailing = PrevailingCopy.find(GUID);
+ // Not in map means that there was only one copy, which must be prevailing.
+ if (Prevailing == PrevailingCopy.end())
+ return true;
+ return Prevailing->second == S;
+ };
+};
static void computeDeadSymbolsInIndex(
ModuleSummaryIndex &Index,
@@ -629,16 +653,22 @@ void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index,
ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists,
ExportLists);
+ DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
+ computePrevailingCopies(Index, PrevailingCopy);
+
// Resolve prevailing symbols
StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
- resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols);
+ resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
+ PrevailingCopy);
thinLTOResolvePrevailingInModule(
TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]);
// Promote the exported values in the index, so that they are promoted
// in the module.
- internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, Index);
+ thinLTOInternalizeAndPromoteInIndex(
+ Index, IsExported(ExportLists, GUIDPreservedSymbols),
+ IsPrevailing(PrevailingCopy));
promoteModule(TheModule, Index);
}
@@ -785,13 +815,19 @@ void ThinLTOCodeGenerator::internalize(Module &TheModule,
if (ExportList.empty() && GUIDPreservedSymbols.empty())
return;
+ DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
+ computePrevailingCopies(Index, PrevailingCopy);
+
// Resolve prevailing symbols
StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
- resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols);
+ resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols,
+ PrevailingCopy);
// Promote the exported values in the index, so that they are promoted
// in the module.
- internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, Index);
+ thinLTOInternalizeAndPromoteInIndex(
+ Index, IsExported(ExportLists, GUIDPreservedSymbols),
+ IsPrevailing(PrevailingCopy));
promoteModule(TheModule, Index);
@@ -810,7 +846,8 @@ void ThinLTOCodeGenerator::optimize(Module &TheModule) {
initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple()));
// Optimize now
- optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding);
+ optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding,
+ nullptr);
}
/// Write out the generated object file, either from CacheEntryPath or from
@@ -845,7 +882,7 @@ ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath,
}
// No cache entry, just write out the buffer.
std::error_code Err;
- raw_fd_ostream OS(OutputPath, Err, sys::fs::F_None);
+ raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None);
if (Err)
report_fatal_error("Can't open output '" + OutputPath + "'\n");
OS << OutputBuffer.getBuffer();
@@ -900,7 +937,7 @@ void ThinLTOCodeGenerator::run() {
if (!SaveTempsDir.empty()) {
auto SaveTempPath = SaveTempsDir + "index.bc";
std::error_code EC;
- raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None);
+ raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None);
if (EC)
report_fatal_error(Twine("Failed to open ") + SaveTempPath +
" to save optimized bitcode\n");
@@ -931,6 +968,15 @@ void ThinLTOCodeGenerator::run() {
// Synthesize entry counts for functions in the combined index.
computeSyntheticCounts(*Index);
+ // Perform index-based WPD. This will return immediately if there are
+ // no index entries in the typeIdMetadata map (e.g. if we are instead
+ // performing IR-based WPD in hybrid regular/thin LTO mode).
+ std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap;
+ std::set<GlobalValue::GUID> ExportedGUIDs;
+ runWholeProgramDevirtOnIndex(*Index, ExportedGUIDs, LocalWPDTargetsMap);
+ for (auto GUID : ExportedGUIDs)
+ GUIDPreservedSymbols.insert(GUID);
+
// Collect the import/export lists for all modules from the call-graph in the
// combined index.
StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount);
@@ -944,14 +990,23 @@ void ThinLTOCodeGenerator::run() {
// on the index, and nuke this map.
StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR;
+ DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy;
+ computePrevailingCopies(*Index, PrevailingCopy);
+
// Resolve prevailing symbols, this has to be computed early because it
// impacts the caching.
- resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols);
+ resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols,
+ PrevailingCopy);
// Use global summary-based analysis to identify symbols that can be
// internalized (because they aren't exported or preserved as per callback).
// Changes are made in the index, consumed in the ThinLTO backends.
- internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, *Index);
+ updateIndexWPDForExports(*Index,
+ IsExported(ExportLists, GUIDPreservedSymbols),
+ LocalWPDTargetsMap);
+ thinLTOInternalizeAndPromoteInIndex(
+ *Index, IsExported(ExportLists, GUIDPreservedSymbols),
+ IsPrevailing(PrevailingCopy));
// Make sure that every module has an entry in the ExportLists, ImportList,
// GVSummary and ResolvedODR maps to enable threaded access to these maps
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-31 22:29:26 +0000