diff options
Diffstat (limited to 'lib/LTO/ThinLTOCodeGenerator.cpp')
-rw-r--r-- | lib/LTO/ThinLTOCodeGenerator.cpp | 139 |
1 files changed, 97 insertions, 42 deletions
diff --git a/lib/LTO/ThinLTOCodeGenerator.cpp b/lib/LTO/ThinLTOCodeGenerator.cpp index 1c52218836ca..d151de17896f 100644 --- a/lib/LTO/ThinLTOCodeGenerator.cpp +++ b/lib/LTO/ThinLTOCodeGenerator.cpp @@ -39,6 +39,7 @@ #include "llvm/Support/CachePruning.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Error.h" +#include "llvm/Support/FileUtilities.h" #include "llvm/Support/Path.h" #include "llvm/Support/SHA1.h" #include "llvm/Support/SmallVectorMemoryBuffer.h" @@ -52,6 +53,7 @@ #include "llvm/Transforms/IPO/FunctionImport.h" #include "llvm/Transforms/IPO/Internalize.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#include "llvm/Transforms/IPO/WholeProgramDevirt.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" @@ -89,7 +91,7 @@ static void saveTempBitcode(const Module &TheModule, StringRef TempDir, // User asked to save temps, let dump the bitcode file after import. std::string SaveTempPath = (TempDir + llvm::Twine(count) + Suffix).str(); std::error_code EC; - raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None); + raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None); if (EC) report_fatal_error(Twine("Failed to open ") + SaveTempPath + " to save optimized bitcode\n"); @@ -224,7 +226,8 @@ crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index, } static void optimizeModule(Module &TheModule, TargetMachine &TM, - unsigned OptLevel, bool Freestanding) { + unsigned OptLevel, bool Freestanding, + ModuleSummaryIndex *Index) { // Populate the PassManager PassManagerBuilder PMB; PMB.LibraryInfo = new TargetLibraryInfoImpl(TM.getTargetTriple()); @@ -238,6 +241,7 @@ static void optimizeModule(Module &TheModule, TargetMachine &TM, // Already did this in verifyLoadedModule(). PMB.VerifyInput = false; PMB.VerifyOutput = false; + PMB.ImportSummary = Index; legacy::PassManager PM; @@ -295,7 +299,7 @@ std::unique_ptr<MemoryBuffer> codegenModule(Module &TheModule, // Run codegen now. resulting binary is in OutputBuffer. PM.run(TheModule); } - return make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer)); + return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer)); } /// Manage caching for a single Module. @@ -368,23 +372,26 @@ public: // Write to a temporary to avoid race condition SmallString<128> TempFilename; SmallString<128> CachePath(EntryPath); - int TempFD; llvm::sys::path::remove_filename(CachePath); sys::path::append(TempFilename, CachePath, "Thin-%%%%%%.tmp.o"); - std::error_code EC = - sys::fs::createUniqueFile(TempFilename, TempFD, TempFilename); - if (EC) { - errs() << "Error: " << EC.message() << "\n"; - report_fatal_error("ThinLTO: Can't get a temporary file"); - } - { - raw_fd_ostream OS(TempFD, /* ShouldClose */ true); - OS << OutputBuffer.getBuffer(); + + if (auto Err = handleErrors( + llvm::writeFileAtomically(TempFilename, EntryPath, + OutputBuffer.getBuffer()), + [](const llvm::AtomicFileWriteError &E) { + std::string ErrorMsgBuffer; + llvm::raw_string_ostream S(ErrorMsgBuffer); + E.log(S); + + if (E.Error == + llvm::atomic_write_error::failed_to_create_uniq_file) { + errs() << "Error: " << ErrorMsgBuffer << "\n"; + report_fatal_error("ThinLTO: Can't get a temporary file"); + } + })) { + // FIXME + consumeError(std::move(Err)); } - // Rename temp file to final destination; rename is atomic - EC = sys::fs::rename(TempFilename, EntryPath); - if (EC) - sys::fs::remove(TempFilename); } }; @@ -429,7 +436,7 @@ ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index, saveTempBitcode(TheModule, SaveTempsDir, count, ".3.imported.bc"); } - optimizeModule(TheModule, TM, OptLevel, Freestanding); + optimizeModule(TheModule, TM, OptLevel, Freestanding, &Index); saveTempBitcode(TheModule, SaveTempsDir, count, ".4.opt.bc"); @@ -442,7 +449,7 @@ ProcessThinLTOModule(Module &TheModule, ModuleSummaryIndex &Index, auto Index = buildModuleSummaryIndex(TheModule, nullptr, &PSI); WriteBitcodeToFile(TheModule, OS, true, &Index); } - return make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer)); + return std::make_unique<SmallVectorMemoryBuffer>(std::move(OutputBuffer)); } return codegenModule(TheModule, TM); @@ -457,10 +464,9 @@ static void resolvePrevailingInIndex( ModuleSummaryIndex &Index, StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> &ResolvedODR, - const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) { - - DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; - computePrevailingCopies(Index, PrevailingCopy); + const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols, + const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> + &PrevailingCopy) { auto isPrevailing = [&](GlobalValue::GUID GUID, const GlobalValueSummary *S) { const auto &Prevailing = PrevailingCopy.find(GUID); @@ -490,7 +496,8 @@ static void initTMBuilder(TargetMachineBuilder &TMBuilder, TMBuilder.MCpu = "core2"; else if (TheTriple.getArch() == llvm::Triple::x86) TMBuilder.MCpu = "yonah"; - else if (TheTriple.getArch() == llvm::Triple::aarch64) + else if (TheTriple.getArch() == llvm::Triple::aarch64 || + TheTriple.getArch() == llvm::Triple::aarch64_32) TMBuilder.MCpu = "cyclone"; } TMBuilder.TheTriple = std::move(TheTriple); @@ -557,7 +564,7 @@ std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const { */ std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() { std::unique_ptr<ModuleSummaryIndex> CombinedIndex = - llvm::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false); + std::make_unique<ModuleSummaryIndex>(/*HaveGVs=*/false); uint64_t NextModuleId = 0; for (auto &Mod : Modules) { auto &M = Mod->getSingleBitcodeModule(); @@ -573,19 +580,36 @@ std::unique_ptr<ModuleSummaryIndex> ThinLTOCodeGenerator::linkCombinedIndex() { return CombinedIndex; } -static void internalizeAndPromoteInIndex( - const StringMap<FunctionImporter::ExportSetTy> &ExportLists, - const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols, - ModuleSummaryIndex &Index) { - auto isExported = [&](StringRef ModuleIdentifier, GlobalValue::GUID GUID) { +struct IsExported { + const StringMap<FunctionImporter::ExportSetTy> &ExportLists; + const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols; + + IsExported(const StringMap<FunctionImporter::ExportSetTy> &ExportLists, + const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) + : ExportLists(ExportLists), GUIDPreservedSymbols(GUIDPreservedSymbols) {} + + bool operator()(StringRef ModuleIdentifier, GlobalValue::GUID GUID) const { const auto &ExportList = ExportLists.find(ModuleIdentifier); return (ExportList != ExportLists.end() && ExportList->second.count(GUID)) || GUIDPreservedSymbols.count(GUID); - }; + } +}; - thinLTOInternalizeAndPromoteInIndex(Index, isExported); -} +struct IsPrevailing { + const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> &PrevailingCopy; + IsPrevailing(const DenseMap<GlobalValue::GUID, const GlobalValueSummary *> + &PrevailingCopy) + : PrevailingCopy(PrevailingCopy) {} + + bool operator()(GlobalValue::GUID GUID, const GlobalValueSummary *S) const { + const auto &Prevailing = PrevailingCopy.find(GUID); + // Not in map means that there was only one copy, which must be prevailing. + if (Prevailing == PrevailingCopy.end()) + return true; + return Prevailing->second == S; + }; +}; static void computeDeadSymbolsInIndex( ModuleSummaryIndex &Index, @@ -629,16 +653,22 @@ void ThinLTOCodeGenerator::promote(Module &TheModule, ModuleSummaryIndex &Index, ComputeCrossModuleImport(Index, ModuleToDefinedGVSummaries, ImportLists, ExportLists); + DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; + computePrevailingCopies(Index, PrevailingCopy); + // Resolve prevailing symbols StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; - resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols); + resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols, + PrevailingCopy); thinLTOResolvePrevailingInModule( TheModule, ModuleToDefinedGVSummaries[ModuleIdentifier]); // Promote the exported values in the index, so that they are promoted // in the module. - internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, Index); + thinLTOInternalizeAndPromoteInIndex( + Index, IsExported(ExportLists, GUIDPreservedSymbols), + IsPrevailing(PrevailingCopy)); promoteModule(TheModule, Index); } @@ -785,13 +815,19 @@ void ThinLTOCodeGenerator::internalize(Module &TheModule, if (ExportList.empty() && GUIDPreservedSymbols.empty()) return; + DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; + computePrevailingCopies(Index, PrevailingCopy); + // Resolve prevailing symbols StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; - resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols); + resolvePrevailingInIndex(Index, ResolvedODR, GUIDPreservedSymbols, + PrevailingCopy); // Promote the exported values in the index, so that they are promoted // in the module. - internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, Index); + thinLTOInternalizeAndPromoteInIndex( + Index, IsExported(ExportLists, GUIDPreservedSymbols), + IsPrevailing(PrevailingCopy)); promoteModule(TheModule, Index); @@ -810,7 +846,8 @@ void ThinLTOCodeGenerator::optimize(Module &TheModule) { initTMBuilder(TMBuilder, Triple(TheModule.getTargetTriple())); // Optimize now - optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding); + optimizeModule(TheModule, *TMBuilder.create(), OptLevel, Freestanding, + nullptr); } /// Write out the generated object file, either from CacheEntryPath or from @@ -845,7 +882,7 @@ ThinLTOCodeGenerator::writeGeneratedObject(int count, StringRef CacheEntryPath, } // No cache entry, just write out the buffer. std::error_code Err; - raw_fd_ostream OS(OutputPath, Err, sys::fs::F_None); + raw_fd_ostream OS(OutputPath, Err, sys::fs::OF_None); if (Err) report_fatal_error("Can't open output '" + OutputPath + "'\n"); OS << OutputBuffer.getBuffer(); @@ -900,7 +937,7 @@ void ThinLTOCodeGenerator::run() { if (!SaveTempsDir.empty()) { auto SaveTempPath = SaveTempsDir + "index.bc"; std::error_code EC; - raw_fd_ostream OS(SaveTempPath, EC, sys::fs::F_None); + raw_fd_ostream OS(SaveTempPath, EC, sys::fs::OF_None); if (EC) report_fatal_error(Twine("Failed to open ") + SaveTempPath + " to save optimized bitcode\n"); @@ -931,6 +968,15 @@ void ThinLTOCodeGenerator::run() { // Synthesize entry counts for functions in the combined index. computeSyntheticCounts(*Index); + // Perform index-based WPD. This will return immediately if there are + // no index entries in the typeIdMetadata map (e.g. if we are instead + // performing IR-based WPD in hybrid regular/thin LTO mode). + std::map<ValueInfo, std::vector<VTableSlotSummary>> LocalWPDTargetsMap; + std::set<GlobalValue::GUID> ExportedGUIDs; + runWholeProgramDevirtOnIndex(*Index, ExportedGUIDs, LocalWPDTargetsMap); + for (auto GUID : ExportedGUIDs) + GUIDPreservedSymbols.insert(GUID); + // Collect the import/export lists for all modules from the call-graph in the // combined index. StringMap<FunctionImporter::ImportMapTy> ImportLists(ModuleCount); @@ -944,14 +990,23 @@ void ThinLTOCodeGenerator::run() { // on the index, and nuke this map. StringMap<std::map<GlobalValue::GUID, GlobalValue::LinkageTypes>> ResolvedODR; + DenseMap<GlobalValue::GUID, const GlobalValueSummary *> PrevailingCopy; + computePrevailingCopies(*Index, PrevailingCopy); + // Resolve prevailing symbols, this has to be computed early because it // impacts the caching. - resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols); + resolvePrevailingInIndex(*Index, ResolvedODR, GUIDPreservedSymbols, + PrevailingCopy); // Use global summary-based analysis to identify symbols that can be // internalized (because they aren't exported or preserved as per callback). // Changes are made in the index, consumed in the ThinLTO backends. - internalizeAndPromoteInIndex(ExportLists, GUIDPreservedSymbols, *Index); + updateIndexWPDForExports(*Index, + IsExported(ExportLists, GUIDPreservedSymbols), + LocalWPDTargetsMap); + thinLTOInternalizeAndPromoteInIndex( + *Index, IsExported(ExportLists, GUIDPreservedSymbols), + IsPrevailing(PrevailingCopy)); // Make sure that every module has an entry in the ExportLists, ImportList, // GVSummary and ResolvedODR maps to enable threaded access to these maps |