diff options
Diffstat (limited to 'llvm/lib/LTO')
| -rw-r--r-- | llvm/lib/LTO/LTO.cpp | 63 | ||||
| -rw-r--r-- | llvm/lib/LTO/LTOBackend.cpp | 12 | ||||
| -rw-r--r-- | llvm/lib/LTO/LTOCodeGenerator.cpp | 126 | ||||
| -rw-r--r-- | llvm/lib/LTO/LTOModule.cpp | 4 | ||||
| -rw-r--r-- | llvm/lib/LTO/ThinLTOCodeGenerator.cpp | 10 |
5 files changed, 168 insertions, 47 deletions
diff --git a/llvm/lib/LTO/LTO.cpp b/llvm/lib/LTO/LTO.cpp index cc7be24c1dbd..1cd48adac3f0 100644 --- a/llvm/lib/LTO/LTO.cpp +++ b/llvm/lib/LTO/LTO.cpp @@ -49,14 +49,13 @@ #include "llvm/Support/ToolOutputFile.h" #include "llvm/Support/VCSRevision.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/IPO.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/IPO/WholeProgramDevirt.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" #include "llvm/Transforms/Utils/SplitModule.h" +#include <optional> #include <set> using namespace llvm; @@ -69,10 +68,12 @@ static cl::opt<bool> DumpThinCGSCCs("dump-thin-cg-sccs", cl::init(false), cl::Hidden, cl::desc("Dump the SCCs in the ThinLTO index's callgraph")); +namespace llvm { /// Enable global value internalization in LTO. cl::opt<bool> EnableLTOInternalization( "enable-lto-internalization", cl::init(true), cl::Hidden, cl::desc("Enable global value internalization in LTO")); +} // Computes a unique hash for the Module considering the current list of // export/import and other global analysis results. @@ -131,6 +132,8 @@ void llvm::computeLTOCacheKey( AddUnsigned(*Conf.CodeModel); else AddUnsigned(-1); + for (const auto &S : Conf.MllvmArgs) + AddString(S); AddUnsigned(Conf.CGOptLevel); AddUnsigned(Conf.CGFileType); AddUnsigned(Conf.OptLevel); @@ -565,6 +568,22 @@ void LTO::addModuleToGlobalRes(ArrayRef<InputFile::Symbol> Syms, GlobalRes.IRName = std::string(Sym.getIRName()); } + // In rare occasion, the symbol used to initialize GlobalRes has a different + // IRName from the inspected Symbol. This can happen on macOS + iOS, when a + // symbol is referenced through its mangled name, say @"\01_symbol" while + // the IRName is @symbol (the prefix underscore comes from MachO mangling). + // In that case, we have the same actual Symbol that can get two different + // GUID, leading to some invalid internalization. Workaround this by marking + // the GlobalRes external. + + // FIXME: instead of this check, it would be desirable to compute GUIDs + // based on mangled name, but this requires an access to the Target Triple + // and would be relatively invasive on the codebase. + if (GlobalRes.IRName != Sym.getIRName()) { + GlobalRes.Partition = GlobalResolution::External; + GlobalRes.VisibleOutsideSummary = true; + } + // Set the partition to external if we know it is re-defined by the linker // with -defsym or -wrap options, used elsewhere, e.g. it is visible to a // regular object, is referenced from llvm.compiler.used/llvm.used, or was @@ -694,11 +713,11 @@ handleNonPrevailingComdat(GlobalValue &GV, if (!NonPrevailingComdats.count(C)) return; - // Additionally need to drop externally visible global values from the comdat - // to available_externally, so that there aren't multiply defined linker - // errors. - if (!GV.hasLocalLinkage()) - GV.setLinkage(GlobalValue::AvailableExternallyLinkage); + // Additionally need to drop all global values from the comdat to + // available_externally, to satisfy the COMDAT requirement that all members + // are discarded as a unit. The non-local linkage global values avoid + // duplicate definition linker errors. + GV.setLinkage(GlobalValue::AvailableExternallyLinkage); if (auto GO = dyn_cast<GlobalObject>(&GV)) GO->setComdat(nullptr); @@ -893,9 +912,25 @@ Error LTO::linkRegularLTO(RegularLTOState::AddedModule Mod, Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms, const SymbolResolution *&ResI, const SymbolResolution *ResE) { + const SymbolResolution *ResITmp = ResI; + for (const InputFile::Symbol &Sym : Syms) { + assert(ResITmp != ResE); + SymbolResolution Res = *ResITmp++; + + if (!Sym.getIRName().empty()) { + auto GUID = GlobalValue::getGUID(GlobalValue::getGlobalIdentifier( + Sym.getIRName(), GlobalValue::ExternalLinkage, "")); + if (Res.Prevailing) + ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier(); + } + } + if (Error Err = BM.readSummary(ThinLTO.CombinedIndex, BM.getModuleIdentifier(), - ThinLTO.ModuleMap.size())) + ThinLTO.ModuleMap.size(), [&](GlobalValue::GUID GUID) { + return ThinLTO.PrevailingModuleForGUID[GUID] == + BM.getModuleIdentifier(); + })) return Err; for (const InputFile::Symbol &Sym : Syms) { @@ -906,7 +941,8 @@ Error LTO::addThinLTO(BitcodeModule BM, ArrayRef<InputFile::Symbol> Syms, auto GUID = GlobalValue::getGUID(GlobalValue::getGlobalIdentifier( Sym.getIRName(), GlobalValue::ExternalLinkage, "")); if (Res.Prevailing) { - ThinLTO.PrevailingModuleForGUID[GUID] = BM.getModuleIdentifier(); + assert(ThinLTO.PrevailingModuleForGUID[GUID] == + BM.getModuleIdentifier()); // For linker redefined symbols (via --wrap or --defsym) we want to // switch the linkage to `weak` to prevent IPOs from happening. @@ -1154,7 +1190,7 @@ static const char *libcallRoutineNames[] = { }; ArrayRef<const char*> LTO::getRuntimeLibcallSymbols() { - return makeArrayRef(libcallRoutineNames); + return ArrayRef(libcallRoutineNames); } /// This class defines the interface to the ThinLTO backend. @@ -1217,7 +1253,7 @@ class InProcessThinBackend : public ThinBackendProc { std::set<GlobalValue::GUID> CfiFunctionDefs; std::set<GlobalValue::GUID> CfiFunctionDecls; - Optional<Error> Err; + std::optional<Error> Err; std::mutex ErrMu; bool ShouldEmitIndexFiles; @@ -1278,7 +1314,7 @@ public: computeLTOCacheKey(Key, Conf, CombinedIndex, ModuleID, ImportList, ExportList, ResolvedODR, DefinedGlobals, CfiFunctionDefs, CfiFunctionDecls); - Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key); + Expected<AddStreamFn> CacheAddStreamOrErr = Cache(Task, Key, ModuleID); if (Error Err = CacheAddStreamOrErr.takeError()) return Err; AddStreamFn &CacheAddStream = *CacheAddStreamOrErr; @@ -1436,6 +1472,7 @@ ThinBackend lto::createWriteIndexesThinBackend( Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) { + ThinLTO.CombinedIndex.releaseTemporaryMemory(); timeTraceProfilerBegin("ThinLink", StringRef("")); auto TimeTraceScopeExit = llvm::make_scope_exit([]() { if (llvm::timeTraceProfilerEnabled()) @@ -1608,7 +1645,7 @@ Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache, Expected<std::unique_ptr<ToolOutputFile>> lto::setupLLVMOptimizationRemarks( LLVMContext &Context, StringRef RemarksFilename, StringRef RemarksPasses, StringRef RemarksFormat, bool RemarksWithHotness, - Optional<uint64_t> RemarksHotnessThreshold, int Count) { + std::optional<uint64_t> RemarksHotnessThreshold, int Count) { std::string Filename = std::string(RemarksFilename); // For ThinLTO, file.opt.<format> becomes // file.opt.<format>.thin.<num>.<format>. diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp index 2e32469b4926..1c2ca253af35 100644 --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -44,6 +44,7 @@ #include "llvm/Transforms/Scalar/LoopPassManager.h" #include "llvm/Transforms/Utils/FunctionImportUtils.h" #include "llvm/Transforms/Utils/SplitModule.h" +#include <optional> using namespace llvm; using namespace lto; @@ -207,14 +208,14 @@ createTargetMachine(const Config &Conf, const Target *TheTarget, Module &M) { for (const std::string &A : Conf.MAttrs) Features.AddFeature(A); - Optional<Reloc::Model> RelocModel = None; + std::optional<Reloc::Model> RelocModel; if (Conf.RelocModel) RelocModel = *Conf.RelocModel; else if (M.getModuleFlag("PIC Level")) RelocModel = M.getPICLevel() == PICLevel::NotPIC ? Reloc::Static : Reloc::PIC_; - Optional<CodeModel::Model> CodeModel; + std::optional<CodeModel::Model> CodeModel; if (Conf.CodeModel) CodeModel = *Conf.CodeModel; else @@ -231,7 +232,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM, unsigned OptLevel, bool IsThinLTO, ModuleSummaryIndex *ExportSummary, const ModuleSummaryIndex *ImportSummary) { - Optional<PGOOptions> PGOOpt; + std::optional<PGOOptions> PGOOpt; if (!Conf.SampleProfile.empty()) PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping, PGOOptions::SampleUse, PGOOptions::NoCSAction, true); @@ -256,7 +257,7 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM, ModuleAnalysisManager MAM; PassInstrumentationCallbacks PIC; - StandardInstrumentations SI(Conf.DebugPassManager); + StandardInstrumentations SI(Mod.getContext(), Conf.DebugPassManager); SI.registerCallbacks(PIC, &FAM); PassBuilder PB(TM, Conf.PTO, PGOOpt, &PIC); @@ -391,7 +392,8 @@ static void codegen(const Config &Conf, TargetMachine *TM, EC.message()); } - Expected<std::unique_ptr<CachedFileStream>> StreamOrErr = AddStream(Task); + Expected<std::unique_ptr<CachedFileStream>> StreamOrErr = + AddStream(Task, Mod.getModuleIdentifier()); if (Error Err = StreamOrErr.takeError()) report_fatal_error(std::move(Err)); std::unique_ptr<CachedFileStream> &Stream = *StreamOrErr; diff --git a/llvm/lib/LTO/LTOCodeGenerator.cpp b/llvm/lib/LTO/LTOCodeGenerator.cpp index 2f7c485b9fc8..ae7b7e4b5481 100644 --- a/llvm/lib/LTO/LTOCodeGenerator.cpp +++ b/llvm/lib/LTO/LTOCodeGenerator.cpp @@ -19,6 +19,7 @@ #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Bitcode/BitcodeWriter.h" +#include "llvm/CodeGen/CommandFlags.h" #include "llvm/CodeGen/ParallelCG.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/Config/config.h" @@ -35,7 +36,6 @@ #include "llvm/IR/Module.h" #include "llvm/IR/PassTimingInfo.h" #include "llvm/IR/Verifier.h" -#include "llvm/InitializePasses.h" #include "llvm/LTO/LTO.h" #include "llvm/LTO/LTOBackend.h" #include "llvm/LTO/legacy/LTOModule.h" @@ -50,6 +50,7 @@ #include "llvm/Support/FileSystem.h" #include "llvm/Support/Host.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Process.h" #include "llvm/Support/Signals.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/ToolOutputFile.h" @@ -58,10 +59,10 @@ #include "llvm/Target/TargetOptions.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/Internalize.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Transforms/IPO/WholeProgramDevirt.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Utils/ModuleUtils.h" +#include <optional> #include <system_error> using namespace llvm; @@ -85,7 +86,7 @@ cl::opt<bool> RemarksWithHotness( cl::desc("With PGO, include profile count in optimization remarks"), cl::Hidden); -cl::opt<Optional<uint64_t>, false, remarks::HotnessThresholdParser> +cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser> RemarksHotnessThreshold( "lto-pass-remarks-hotness-threshold", cl::desc("Minimum profile count required for an " @@ -113,7 +114,20 @@ cl::opt<std::string> LTOStatsFile( "lto-stats-file", cl::desc("Save statistics to the specified file"), cl::Hidden); -} + +cl::opt<std::string> AIXSystemAssemblerPath( + "lto-aix-system-assembler", + cl::desc("Path to a system assembler, picked up on AIX only"), + cl::value_desc("path")); + +cl::opt<bool> + LTORunCSIRInstr("cs-profile-generate", + cl::desc("Perform context sensitive PGO instrumentation")); + +cl::opt<std::string> + LTOCSIRProfile("cs-profile-path", + cl::desc("Context sensitive profile file path")); +} // namespace llvm LTOCodeGenerator::LTOCodeGenerator(LLVMContext &Context) : Context(Context), MergedModule(new Module("ld-temp.o", Context)), @@ -121,11 +135,14 @@ LTOCodeGenerator::LTOCodeGenerator(LLVMContext &Context) Context.setDiscardValueNames(LTODiscardValueNames); Context.enableDebugTypeODRUniquing(); - Config.CodeModel = None; + Config.CodeModel = std::nullopt; Config.StatsFile = LTOStatsFile; Config.PreCodeGenPassesHook = [](legacy::PassManager &PM) { PM.add(createObjCARCContractPass()); }; + + Config.RunCSIRInstr = LTORunCSIRInstr; + Config.CSIRProfile = LTOCSIRProfile; } LTOCodeGenerator::~LTOCodeGenerator() = default; @@ -183,21 +200,10 @@ void LTOCodeGenerator::setOptLevel(unsigned Level) { Config.OptLevel = Level; Config.PTO.LoopVectorization = Config.OptLevel > 1; Config.PTO.SLPVectorization = Config.OptLevel > 1; - switch (Config.OptLevel) { - case 0: - Config.CGOptLevel = CodeGenOpt::None; - return; - case 1: - Config.CGOptLevel = CodeGenOpt::Less; - return; - case 2: - Config.CGOptLevel = CodeGenOpt::Default; - return; - case 3: - Config.CGOptLevel = CodeGenOpt::Aggressive; - return; - } - llvm_unreachable("Unknown optimization level!"); + std::optional<CodeGenOpt::Level> CGOptLevelOrNone = + CodeGenOpt::getLevel(Config.OptLevel); + assert(CGOptLevelOrNone && "Unknown optimization level!"); + Config.CGOptLevel = *CGOptLevelOrNone; } bool LTOCodeGenerator::writeMergedModules(StringRef Path) { @@ -236,11 +242,78 @@ bool LTOCodeGenerator::writeMergedModules(StringRef Path) { return true; } +bool LTOCodeGenerator::useAIXSystemAssembler() { + const auto &Triple = TargetMach->getTargetTriple(); + return Triple.isOSAIX(); +} + +bool LTOCodeGenerator::runAIXSystemAssembler(SmallString<128> &AssemblyFile) { + assert(useAIXSystemAssembler() && + "Runing AIX system assembler when integrated assembler is available!"); + + // Set the system assembler path. + SmallString<256> AssemblerPath("/usr/bin/as"); + if (!llvm::AIXSystemAssemblerPath.empty()) { + if (llvm::sys::fs::real_path(llvm::AIXSystemAssemblerPath, AssemblerPath, + /* expand_tilde */ true)) { + emitError( + "Cannot find the assembler specified by lto-aix-system-assembler"); + return false; + } + } + + // Setup the LDR_CNTRL variable + std::string LDR_CNTRL_var = "LDR_CNTRL=MAXDATA32=0xA0000000@DSA"; + if (std::optional<std::string> V = sys::Process::GetEnv("LDR_CNTRL")) + LDR_CNTRL_var += ("@" + *V); + + // Prepare inputs for the assember. + const auto &Triple = TargetMach->getTargetTriple(); + const char *Arch = Triple.isArch64Bit() ? "-a64" : "-a32"; + std::string ObjectFileName(AssemblyFile); + ObjectFileName[ObjectFileName.size() - 1] = 'o'; + SmallVector<StringRef, 8> Args = { + "/bin/env", LDR_CNTRL_var, + AssemblerPath, Arch, + "-many", "-o", + ObjectFileName, AssemblyFile}; + + // Invoke the assembler. + int RC = sys::ExecuteAndWait(Args[0], Args); + + // Handle errors. + if (RC < -1) { + emitError("LTO assembler exited abnormally"); + return false; + } + if (RC < 0) { + emitError("Unable to invoke LTO assembler"); + return false; + } + if (RC > 0) { + emitError("LTO assembler invocation returned non-zero"); + return false; + } + + // Cleanup. + remove(AssemblyFile.c_str()); + + // Fix the output file name. + AssemblyFile = ObjectFileName; + + return true; +} + bool LTOCodeGenerator::compileOptimizedToFile(const char **Name) { + if (useAIXSystemAssembler()) + setFileType(CGFT_AssemblyFile); + // make unique temp output file to put generated code SmallString<128> Filename; - auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> { + auto AddStream = + [&](size_t Task, + const Twine &ModuleName) -> std::unique_ptr<CachedFileStream> { StringRef Extension(Config.CGFileType == CGFT_AssemblyFile ? "s" : "o"); int FD; @@ -267,6 +340,10 @@ bool LTOCodeGenerator::compileOptimizedToFile(const char **Name) { else if (AreStatisticsEnabled()) PrintStatistics(); + if (useAIXSystemAssembler()) + if (!runAIXSystemAssembler(Filename)) + return false; + NativeObjectPath = Filename.c_str(); *Name = NativeObjectPath.c_str(); return true; @@ -344,6 +421,11 @@ bool LTOCodeGenerator::determineTarget() { Config.CPU = "cyclone"; } + // If data-sections is not explicitly set or unset, set data-sections by + // default to match the behaviour of lld and gold plugin. + if (!codegen::getExplicitDataSections()) + Config.Options.DataSections = true; + TargetMach = createTargetMachine(); assert(TargetMach && "Unable to create target machine"); @@ -354,7 +436,7 @@ std::unique_ptr<TargetMachine> LTOCodeGenerator::createTargetMachine() { assert(MArch && "MArch is not set!"); return std::unique_ptr<TargetMachine>(MArch->createTargetMachine( TripleStr, Config.CPU, FeatureStr, Config.Options, Config.RelocModel, - None, Config.CGOptLevel)); + std::nullopt, Config.CGOptLevel)); } // If a linkonce global is present in the MustPreserveSymbols, we need to make diff --git a/llvm/lib/LTO/LTOModule.cpp b/llvm/lib/LTO/LTOModule.cpp index 5ad5e857296d..39f0f9842ec3 100644 --- a/llvm/lib/LTO/LTOModule.cpp +++ b/llvm/lib/LTO/LTOModule.cpp @@ -229,8 +229,8 @@ LTOModule::makeLTOModule(MemoryBufferRef Buffer, const TargetOptions &options, CPU = "cyclone"; } - TargetMachine *target = - march->createTargetMachine(TripleStr, CPU, FeatureStr, options, None); + TargetMachine *target = march->createTargetMachine(TripleStr, CPU, FeatureStr, + options, std::nullopt); std::unique_ptr<LTOModule> Ret(new LTOModule(std::move(M), Buffer, target)); Ret->parseSymbols(); diff --git a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp index 2c723bef7d12..5b137a8f8cb3 100644 --- a/llvm/lib/LTO/ThinLTOCodeGenerator.cpp +++ b/llvm/lib/LTO/ThinLTOCodeGenerator.cpp @@ -78,7 +78,7 @@ extern cl::opt<bool> LTODiscardValueNames; extern cl::opt<std::string> RemarksFilename; extern cl::opt<std::string> RemarksPasses; extern cl::opt<bool> RemarksWithHotness; -extern cl::opt<Optional<uint64_t>, false, remarks::HotnessThresholdParser> +extern cl::opt<std::optional<uint64_t>, false, remarks::HotnessThresholdParser> RemarksHotnessThreshold; extern cl::opt<std::string> RemarksFormat; } @@ -237,14 +237,14 @@ crossImportIntoModule(Module &TheModule, const ModuleSummaryIndex &Index, static void optimizeModule(Module &TheModule, TargetMachine &TM, unsigned OptLevel, bool Freestanding, bool DebugPassManager, ModuleSummaryIndex *Index) { - Optional<PGOOptions> PGOOpt; + std::optional<PGOOptions> PGOOpt; LoopAnalysisManager LAM; FunctionAnalysisManager FAM; CGSCCAnalysisManager CGAM; ModuleAnalysisManager MAM; PassInstrumentationCallbacks PIC; - StandardInstrumentations SI(DebugPassManager); + StandardInstrumentations SI(TheModule.getContext(), DebugPassManager); SI.registerCallbacks(PIC, &FAM); PipelineTuningOptions PTO; PTO.LoopVectorization = true; @@ -618,7 +618,7 @@ std::unique_ptr<TargetMachine> TargetMachineBuilder::create() const { std::unique_ptr<TargetMachine> TM( TheTarget->createTargetMachine(TheTriple.str(), MCpu, FeatureStr, Options, - RelocModel, None, CGOptLevel)); + RelocModel, std::nullopt, CGOptLevel)); assert(TM && "Cannot create target machine"); return TM; @@ -1217,7 +1217,7 @@ void ThinLTOCodeGenerator::run() { } } - pruneCache(CacheOptions.Path, CacheOptions.Policy); + pruneCache(CacheOptions.Path, CacheOptions.Policy, ProducedBinaries); // If statistics were requested, print them out now. if (llvm::AreStatisticsEnabled()) |
