diff options
Diffstat (limited to 'clang/lib/CodeGen/BackendUtil.cpp')
-rw-r--r-- | clang/lib/CodeGen/BackendUtil.cpp | 249 |
1 files changed, 162 insertions, 87 deletions
diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 7c4e35634e5d..10d6bff25e6d 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -19,6 +19,7 @@ #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Triple.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/StackSafetyAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" @@ -29,12 +30,13 @@ #include "llvm/CodeGen/SchedulerRegistry.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/IRPrintingPasses.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/Verifier.h" +#include "llvm/IRPrinter/IRPrintingPasses.h" #include "llvm/LTO/LTOBackend.h" #include "llvm/MC/MCAsmInfo.h" #include "llvm/MC/SubtargetFeature.h" @@ -70,14 +72,17 @@ #include "llvm/Transforms/Instrumentation/GCOVProfiler.h" #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" #include "llvm/Transforms/Instrumentation/InstrProfiling.h" +#include "llvm/Transforms/Instrumentation/KCFI.h" #include "llvm/Transforms/Instrumentation/MemProfiler.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h" #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" #include "llvm/Transforms/Instrumentation/ThreadSanitizer.h" #include "llvm/Transforms/ObjCARC.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Scalar/EarlyCSE.h" #include "llvm/Transforms/Scalar/GVN.h" +#include "llvm/Transforms/Scalar/JumpThreading.h" #include "llvm/Transforms/Scalar/LowerMatrixIntrinsics.h" #include "llvm/Transforms/Utils.h" #include "llvm/Transforms/Utils/CanonicalizeAliases.h" @@ -87,6 +92,7 @@ #include "llvm/Transforms/Utils/NameAnonGlobals.h" #include "llvm/Transforms/Utils/SymbolRewriter.h" #include <memory> +#include <optional> using namespace clang; using namespace llvm; @@ -96,6 +102,11 @@ using namespace llvm; namespace llvm { extern cl::opt<bool> DebugInfoCorrelate; + +// Experiment to move sanitizers earlier. +static cl::opt<bool> ClSanitizeOnOptimizerEarlyEP( + "sanitizer-early-opt-ep", cl::Optional, + cl::desc("Insert sanitizers on OptimizerEarlyEP."), cl::init(false)); } namespace { @@ -215,6 +226,16 @@ getSancovOptsFromCGOpts(const CodeGenOptions &CGOpts) { Opts.StackDepth = CGOpts.SanitizeCoverageStackDepth; Opts.TraceLoads = CGOpts.SanitizeCoverageTraceLoads; Opts.TraceStores = CGOpts.SanitizeCoverageTraceStores; + Opts.CollectControlFlow = CGOpts.SanitizeCoverageControlFlow; + return Opts; +} + +static SanitizerBinaryMetadataOptions +getSanitizerBinaryMetadataOptions(const CodeGenOptions &CGOpts) { + SanitizerBinaryMetadataOptions Opts; + Opts.Covered = CGOpts.SanitizeBinaryMetadataCovered; + Opts.Atomics = CGOpts.SanitizeBinaryMetadataAtomics; + Opts.UAR = CGOpts.SanitizeBinaryMetadataUAR; return Opts; } @@ -250,27 +271,28 @@ static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple, switch (CodeGenOpts.getVecLib()) { case CodeGenOptions::Accelerate: - TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate); + TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::Accelerate, + TargetTriple); break; case CodeGenOptions::LIBMVEC: - switch(TargetTriple.getArch()) { - default: - break; - case llvm::Triple::x86_64: - TLII->addVectorizableFunctionsFromVecLib - (TargetLibraryInfoImpl::LIBMVEC_X86); - break; - } + TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::LIBMVEC_X86, + TargetTriple); break; case CodeGenOptions::MASSV: - TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::MASSV); + TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::MASSV, + TargetTriple); break; case CodeGenOptions::SVML: - TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::SVML); + TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::SVML, + TargetTriple); + break; + case CodeGenOptions::SLEEF: + TLII->addVectorizableFunctionsFromVecLib(TargetLibraryInfoImpl::SLEEFGNUABI, + TargetTriple); break; case CodeGenOptions::Darwin_libsystem_m: TLII->addVectorizableFunctionsFromVecLib( - TargetLibraryInfoImpl::DarwinLibSystemM); + TargetLibraryInfoImpl::DarwinLibSystemM, TargetTriple); break; default: break; @@ -278,22 +300,7 @@ static TargetLibraryInfoImpl *createTLII(llvm::Triple &TargetTriple, return TLII; } -static CodeGenOpt::Level getCGOptLevel(const CodeGenOptions &CodeGenOpts) { - switch (CodeGenOpts.OptimizationLevel) { - default: - llvm_unreachable("Invalid optimization level!"); - case 0: - return CodeGenOpt::None; - case 1: - return CodeGenOpt::Less; - case 2: - return CodeGenOpt::Default; // O2/Os/Oz - case 3: - return CodeGenOpt::Aggressive; - } -} - -static Optional<llvm::CodeModel::Model> +static std::optional<llvm::CodeModel::Model> getCodeModel(const CodeGenOptions &CodeGenOpts) { unsigned CodeModel = llvm::StringSwitch<unsigned>(CodeGenOpts.CodeModel) .Case("tiny", llvm::CodeModel::Tiny) @@ -305,7 +312,7 @@ getCodeModel(const CodeGenOptions &CodeGenOpts) { .Default(~0u); assert(CodeModel != ~0u && "invalid code model!"); if (CodeModel == ~1u) - return None; + return std::nullopt; return static_cast<llvm::CodeModel::Model>(CodeModel); } @@ -391,7 +398,12 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, Options.NoInfsFPMath = LangOpts.NoHonorInfs; Options.NoNaNsFPMath = LangOpts.NoHonorNaNs; Options.NoZerosInBSS = CodeGenOpts.NoZeroInitializedInBSS; - Options.UnsafeFPMath = LangOpts.UnsafeFPMath; + Options.UnsafeFPMath = LangOpts.AllowFPReassoc && LangOpts.AllowRecip && + LangOpts.NoSignedZero && LangOpts.ApproxFunc && + (LangOpts.getDefaultFPContractMode() == + LangOptions::FPModeKind::FPM_Fast || + LangOpts.getDefaultFPContractMode() == + LangOptions::FPModeKind::FPM_FastHonorPragmas); Options.ApproxFuncFPMath = LangOpts.ApproxFunc; Options.BBSections = @@ -422,7 +434,7 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, CodeGenOpts.UniqueBasicBlockSectionNames; Options.TLSSize = CodeGenOpts.TLSSize; Options.EmulatedTLS = CodeGenOpts.EmulatedTLS; - Options.ExplicitEmulatedTLS = CodeGenOpts.ExplicitEmulatedTLS; + Options.ExplicitEmulatedTLS = true; Options.DebuggerTuning = CodeGenOpts.getDebuggerTuning(); Options.EmitStackSizeSection = CodeGenOpts.StackSizeSection; Options.StackUsageOutput = CodeGenOpts.StackUsageOutput; @@ -478,15 +490,16 @@ static bool initTargetOptions(DiagnosticsEngine &Diags, Entry.IgnoreSysRoot ? Entry.Path : HSOpts.Sysroot + Entry.Path); Options.MCOptions.Argv0 = CodeGenOpts.Argv0; Options.MCOptions.CommandLineArgs = CodeGenOpts.CommandLineArgs; + Options.MCOptions.AsSecureLogFile = CodeGenOpts.AsSecureLogFile; Options.MisExpect = CodeGenOpts.MisExpect; return true; } -static Optional<GCOVOptions> getGCOVOptions(const CodeGenOptions &CodeGenOpts, - const LangOptions &LangOpts) { +static std::optional<GCOVOptions> +getGCOVOptions(const CodeGenOptions &CodeGenOpts, const LangOptions &LangOpts) { if (!CodeGenOpts.EmitGcovArcs && !CodeGenOpts.EmitGcovNotes) - return None; + return std::nullopt; // Not using 'GCOVOptions::getDefault' allows us to avoid exiting if // LLVM's -default-gcov-version flag is set to something invalid. GCOVOptions Options; @@ -500,11 +513,11 @@ static Optional<GCOVOptions> getGCOVOptions(const CodeGenOptions &CodeGenOpts, return Options; } -static Optional<InstrProfOptions> +static std::optional<InstrProfOptions> getInstrProfOptions(const CodeGenOptions &CodeGenOpts, const LangOptions &LangOpts) { if (!CodeGenOpts.hasProfileClangInstr()) - return None; + return std::nullopt; InstrProfOptions Options; Options.NoRedZone = CodeGenOpts.DisableRedZone; Options.InstrProfileOutput = CodeGenOpts.InstrProfileOutput; @@ -547,11 +560,14 @@ void EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) { return; } - Optional<llvm::CodeModel::Model> CM = getCodeModel(CodeGenOpts); + std::optional<llvm::CodeModel::Model> CM = getCodeModel(CodeGenOpts); std::string FeaturesStr = llvm::join(TargetOpts.Features.begin(), TargetOpts.Features.end(), ","); llvm::Reloc::Model RM = CodeGenOpts.RelocationModel; - CodeGenOpt::Level OptLevel = getCGOptLevel(CodeGenOpts); + std::optional<CodeGenOpt::Level> OptLevelOrNone = + CodeGenOpt::getLevel(CodeGenOpts.OptimizationLevel); + assert(OptLevelOrNone && "Invalid optimization level!"); + CodeGenOpt::Level OptLevel = *OptLevelOrNone; llvm::TargetOptions Options; if (!initTargetOptions(Diags, Options, CodeGenOpts, TargetOpts, LangOpts, @@ -620,18 +636,48 @@ static OptimizationLevel mapToLevel(const CodeGenOptions &Opts) { } } +static void addKCFIPass(const Triple &TargetTriple, const LangOptions &LangOpts, + PassBuilder &PB) { + // If the back-end supports KCFI operand bundle lowering, skip KCFIPass. + if (TargetTriple.getArch() == llvm::Triple::x86_64 || + TargetTriple.isAArch64(64)) + return; + + // Ensure we lower KCFI operand bundles with -O0. + PB.registerOptimizerLastEPCallback( + [&](ModulePassManager &MPM, OptimizationLevel Level) { + if (Level == OptimizationLevel::O0 && + LangOpts.Sanitize.has(SanitizerKind::KCFI)) + MPM.addPass(createModuleToFunctionPassAdaptor(KCFIPass())); + }); + + // When optimizations are requested, run KCIFPass after InstCombine to + // avoid unnecessary checks. + PB.registerPeepholeEPCallback( + [&](FunctionPassManager &FPM, OptimizationLevel Level) { + if (Level != OptimizationLevel::O0 && + LangOpts.Sanitize.has(SanitizerKind::KCFI)) + FPM.addPass(KCFIPass()); + }); +} + static void addSanitizers(const Triple &TargetTriple, const CodeGenOptions &CodeGenOpts, const LangOptions &LangOpts, PassBuilder &PB) { - PB.registerOptimizerLastEPCallback([&](ModulePassManager &MPM, - OptimizationLevel Level) { + auto SanitizersCallback = [&](ModulePassManager &MPM, + OptimizationLevel Level) { if (CodeGenOpts.hasSanitizeCoverage()) { auto SancovOpts = getSancovOptsFromCGOpts(CodeGenOpts); - MPM.addPass(ModuleSanitizerCoveragePass( + MPM.addPass(SanitizerCoveragePass( SancovOpts, CodeGenOpts.SanitizeCoverageAllowlistFiles, CodeGenOpts.SanitizeCoverageIgnorelistFiles)); } + if (CodeGenOpts.hasSanitizeBinaryMetadata()) { + MPM.addPass(SanitizerBinaryMetadataPass( + getSanitizerBinaryMetadataOptions(CodeGenOpts))); + } + auto MSanPass = [&](SanitizerMask Mask, bool CompileKernel) { if (LangOpts.Sanitize.has(Mask)) { int TrackOrigins = CodeGenOpts.SanitizeMemoryTrackOrigins; @@ -639,22 +685,21 @@ static void addSanitizers(const Triple &TargetTriple, MemorySanitizerOptions options(TrackOrigins, Recover, CompileKernel, CodeGenOpts.SanitizeMemoryParamRetval); - MPM.addPass(ModuleMemorySanitizerPass(options)); - FunctionPassManager FPM; - FPM.addPass(MemorySanitizerPass(options)); + MPM.addPass(MemorySanitizerPass(options)); if (Level != OptimizationLevel::O0) { - // MemorySanitizer inserts complex instrumentation that mostly - // follows the logic of the original code, but operates on - // "shadow" values. It can benefit from re-running some - // general purpose optimization passes. - FPM.addPass(EarlyCSEPass()); - // TODO: Consider add more passes like in - // addGeneralOptsForMemorySanitizer. EarlyCSEPass makes visible - // difference on size. It's not clear if the rest is still - // usefull. InstCombinePass breakes - // compiler-rt/test/msan/select_origin.cpp. + // MemorySanitizer inserts complex instrumentation that mostly follows + // the logic of the original code, but operates on "shadow" values. It + // can benefit from re-running some general purpose optimization + // passes. + MPM.addPass(RequireAnalysisPass<GlobalsAA, Module>()); + FunctionPassManager FPM; + FPM.addPass(EarlyCSEPass(true /* Enable mem-ssa. */)); + FPM.addPass(InstCombinePass()); + FPM.addPass(JumpThreadingPass()); + FPM.addPass(GVNPass()); + FPM.addPass(InstCombinePass()); + MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } - MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); } }; MSanPass(SanitizerKind::Memory, false); @@ -676,8 +721,8 @@ static void addSanitizers(const Triple &TargetTriple, Opts.Recover = CodeGenOpts.SanitizeRecover.has(Mask); Opts.UseAfterScope = CodeGenOpts.SanitizeAddressUseAfterScope; Opts.UseAfterReturn = CodeGenOpts.getSanitizeAddressUseAfterReturn(); - MPM.addPass(ModuleAddressSanitizerPass( - Opts, UseGlobalGC, UseOdrIndicator, DestructorKind)); + MPM.addPass(AddressSanitizerPass(Opts, UseGlobalGC, UseOdrIndicator, + DestructorKind)); } }; ASanPass(SanitizerKind::Address, false); @@ -697,13 +742,28 @@ static void addSanitizers(const Triple &TargetTriple, if (LangOpts.Sanitize.has(SanitizerKind::DataFlow)) { MPM.addPass(DataFlowSanitizerPass(LangOpts.NoSanitizeFiles)); } - }); + }; + if (ClSanitizeOnOptimizerEarlyEP) { + PB.registerOptimizerEarlyEPCallback( + [SanitizersCallback](ModulePassManager &MPM, OptimizationLevel Level) { + ModulePassManager NewMPM; + SanitizersCallback(NewMPM, Level); + if (!NewMPM.isEmpty()) { + // Sanitizers can abandon<GlobalsAA>. + NewMPM.addPass(RequireAnalysisPass<GlobalsAA, Module>()); + MPM.addPass(std::move(NewMPM)); + } + }); + } else { + // LastEP does not need GlobalsAA. + PB.registerOptimizerLastEPCallback(SanitizersCallback); + } } void EmitAssemblyHelper::RunOptimizationPipeline( BackendAction Action, std::unique_ptr<raw_pwrite_stream> &OS, std::unique_ptr<llvm::ToolOutputFile> &ThinLinkOS) { - Optional<PGOOptions> PGOOpt; + std::optional<PGOOptions> PGOOpt; if (CodeGenOpts.hasProfileIRInstr()) // -fprofile-generate. @@ -782,12 +842,20 @@ void EmitAssemblyHelper::RunOptimizationPipeline( PrintPassOptions PrintPassOpts; PrintPassOpts.Indent = DebugPassStructure; PrintPassOpts.SkipAnalyses = DebugPassStructure; - StandardInstrumentations SI(CodeGenOpts.DebugPassManager || - DebugPassStructure, - /*VerifyEach*/ false, PrintPassOpts); + StandardInstrumentations SI( + TheModule->getContext(), + (CodeGenOpts.DebugPassManager || DebugPassStructure), + /*VerifyEach*/ false, PrintPassOpts); SI.registerCallbacks(PIC, &FAM); PassBuilder PB(TM.get(), PTO, PGOOpt, &PIC); + if (CodeGenOpts.EnableAssignmentTracking) { + PB.registerPipelineStartEPCallback( + [&](ModulePassManager &MPM, OptimizationLevel Level) { + MPM.addPass(AssignmentTrackingPass()); + }); + } + // Enable verify-debuginfo-preserve-each for new PM. DebugifyEachInstrumentation Debugify; DebugInfoPerPass DebugInfoBeforePass; @@ -896,15 +964,18 @@ void EmitAssemblyHelper::RunOptimizationPipeline( // Don't add sanitizers if we are here from ThinLTO PostLink. That already // done on PreLink stage. - if (!IsThinLTOPostLink) + if (!IsThinLTOPostLink) { addSanitizers(TargetTriple, CodeGenOpts, LangOpts, PB); + addKCFIPass(TargetTriple, LangOpts, PB); + } - if (Optional<GCOVOptions> Options = getGCOVOptions(CodeGenOpts, LangOpts)) + if (std::optional<GCOVOptions> Options = + getGCOVOptions(CodeGenOpts, LangOpts)) PB.registerPipelineStartEPCallback( [Options](ModulePassManager &MPM, OptimizationLevel Level) { MPM.addPass(GCOVProfilerPass(*Options)); }); - if (Optional<InstrProfOptions> Options = + if (std::optional<InstrProfOptions> Options = getInstrProfOptions(CodeGenOpts, LangOpts)) PB.registerPipelineStartEPCallback( [Options](ModulePassManager &MPM, OptimizationLevel Level) { @@ -933,19 +1004,24 @@ void EmitAssemblyHelper::RunOptimizationPipeline( if (!actionRequiresCodeGen(Action) && CodeGenOpts.VerifyModule) MPM.addPass(VerifierPass()); - switch (Action) { - case Backend_EmitBC: + if (Action == Backend_EmitBC || Action == Backend_EmitLL) { if (CodeGenOpts.PrepareForThinLTO && !CodeGenOpts.DisableLLVMPasses) { - if (!CodeGenOpts.ThinLinkBitcodeFile.empty()) { - ThinLinkOS = openOutputFile(CodeGenOpts.ThinLinkBitcodeFile); - if (!ThinLinkOS) - return; - } if (!TheModule->getModuleFlag("EnableSplitLTOUnit")) TheModule->addModuleFlag(Module::Error, "EnableSplitLTOUnit", CodeGenOpts.EnableSplitLTOUnit); - MPM.addPass(ThinLTOBitcodeWriterPass(*OS, ThinLinkOS ? &ThinLinkOS->os() - : nullptr)); + if (Action == Backend_EmitBC) { + if (!CodeGenOpts.ThinLinkBitcodeFile.empty()) { + ThinLinkOS = openOutputFile(CodeGenOpts.ThinLinkBitcodeFile); + if (!ThinLinkOS) + return; + } + MPM.addPass(ThinLTOBitcodeWriterPass(*OS, ThinLinkOS ? &ThinLinkOS->os() + : nullptr)); + } else { + MPM.addPass(PrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists, + /*EmitLTOSummary=*/true)); + } + } else { // Emit a module summary by default for Regular LTO except for ld64 // targets @@ -957,17 +1033,13 @@ void EmitAssemblyHelper::RunOptimizationPipeline( TheModule->addModuleFlag(Module::Error, "EnableSplitLTOUnit", uint32_t(1)); } - MPM.addPass( - BitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists, EmitLTOSummary)); + if (Action == Backend_EmitBC) + MPM.addPass(BitcodeWriterPass(*OS, CodeGenOpts.EmitLLVMUseLists, + EmitLTOSummary)); + else + MPM.addPass(PrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists, + EmitLTOSummary)); } - break; - - case Backend_EmitLL: - MPM.addPass(PrintModulePass(*OS, "", CodeGenOpts.EmitLLVMUseLists)); - break; - - default: - break; } // Now that we have all of the passes ready, run them. @@ -1059,7 +1131,7 @@ static void runThinLTOBackend( if (!lto::initImportList(*M, *CombinedIndex, ImportList)) return; - auto AddStream = [&](size_t Task) { + auto AddStream = [&](size_t Task, const Twine &ModuleName) { return std::make_unique<CachedFileStream>(std::move(OS), CGOpts.ObjectFilenameForDebug); }; @@ -1077,7 +1149,10 @@ static void runThinLTOBackend( Conf.CodeModel = getCodeModel(CGOpts); Conf.MAttrs = TOpts.Features; Conf.RelocModel = CGOpts.RelocationModel; - Conf.CGOptLevel = getCGOptLevel(CGOpts); + std::optional<CodeGenOpt::Level> OptLevelOrNone = + CodeGenOpt::getLevel(CGOpts.OptimizationLevel); + assert(OptLevelOrNone && "Invalid optimization level!"); + Conf.CGOptLevel = *OptLevelOrNone; Conf.OptLevel = CGOpts.OptimizationLevel; initTargetOptions(Diags, Conf.Options, CGOpts, TOpts, LOpts, HeaderOpts); Conf.SampleProfile = std::move(SampleProfile); |