diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp')
| -rw-r--r-- | llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 391 |
1 files changed, 328 insertions, 63 deletions
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index ce7c82e2a88a..e4485f87fb79 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -32,6 +32,8 @@ #include "llvm/CodeGen/GlobalISel/Localizer.h" #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" #include "llvm/CodeGen/MIRParser/MIParser.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/PassManager.h" @@ -52,6 +54,115 @@ using namespace llvm; +namespace { +class SGPRRegisterRegAlloc : public RegisterRegAllocBase<SGPRRegisterRegAlloc> { +public: + SGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C) + : RegisterRegAllocBase(N, D, C) {} +}; + +class VGPRRegisterRegAlloc : public RegisterRegAllocBase<VGPRRegisterRegAlloc> { +public: + VGPRRegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C) + : RegisterRegAllocBase(N, D, C) {} +}; + +static bool onlyAllocateSGPRs(const TargetRegisterInfo &TRI, + const TargetRegisterClass &RC) { + return static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(&RC); +} + +static bool onlyAllocateVGPRs(const TargetRegisterInfo &TRI, + const TargetRegisterClass &RC) { + return !static_cast<const SIRegisterInfo &>(TRI).isSGPRClass(&RC); +} + + +/// -{sgpr|vgpr}-regalloc=... command line option. +static FunctionPass *useDefaultRegisterAllocator() { return nullptr; } + +/// A dummy default pass factory indicates whether the register allocator is +/// overridden on the command line. +static llvm::once_flag InitializeDefaultSGPRRegisterAllocatorFlag; +static llvm::once_flag InitializeDefaultVGPRRegisterAllocatorFlag; + +static SGPRRegisterRegAlloc +defaultSGPRRegAlloc("default", + "pick SGPR register allocator based on -O option", + useDefaultRegisterAllocator); + +static cl::opt<SGPRRegisterRegAlloc::FunctionPassCtor, false, + RegisterPassParser<SGPRRegisterRegAlloc>> +SGPRRegAlloc("sgpr-regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator), + cl::desc("Register allocator to use for SGPRs")); + +static cl::opt<VGPRRegisterRegAlloc::FunctionPassCtor, false, + RegisterPassParser<VGPRRegisterRegAlloc>> +VGPRRegAlloc("vgpr-regalloc", cl::Hidden, cl::init(&useDefaultRegisterAllocator), + cl::desc("Register allocator to use for VGPRs")); + + +static void initializeDefaultSGPRRegisterAllocatorOnce() { + RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault(); + + if (!Ctor) { + Ctor = SGPRRegAlloc; + SGPRRegisterRegAlloc::setDefault(SGPRRegAlloc); + } +} + +static void initializeDefaultVGPRRegisterAllocatorOnce() { + RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault(); + + if (!Ctor) { + Ctor = VGPRRegAlloc; + VGPRRegisterRegAlloc::setDefault(VGPRRegAlloc); + } +} + +static FunctionPass *createBasicSGPRRegisterAllocator() { + return createBasicRegisterAllocator(onlyAllocateSGPRs); +} + +static FunctionPass *createGreedySGPRRegisterAllocator() { + return createGreedyRegisterAllocator(onlyAllocateSGPRs); +} + +static FunctionPass *createFastSGPRRegisterAllocator() { + return createFastRegisterAllocator(onlyAllocateSGPRs, false); +} + +static FunctionPass *createBasicVGPRRegisterAllocator() { + return createBasicRegisterAllocator(onlyAllocateVGPRs); +} + +static FunctionPass *createGreedyVGPRRegisterAllocator() { + return createGreedyRegisterAllocator(onlyAllocateVGPRs); +} + +static FunctionPass *createFastVGPRRegisterAllocator() { + return createFastRegisterAllocator(onlyAllocateVGPRs, true); +} + +static SGPRRegisterRegAlloc basicRegAllocSGPR( + "basic", "basic register allocator", createBasicSGPRRegisterAllocator); +static SGPRRegisterRegAlloc greedyRegAllocSGPR( + "greedy", "greedy register allocator", createGreedySGPRRegisterAllocator); + +static SGPRRegisterRegAlloc fastRegAllocSGPR( + "fast", "fast register allocator", createFastSGPRRegisterAllocator); + + +static VGPRRegisterRegAlloc basicRegAllocVGPR( + "basic", "basic register allocator", createBasicVGPRRegisterAllocator); +static VGPRRegisterRegAlloc greedyRegAllocVGPR( + "greedy", "greedy register allocator", createGreedyVGPRRegisterAllocator); + +static VGPRRegisterRegAlloc fastRegAllocVGPR( + "fast", "fast register allocator", createFastVGPRRegisterAllocator); +} + + static cl::opt<bool> EnableR600StructurizeCFG( "r600-ir-structurize", cl::desc("Use StructurizeCFG IR pass"), @@ -162,6 +273,11 @@ static cl::opt<bool> EnableRegReassign( cl::init(true), cl::Hidden); +static cl::opt<bool> OptVGPRLiveRange( + "amdgpu-opt-vgpr-liverange", + cl::desc("Enable VGPR liverange optimizations for if-else structure"), + cl::init(true), cl::Hidden); + // Enable atomic optimization static cl::opt<bool> EnableAtomicOptimizations( "amdgpu-atomic-optimizations", @@ -193,6 +309,21 @@ static cl::opt<bool> EnableStructurizerWorkarounds( cl::desc("Enable workarounds for the StructurizeCFG pass"), cl::init(true), cl::Hidden); +static cl::opt<bool> EnableLDSReplaceWithPointer( + "amdgpu-enable-lds-replace-with-pointer", + cl::desc("Enable LDS replace with pointer pass"), cl::init(false), + cl::Hidden); + +static cl::opt<bool, true> EnableLowerModuleLDS( + "amdgpu-enable-lower-module-lds", cl::desc("Enable lower module lds pass"), + cl::location(AMDGPUTargetMachine::EnableLowerModuleLDS), cl::init(true), + cl::Hidden); + +static cl::opt<bool> EnablePreRAOptimizations( + "amdgpu-enable-pre-ra-optimizations", + cl::desc("Enable Pre-RA optimizations pass"), cl::init(true), + cl::Hidden); + extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() { // Register the target RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget()); @@ -215,9 +346,11 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() { initializeSIPeepholeSDWAPass(*PR); initializeSIShrinkInstructionsPass(*PR); initializeSIOptimizeExecMaskingPreRAPass(*PR); + initializeSIOptimizeVGPRLiveRangePass(*PR); initializeSILoadStoreOptimizerPass(*PR); initializeAMDGPUFixFunctionBitcastsPass(*PR); initializeAMDGPUAlwaysInlinePass(*PR); + initializeAMDGPUAttributorPass(*PR); initializeAMDGPUAnnotateKernelFeaturesPass(*PR); initializeAMDGPUAnnotateUniformValuesPass(*PR); initializeAMDGPUArgumentUsageInfoPass(*PR); @@ -228,12 +361,15 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() { initializeAMDGPUOpenCLEnqueuedBlockLoweringPass(*PR); initializeAMDGPUPostLegalizerCombinerPass(*PR); initializeAMDGPUPreLegalizerCombinerPass(*PR); + initializeAMDGPURegBankCombinerPass(*PR); initializeAMDGPUPromoteAllocaPass(*PR); initializeAMDGPUPromoteAllocaToVectorPass(*PR); initializeAMDGPUCodeGenPreparePass(*PR); initializeAMDGPULateCodeGenPreparePass(*PR); initializeAMDGPUPropagateAttributesEarlyPass(*PR); initializeAMDGPUPropagateAttributesLatePass(*PR); + initializeAMDGPUReplaceLDSUseWithPointerPass(*PR); + initializeAMDGPULowerModuleLDSPass(*PR); initializeAMDGPURewriteOutArgumentsPass(*PR); initializeAMDGPUUnifyMetadataPass(*PR); initializeSIAnnotateControlFlowPass(*PR); @@ -242,9 +378,8 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() { initializeSIModeRegisterPass(*PR); initializeSIWholeQuadModePass(*PR); initializeSILowerControlFlowPass(*PR); - initializeSIRemoveShortExecBranchesPass(*PR); initializeSIPreEmitPeepholePass(*PR); - initializeSIInsertSkipsPass(*PR); + initializeSILateBranchLoweringPass(*PR); initializeSIMemoryLegalizerPass(*PR); initializeSIOptimizeExecMaskingPass(*PR); initializeSIPreAllocateWWMRegsPass(*PR); @@ -256,9 +391,9 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() { initializeAMDGPUUseNativeCallsPass(*PR); initializeAMDGPUSimplifyLibCallsPass(*PR); initializeAMDGPUPrintfRuntimeBindingPass(*PR); - initializeGCNRegBankReassignPass(*PR); + initializeAMDGPUResourceUsageAnalysisPass(*PR); initializeGCNNSAReassignPass(*PR); - initializeSIAddIMGInitPass(*PR); + initializeGCNPreRAOptimizationsPass(*PR); } static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) { @@ -388,6 +523,7 @@ AMDGPUTargetMachine::AMDGPUTargetMachine(const Target &T, const Triple &TT, bool AMDGPUTargetMachine::EnableLateStructurizeCFG = false; bool AMDGPUTargetMachine::EnableFunctionCalls = false; bool AMDGPUTargetMachine::EnableFixedFunctionABI = false; +bool AMDGPUTargetMachine::EnableLowerModuleLDS = true; AMDGPUTargetMachine::~AMDGPUTargetMachine() = default; @@ -408,6 +544,7 @@ static bool mustPreserveGV(const GlobalValue &GV) { if (const Function *F = dyn_cast<Function>(&GV)) return F->isDeclaration() || AMDGPU::isEntryFunctionCC(F->getCallingConv()); + GV.removeDeadConstantUsers(); return !GV.use_empty(); } @@ -480,8 +617,7 @@ void AMDGPUTargetMachine::registerDefaultAliasAnalyses(AAManager &AAM) { AAM.registerFunctionAnalysis<AMDGPUAA>(); } -void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB, - bool DebugPassManager) { +void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB) { PB.registerPipelineParsingCallback( [this](StringRef PassName, ModulePassManager &PM, ArrayRef<PassBuilder::PipelineElement>) { @@ -501,6 +637,14 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB, PM.addPass(AMDGPUAlwaysInlinePass()); return true; } + if (PassName == "amdgpu-replace-lds-use-with-pointer") { + PM.addPass(AMDGPUReplaceLDSUseWithPointerPass()); + return true; + } + if (PassName == "amdgpu-lower-module-lds") { + PM.addPass(AMDGPULowerModuleLDSPass()); + return true; + } return false; }); PB.registerPipelineParsingCallback( @@ -530,7 +674,6 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB, PM.addPass(AMDGPUPropagateAttributesEarlyPass(*this)); return true; } - return false; }); @@ -546,16 +689,16 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB, return false; }); - PB.registerPipelineStartEPCallback([this, DebugPassManager]( - ModulePassManager &PM, - PassBuilder::OptimizationLevel Level) { - FunctionPassManager FPM(DebugPassManager); - FPM.addPass(AMDGPUPropagateAttributesEarlyPass(*this)); - FPM.addPass(AMDGPUUseNativeCallsPass()); - if (EnableLibCallSimplify && Level != PassBuilder::OptimizationLevel::O0) - FPM.addPass(AMDGPUSimplifyLibCallsPass(*this)); - PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); - }); + PB.registerPipelineStartEPCallback( + [this](ModulePassManager &PM, PassBuilder::OptimizationLevel Level) { + FunctionPassManager FPM; + FPM.addPass(AMDGPUPropagateAttributesEarlyPass(*this)); + FPM.addPass(AMDGPUUseNativeCallsPass()); + if (EnableLibCallSimplify && + Level != PassBuilder::OptimizationLevel::O0) + FPM.addPass(AMDGPUSimplifyLibCallsPass(*this)); + PM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM))); + }); PB.registerPipelineEarlySimplificationEPCallback( [this](ModulePassManager &PM, PassBuilder::OptimizationLevel Level) { @@ -577,12 +720,11 @@ void AMDGPUTargetMachine::registerPassBuilderCallbacks(PassBuilder &PB, }); PB.registerCGSCCOptimizerLateEPCallback( - [this, DebugPassManager](CGSCCPassManager &PM, - PassBuilder::OptimizationLevel Level) { + [this](CGSCCPassManager &PM, PassBuilder::OptimizationLevel Level) { if (Level == PassBuilder::OptimizationLevel::O0) return; - FunctionPassManager FPM(DebugPassManager); + FunctionPassManager FPM; // Add infer address spaces pass to the opt pipeline after inlining // but before SROA to increase SROA opportunities. @@ -732,6 +874,9 @@ public: // anything. disablePass(&StackMapLivenessID); disablePass(&FuncletLayoutID); + // Garbage collection is not supported. + disablePass(&GCLoweringID); + disablePass(&ShadowStackGCLoweringID); } AMDGPUTargetMachine &getAMDGPUTargetMachine() const { @@ -754,6 +899,19 @@ public: bool addGCPasses() override; std::unique_ptr<CSEConfigBase> getCSEConfig() const override; + + /// Check if a pass is enabled given \p Opt option. The option always + /// overrides defaults if explicitely used. Otherwise its default will + /// be used given that a pass shall work at an optimization \p Level + /// minimum. + bool isPassEnabled(const cl::opt<bool> &Opt, + CodeGenOpt::Level Level = CodeGenOpt::Default) const { + if (Opt.getNumOccurrences()) + return Opt; + if (TM->getOptLevel() < Level) + return false; + return Opt; + } }; std::unique_ptr<CSEConfigBase> AMDGPUPassConfig::getCSEConfig() const { @@ -803,9 +961,18 @@ public: bool addLegalizeMachineIR() override; void addPreRegBankSelect() override; bool addRegBankSelect() override; + void addPreGlobalInstructionSelect() override; bool addGlobalInstructionSelect() override; void addFastRegAlloc() override; void addOptimizedRegAlloc() override; + + FunctionPass *createSGPRAllocPass(bool Optimized); + FunctionPass *createVGPRAllocPass(bool Optimized); + FunctionPass *createRegAllocPass(bool Optimized) override; + + bool addRegAssignAndRewriteFast() override; + bool addRegAssignAndRewriteOptimized() override; + void addPreRegAlloc() override; bool addPreRewrite() override; void addPostRegAlloc() override; @@ -856,9 +1023,6 @@ void AMDGPUPassConfig::addIRPasses() { // A call to propagate attributes pass in the backend in case opt was not run. addPass(createAMDGPUPropagateAttributesEarlyPass(&TM)); - addPass(createAtomicExpandPass()); - - addPass(createAMDGPULowerIntrinsicsPass()); // Function calls are not supported, so make sure we inline everything. @@ -878,14 +1042,28 @@ void AMDGPUPassConfig::addIRPasses() { // Replace OpenCL enqueued block function pointers with global variables. addPass(createAMDGPUOpenCLEnqueuedBlockLoweringPass()); - if (TM.getOptLevel() > CodeGenOpt::None) { + // Can increase LDS used by kernel so runs before PromoteAlloca + if (EnableLowerModuleLDS) { + // The pass "amdgpu-replace-lds-use-with-pointer" need to be run before the + // pass "amdgpu-lower-module-lds", and also it required to be run only if + // "amdgpu-lower-module-lds" pass is enabled. + if (EnableLDSReplaceWithPointer) + addPass(createAMDGPUReplaceLDSUseWithPointerPass()); + + addPass(createAMDGPULowerModuleLDSPass()); + } + + if (TM.getOptLevel() > CodeGenOpt::None) addPass(createInferAddressSpacesPass()); + + addPass(createAtomicExpandPass()); + + if (TM.getOptLevel() > CodeGenOpt::None) { addPass(createAMDGPUPromoteAlloca()); if (EnableSROA) addPass(createSROAPass()); - - if (EnableScalarIRPasses) + if (isPassEnabled(EnableScalarIRPasses)) addStraightLineScalarOptimizationPasses(); if (EnableAMDGPUAliasAnalysis) { @@ -896,11 +1074,11 @@ void AMDGPUPassConfig::addIRPasses() { AAR.addAAResult(WrapperPass->getResult()); })); } - } - if (TM.getTargetTriple().getArch() == Triple::amdgcn) { - // TODO: May want to move later or split into an early and late one. - addPass(createAMDGPUCodeGenPreparePass()); + if (TM.getTargetTriple().getArch() == Triple::amdgcn) { + // TODO: May want to move later or split into an early and late one. + addPass(createAMDGPUCodeGenPreparePass()); + } } TargetPassConfig::addIRPasses(); @@ -917,7 +1095,7 @@ void AMDGPUPassConfig::addIRPasses() { // %1 = shl %a, 2 // // but EarlyCSE can do neither of them. - if (getOptLevel() != CodeGenOpt::None && EnableScalarIRPasses) + if (isPassEnabled(EnableScalarIRPasses)) addEarlyCSEOrGVNPass(); } @@ -929,11 +1107,9 @@ void AMDGPUPassConfig::addCodeGenPrepare() { EnableLowerKernelArguments) addPass(createAMDGPULowerKernelArgumentsPass()); - addPass(&AMDGPUPerfHintAnalysisID); - TargetPassConfig::addCodeGenPrepare(); - if (EnableLoadStoreVectorizer) + if (isPassEnabled(EnableLoadStoreVectorizer)) addPass(createLoadStoreVectorizerPass()); // LowerSwitch pass may introduce unreachable blocks that can @@ -944,7 +1120,8 @@ void AMDGPUPassConfig::addCodeGenPrepare() { } bool AMDGPUPassConfig::addPreISel() { - addPass(createFlattenCFGPass()); + if (TM->getOptLevel() > CodeGenOpt::None) + addPass(createFlattenCFGPass()); return false; } @@ -1014,13 +1191,15 @@ ScheduleDAGInstrs *GCNPassConfig::createMachineScheduler( bool GCNPassConfig::addPreISel() { AMDGPUPassConfig::addPreISel(); - addPass(createAMDGPULateCodeGenPreparePass()); - if (EnableAtomicOptimizations) { + if (TM->getOptLevel() > CodeGenOpt::None) + addPass(createAMDGPULateCodeGenPreparePass()); + + if (isPassEnabled(EnableAtomicOptimizations, CodeGenOpt::Less)) { addPass(createAMDGPUAtomicOptimizerPass()); } - // FIXME: We need to run a pass to propagate the attributes when calls are - // supported. + if (TM->getOptLevel() > CodeGenOpt::None) + addPass(createSinkingPass()); // Merge divergent exit nodes. StructurizeCFG won't recognize the multi-exit // regions formed by them. @@ -1032,13 +1211,15 @@ bool GCNPassConfig::addPreISel() { } addPass(createStructurizeCFGPass(false)); // true -> SkipUniformRegions } - addPass(createSinkingPass()); addPass(createAMDGPUAnnotateUniformValues()); if (!LateCFGStructurize) { addPass(createSIAnnotateControlFlowPass()); } addPass(createLCSSAPass()); + if (TM->getOptLevel() > CodeGenOpt::Less) + addPass(&AMDGPUPerfHintAnalysisID); + return false; } @@ -1055,15 +1236,14 @@ void GCNPassConfig::addMachineSSAOptimization() { addPass(&SIFoldOperandsID); if (EnableDPPCombine) addPass(&GCNDPPCombineID); - addPass(&DeadMachineInstructionElimID); addPass(&SILoadStoreOptimizerID); - if (EnableSDWAPeephole) { + if (isPassEnabled(EnableSDWAPeephole)) { addPass(&SIPeepholeSDWAID); addPass(&EarlyMachineLICMID); addPass(&MachineCSEID); addPass(&SIFoldOperandsID); - addPass(&DeadMachineInstructionElimID); } + addPass(&DeadMachineInstructionElimID); addPass(createSIShrinkInstructionsPass()); } @@ -1079,7 +1259,6 @@ bool GCNPassConfig::addInstSelector() { AMDGPUPassConfig::addInstSelector(); addPass(&SIFixSGPRCopiesID); addPass(createSILowerI1CopiesPass()); - addPass(createSIAddIMGInitPass()); return false; } @@ -1109,12 +1288,13 @@ bool GCNPassConfig::addRegBankSelect() { return false; } +void GCNPassConfig::addPreGlobalInstructionSelect() { + bool IsOptNone = getOptLevel() == CodeGenOpt::None; + addPass(createAMDGPURegBankCombiner(IsOptNone)); +} + bool GCNPassConfig::addGlobalInstructionSelect() { - addPass(new InstructionSelect()); - // TODO: Fix instruction selection to do the right thing for image - // instructions with tfe or lwe in the first place, instead of running a - // separate pass to fix them up? - addPass(createSIAddIMGInitPass()); + addPass(new InstructionSelect(getOptLevel())); return false; } @@ -1147,8 +1327,21 @@ void GCNPassConfig::addOptimizedRegAlloc() { if (OptExecMaskPreRA) insertPass(&MachineSchedulerID, &SIOptimizeExecMaskingPreRAID); - insertPass(&MachineSchedulerID, &SIFormMemoryClausesID); + if (isPassEnabled(EnablePreRAOptimizations)) + insertPass(&RenameIndependentSubregsID, &GCNPreRAOptimizationsID); + + // This is not an essential optimization and it has a noticeable impact on + // compilation time, so we only enable it from O2. + if (TM->getOptLevel() > CodeGenOpt::Less) + insertPass(&MachineSchedulerID, &SIFormMemoryClausesID); + + // FIXME: when an instruction has a Killed operand, and the instruction is + // inside a bundle, seems only the BUNDLE instruction appears as the Kills of + // the register in LiveVariables, this would trigger a failure in verifier, + // we should fix it and enable the verifier. + if (OptVGPRLiveRange) + insertPass(&LiveVariablesID, &SIOptimizeVGPRLiveRangeID, false); // This must be run immediately after phi elimination and before // TwoAddressInstructions, otherwise the processing of the tied operand of // SI_ELSE will introduce a copy of the tied operand source after the else. @@ -1161,10 +1354,81 @@ void GCNPassConfig::addOptimizedRegAlloc() { } bool GCNPassConfig::addPreRewrite() { - if (EnableRegReassign) { + if (EnableRegReassign) addPass(&GCNNSAReassignID); - addPass(&GCNRegBankReassignID); - } + return true; +} + +FunctionPass *GCNPassConfig::createSGPRAllocPass(bool Optimized) { + // Initialize the global default. + llvm::call_once(InitializeDefaultSGPRRegisterAllocatorFlag, + initializeDefaultSGPRRegisterAllocatorOnce); + + RegisterRegAlloc::FunctionPassCtor Ctor = SGPRRegisterRegAlloc::getDefault(); + if (Ctor != useDefaultRegisterAllocator) + return Ctor(); + + if (Optimized) + return createGreedyRegisterAllocator(onlyAllocateSGPRs); + + return createFastRegisterAllocator(onlyAllocateSGPRs, false); +} + +FunctionPass *GCNPassConfig::createVGPRAllocPass(bool Optimized) { + // Initialize the global default. + llvm::call_once(InitializeDefaultVGPRRegisterAllocatorFlag, + initializeDefaultVGPRRegisterAllocatorOnce); + + RegisterRegAlloc::FunctionPassCtor Ctor = VGPRRegisterRegAlloc::getDefault(); + if (Ctor != useDefaultRegisterAllocator) + return Ctor(); + + if (Optimized) + return createGreedyVGPRRegisterAllocator(); + + return createFastVGPRRegisterAllocator(); +} + +FunctionPass *GCNPassConfig::createRegAllocPass(bool Optimized) { + llvm_unreachable("should not be used"); +} + +static const char RegAllocOptNotSupportedMessage[] = + "-regalloc not supported with amdgcn. Use -sgpr-regalloc and -vgpr-regalloc"; + +bool GCNPassConfig::addRegAssignAndRewriteFast() { + if (!usingDefaultRegAlloc()) + report_fatal_error(RegAllocOptNotSupportedMessage); + + addPass(createSGPRAllocPass(false)); + + // Equivalent of PEI for SGPRs. + addPass(&SILowerSGPRSpillsID); + + addPass(createVGPRAllocPass(false)); + return true; +} + +bool GCNPassConfig::addRegAssignAndRewriteOptimized() { + if (!usingDefaultRegAlloc()) + report_fatal_error(RegAllocOptNotSupportedMessage); + + addPass(createSGPRAllocPass(true)); + + // Commit allocated register changes. This is mostly necessary because too + // many things rely on the use lists of the physical registers, such as the + // verifier. This is only necessary with allocators which use LiveIntervals, + // since FastRegAlloc does the replacments itself. + addPass(createVirtRegRewriter(false)); + + // Equivalent of PEI for SGPRs. + addPass(&SILowerSGPRSpillsID); + + addPass(createVGPRAllocPass(true)); + + addPreRewrite(); + addPass(&VirtRegRewriterID); + return true; } @@ -1173,9 +1437,6 @@ void GCNPassConfig::addPostRegAlloc() { if (getOptLevel() > CodeGenOpt::None) addPass(&SIOptimizeExecMaskingID); TargetPassConfig::addPostRegAlloc(); - - // Equivalent of PEI for SGPRs. - addPass(&SILowerSGPRSpillsID); } void GCNPassConfig::addPreSched2() { @@ -1185,15 +1446,18 @@ void GCNPassConfig::addPreSched2() { void GCNPassConfig::addPreEmitPass() { addPass(createSIMemoryLegalizerPass()); addPass(createSIInsertWaitcntsPass()); - addPass(createSIShrinkInstructionsPass()); + + if (TM->getOptLevel() > CodeGenOpt::None) + addPass(createSIShrinkInstructionsPass()); + addPass(createSIModeRegisterPass()); if (getOptLevel() > CodeGenOpt::None) addPass(&SIInsertHardClausesID); - addPass(&SIRemoveShortExecBranchesID); - addPass(&SIInsertSkipsPassID); - addPass(&SIPreEmitPeepholeID); + addPass(&SILateBranchLoweringPassID); + if (getOptLevel() > CodeGenOpt::None) + addPass(&SIPreEmitPeepholeID); // The hazard recognizer that runs as part of the post-ra scheduler does not // guarantee to be able handle all hazards correctly. This is because if there // are multiple scheduling regions in a basic block, the regions are scheduled @@ -1217,8 +1481,8 @@ yaml::MachineFunctionInfo *GCNTargetMachine::createDefaultFuncInfoYAML() const { yaml::MachineFunctionInfo * GCNTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const { const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); - return new yaml::SIMachineFunctionInfo(*MFI, - *MF.getSubtarget().getRegisterInfo()); + return new yaml::SIMachineFunctionInfo( + *MFI, *MF.getSubtarget().getRegisterInfo(), MF); } bool GCNTargetMachine::parseMachineFunctionInfo( @@ -1229,7 +1493,8 @@ bool GCNTargetMachine::parseMachineFunctionInfo( MachineFunction &MF = PFS.MF; SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); - MFI->initializeBaseYamlFields(YamlMFI); + if (MFI->initializeBaseYamlFields(YamlMFI, MF, PFS, Error, SourceRange)) + return true; if (MFI->Occupancy == 0) { // Fixup the subtarget dependent default value. |