diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU')
106 files changed, 4175 insertions, 1840 deletions
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h index b64422ae5427..fbed51de0ea4 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -11,6 +11,8 @@ #define LLVM_LIB_TARGET_AMDGPU_AMDGPU_H #include "llvm/Target/TargetMachine.h" +#include "llvm/IR/IntrinsicsR600.h" // TODO: Sink this. +#include "llvm/IR/IntrinsicsAMDGPU.h" // TODO: Sink this. namespace llvm { @@ -154,6 +156,9 @@ extern char &SIWholeQuadModeID; void initializeSILowerControlFlowPass(PassRegistry &); extern char &SILowerControlFlowID; +void initializeSIRemoveShortExecBranchesPass(PassRegistry &); +extern char &SIRemoveShortExecBranchesID; + void initializeSIInsertSkipsPass(PassRegistry &); extern char &SIInsertSkipsPassID; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp index 4c1dbd4c5304..ff2bda6bed53 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp @@ -118,7 +118,7 @@ bool AMDGPUAlwaysInline::runOnModule(Module &M) { for (GlobalVariable &GV : M.globals()) { // TODO: Region address - unsigned AS = GV.getType()->getAddressSpace(); + unsigned AS = GV.getAddressSpace(); if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS) continue; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp index 71121ade0a49..6fb507083cef 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp @@ -19,6 +19,7 @@ #include "llvm/Analysis/MemoryDependenceAnalysis.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp index f2d903c8e7b1..9e07b4d252b7 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp @@ -1,4 +1,4 @@ -//===-- AMDGPUAsmPrinter.cpp - AMDGPU assembly printer -------------------===// +//===-- AMDGPUAsmPrinter.cpp - AMDGPU assembly printer --------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -69,15 +69,14 @@ using namespace llvm::AMDGPU::HSAMD; // We want to use these instructions, and using fp32 denormals also causes // instructions to run at the double precision rate for the device so it's // probably best to just report no single precision denormals. -static uint32_t getFPMode(const MachineFunction &F) { - const GCNSubtarget& ST = F.getSubtarget<GCNSubtarget>(); - // TODO: Is there any real use for the flush in only / flush out only modes? +static uint32_t getFPMode(AMDGPU::SIModeRegisterDefaults Mode) { + // TODO: Is there any real use for the flush in only / flush out only modes? uint32_t FP32Denormals = - ST.hasFP32Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; + Mode.FP32Denormals ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; uint32_t FP64Denormals = - ST.hasFP64Denormals() ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; + Mode.FP64FP16Denormals ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; return FP_ROUND_MODE_SP(FP_ROUND_ROUND_TO_NEAREST) | FP_ROUND_MODE_DP(FP_ROUND_ROUND_TO_NEAREST) | @@ -91,7 +90,7 @@ createAMDGPUAsmPrinterPass(TargetMachine &tm, return new AMDGPUAsmPrinter(tm, std::move(Streamer)); } -extern "C" void LLVMInitializeAMDGPUAsmPrinter() { +extern "C" void LLVM_EXTERNAL_VISIBILITY LLVMInitializeAMDGPUAsmPrinter() { TargetRegistry::RegisterAsmPrinter(getTheAMDGPUTarget(), llvm::createR600AsmPrinterPass); TargetRegistry::RegisterAsmPrinter(getTheGCNTarget(), @@ -793,6 +792,7 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage( IsSGPR = false; Width = 3; } else if (AMDGPU::SReg_96RegClass.contains(Reg)) { + IsSGPR = true; Width = 3; } else if (AMDGPU::SReg_128RegClass.contains(Reg)) { assert(!AMDGPU::TTMP_128RegClass.contains(Reg) && @@ -806,6 +806,12 @@ AMDGPUAsmPrinter::SIFunctionResourceInfo AMDGPUAsmPrinter::analyzeResourceUsage( IsSGPR = false; IsAGPR = true; Width = 4; + } else if (AMDGPU::VReg_160RegClass.contains(Reg)) { + IsSGPR = false; + Width = 5; + } else if (AMDGPU::SReg_160RegClass.contains(Reg)) { + IsSGPR = true; + Width = 5; } else if (AMDGPU::SReg_256RegClass.contains(Reg)) { assert(!AMDGPU::TTMP_256RegClass.contains(Reg) && "trap handler registers should not be used"); @@ -1026,11 +1032,12 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, ProgInfo.VGPRBlocks = IsaInfo::getNumVGPRBlocks( &STM, ProgInfo.NumVGPRsForWavesPerEU); + const SIModeRegisterDefaults Mode = MFI->getMode(); + // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode // register. - ProgInfo.FloatMode = getFPMode(MF); + ProgInfo.FloatMode = getFPMode(Mode); - const SIModeRegisterDefaults Mode = MFI->getMode(); ProgInfo.IEEEMode = Mode.IEEE; // Make clamp modifier on NaN input returns 0. diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp index ba8343142c63..59aa0ea98aa7 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp @@ -20,6 +20,7 @@ #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" +#include "llvm/InitializePasses.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #define DEBUG_TYPE "amdgpu-atomic-optimizer" diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp index 58c44acde1a7..c657ca71bfdf 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp @@ -356,7 +356,7 @@ Register AMDGPUCallLowering::lowerParameterPtr(MachineIRBuilder &B, Register OffsetReg = MRI.createGenericVirtualRegister(LLT::scalar(64)); B.buildConstant(OffsetReg, Offset); - B.buildGEP(DstReg, KernArgSegmentVReg, OffsetReg); + B.buildPtrAdd(DstReg, KernArgSegmentVReg, OffsetReg); return DstReg; } @@ -368,8 +368,7 @@ void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &B, MachineFunction &MF = B.getMF(); const Function &F = MF.getFunction(); const DataLayout &DL = F.getParent()->getDataLayout(); - PointerType *PtrTy = PointerType::get(ParamTy, AMDGPUAS::CONSTANT_ADDRESS); - MachinePointerInfo PtrInfo(UndefValue::get(PtrTy)); + MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); unsigned TypeSize = DL.getTypeStoreSize(ParamTy); Register PtrReg = lowerParameterPtr(B, ParamTy, Offset); @@ -517,11 +516,26 @@ static void packSplitRegsToOrigType(MachineIRBuilder &B, return; } + MachineRegisterInfo &MRI = *B.getMRI(); + assert(LLTy.isVector() && !PartLLT.isVector()); LLT DstEltTy = LLTy.getElementType(); + + // Pointer information was discarded. We'll need to coerce some register types + // to avoid violating type constraints. + LLT RealDstEltTy = MRI.getType(OrigRegs[0]).getElementType(); + + assert(DstEltTy.getSizeInBits() == RealDstEltTy.getSizeInBits()); + if (DstEltTy == PartLLT) { // Vector was trivially scalarized. + + if (RealDstEltTy.isPointer()) { + for (Register Reg : Regs) + MRI.setType(Reg, RealDstEltTy); + } + B.buildBuildVector(OrigRegs[0], Regs); } else if (DstEltTy.getSizeInBits() > PartLLT.getSizeInBits()) { // Deal with vector with 64-bit elements decomposed to 32-bit @@ -532,8 +546,9 @@ static void packSplitRegsToOrigType(MachineIRBuilder &B, assert(DstEltTy.getSizeInBits() % PartLLT.getSizeInBits() == 0); for (int I = 0, NumElts = LLTy.getNumElements(); I != NumElts; ++I) { - auto Merge = B.buildMerge(DstEltTy, - Regs.take_front(PartsPerElt)); + auto Merge = B.buildMerge(RealDstEltTy, Regs.take_front(PartsPerElt)); + // Fix the type in case this is really a vector of pointers. + MRI.setType(Merge.getReg(0), RealDstEltTy); EltMerges.push_back(Merge.getReg(0)); Regs = Regs.drop_front(PartsPerElt); } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index 1640a4a59ee2..cf908766caa0 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -38,6 +38,7 @@ #include "llvm/IR/Operator.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include <cassert> @@ -69,6 +70,7 @@ class AMDGPUCodeGenPrepare : public FunctionPass, Module *Mod = nullptr; const DataLayout *DL = nullptr; bool HasUnsafeFPMath = false; + bool HasFP32Denormals = false; /// Copies exact/nsw/nuw flags (if any) from binary operation \p I to /// binary operation \p V. @@ -574,7 +576,6 @@ bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) { Value *NewFDiv = nullptr; - bool HasDenormals = ST->hasFP32Denormals(); if (VectorType *VT = dyn_cast<VectorType>(Ty)) { NewFDiv = UndefValue::get(VT); @@ -585,7 +586,7 @@ bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) { Value *DenEltI = Builder.CreateExtractElement(Den, I); Value *NewElt; - if (shouldKeepFDivF32(NumEltI, UnsafeDiv, HasDenormals)) { + if (shouldKeepFDivF32(NumEltI, UnsafeDiv, HasFP32Denormals)) { NewElt = Builder.CreateFDiv(NumEltI, DenEltI); } else { NewElt = Builder.CreateCall(Decl, { NumEltI, DenEltI }); @@ -594,7 +595,7 @@ bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) { NewFDiv = Builder.CreateInsertElement(NewFDiv, NewElt, I); } } else { - if (!shouldKeepFDivF32(Num, UnsafeDiv, HasDenormals)) + if (!shouldKeepFDivF32(Num, UnsafeDiv, HasFP32Denormals)) NewFDiv = Builder.CreateCall(Decl, { Num, Den }); } @@ -1033,6 +1034,7 @@ bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) { AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F); DA = &getAnalysis<LegacyDivergenceAnalysis>(); HasUnsafeFPMath = hasUnsafeFPMath(F); + HasFP32Denormals = ST->hasFP32Denormals(F); bool MadeChange = false; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td index f2be1ca44d34..d420aa02ac28 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGISel.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUGISel.td @@ -30,14 +30,14 @@ def gi_vop3mods : GIComplexOperandMatcher<s32, "selectVOP3Mods">, GIComplexPatternEquiv<VOP3Mods>; +def gi_vop3mods_nnan : + GIComplexOperandMatcher<s32, "selectVOP3Mods_nnan">, + GIComplexPatternEquiv<VOP3Mods_nnan>; + def gi_vop3omods : GIComplexOperandMatcher<s32, "selectVOP3OMods">, GIComplexPatternEquiv<VOP3OMods>; -def gi_vop3omods0clamp0omod : - GIComplexOperandMatcher<s32, "selectVOP3Mods0Clamp0OMod">, - GIComplexPatternEquiv<VOP3Mods0Clamp0OMod>; - def gi_vop3opselmods0 : GIComplexOperandMatcher<s32, "selectVOP3OpSelMods0">, GIComplexPatternEquiv<VOP3OpSelMods0>; @@ -117,6 +117,8 @@ def : GINodeEquiv<G_ATOMICRMW_UMAX, atomic_load_umax_glue>; def : GINodeEquiv<G_ATOMICRMW_FADD, atomic_load_fadd_glue>; def : GINodeEquiv<G_AMDGPU_FFBH_U32, AMDGPUffbh_u32>; +def : GINodeEquiv<G_AMDGPU_ATOMIC_CMPXCHG, AMDGPUatomic_cmp_swap>; + class GISelSop2Pat < SDPatternOperator node, @@ -206,3 +208,15 @@ foreach Ty = [i64, p0, p1, p4] in { def gi_as_i32timm : GICustomOperandRenderer<"renderTruncImm32">, GISDNodeXFormEquiv<as_i32timm>; + +def gi_as_i16timm : GICustomOperandRenderer<"renderTruncTImm">, + GISDNodeXFormEquiv<as_i16timm>; + +def gi_NegateImm : GICustomOperandRenderer<"renderNegateImm">, + GISDNodeXFormEquiv<NegateImm>; + +def gi_bitcast_fpimm_to_i32 : GICustomOperandRenderer<"renderBitcastImm">, + GISDNodeXFormEquiv<bitcast_fpimm_to_i32>; + +def gi_IMMPopCount : GICustomOperandRenderer<"renderPopcntImm">, + GISDNodeXFormEquiv<IMMPopCount>; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def b/llvm/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def index 85d1ad349157..2e92ae51660b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def +++ b/llvm/lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def @@ -15,29 +15,34 @@ namespace AMDGPU { enum PartialMappingIdx { None = - 1, - PM_SGPR1 = 2, - PM_SGPR16 = 6, - PM_SGPR32 = 7, - PM_SGPR64 = 8, - PM_SGPR128 = 9, - PM_SGPR256 = 10, - PM_SGPR512 = 11, - PM_SGPR1024 = 12, - PM_VGPR1 = 13, - PM_VGPR16 = 17, - PM_VGPR32 = 18, - PM_VGPR64 = 19, - PM_VGPR128 = 20, - PM_VGPR256 = 21, - PM_VGPR512 = 22, - PM_VGPR1024 = 23, - PM_SGPR96 = 24, - PM_VGPR96 = 25 + PM_SGPR1 = 1, + PM_SGPR16 = 5, + PM_SGPR32 = 6, + PM_SGPR64 = 7, + PM_SGPR128 = 8, + PM_SGPR256 = 9, + PM_SGPR512 = 10, + PM_SGPR1024 = 11, + PM_VGPR1 = 12, + PM_VGPR16 = 16, + PM_VGPR32 = 17, + PM_VGPR64 = 18, + PM_VGPR128 = 19, + PM_VGPR256 = 20, + PM_VGPR512 = 21, + PM_VGPR1024 = 22, + PM_SGPR96 = 23, + PM_VGPR96 = 24, + PM_AGPR96 = 25, + PM_AGPR32 = 31, + PM_AGPR64 = 32, + PM_AGPR128 = 33, + PM_AGPR512 = 34, + PM_AGPR1024 = 35 }; const RegisterBankInfo::PartialMapping PartMappings[] { // StartIdx, Length, RegBank - {0, 1, SCCRegBank}, {0, 1, VCCRegBank}, {0, 1, SGPRRegBank}, // SGPR begin @@ -58,43 +63,61 @@ const RegisterBankInfo::PartialMapping PartMappings[] { {0, 512, VGPRRegBank}, {0, 1024, VGPRRegBank}, {0, 96, SGPRRegBank}, - {0, 96, VGPRRegBank} + {0, 96, VGPRRegBank}, + {0, 96, AGPRRegBank}, + + {0, 32, AGPRRegBank}, // AGPR begin + {0, 64, AGPRRegBank}, + {0, 128, AGPRRegBank}, + {0, 512, AGPRRegBank}, + {0, 1024, AGPRRegBank} }; const RegisterBankInfo::ValueMapping ValMappings[] { - // SCC - {&PartMappings[0], 1}, - // VCC - {&PartMappings[1], 1}, + {&PartMappings[0], 1}, // SGPRs - {&PartMappings[2], 1}, // 1 + {&PartMappings[1], 1}, // 1 {nullptr, 0}, // Illegal power of 2 sizes {nullptr, 0}, {nullptr, 0}, - {&PartMappings[3], 1}, // 16 - {&PartMappings[4], 1}, // 32 - {&PartMappings[5], 1}, // 64 - {&PartMappings[6], 1}, // 128 - {&PartMappings[7], 1}, // 256 - {&PartMappings[8], 1}, // 512 - {&PartMappings[9], 1}, // 1024 + {&PartMappings[2], 1}, // 16 + {&PartMappings[3], 1}, // 32 + {&PartMappings[4], 1}, // 64 + {&PartMappings[5], 1}, // 128 + {&PartMappings[6], 1}, // 256 + {&PartMappings[7], 1}, // 512 + {&PartMappings[8], 1}, // 1024 // VGPRs - {&PartMappings[10], 1}, // 1 + {&PartMappings[9], 1}, // 1 {nullptr, 0}, {nullptr, 0}, {nullptr, 0}, - {&PartMappings[11], 1}, // 16 - {&PartMappings[12], 1}, // 32 - {&PartMappings[13], 1}, // 64 - {&PartMappings[14], 1}, // 128 - {&PartMappings[15], 1}, // 256 - {&PartMappings[16], 1}, // 512 - {&PartMappings[17], 1}, // 1024 + {&PartMappings[10], 1}, // 16 + {&PartMappings[11], 1}, // 32 + {&PartMappings[12], 1}, // 64 + {&PartMappings[13], 1}, // 128 + {&PartMappings[14], 1}, // 256 + {&PartMappings[15], 1}, // 512 + {&PartMappings[16], 1}, // 1024 + {&PartMappings[17], 1}, {&PartMappings[18], 1}, - {&PartMappings[19], 1} + {&PartMappings[19], 1}, + + // AGPRs + {nullptr, 0}, + {nullptr, 0}, + {nullptr, 0}, + {nullptr, 0}, + {nullptr, 0}, + {&PartMappings[20], 1}, // 32 + {&PartMappings[21], 1}, // 64 + {&PartMappings[22], 1}, // 128 + {nullptr, 0}, + {&PartMappings[23], 1}, // 512 + {&PartMappings[24], 1} // 1024 }; const RegisterBankInfo::PartialMapping SGPROnly64BreakDown[] { @@ -120,9 +143,9 @@ const RegisterBankInfo::ValueMapping ValMappingsSGPR64OnlyVGPR32[] { }; enum ValueMappingIdx { - SCCStartIdx = 0, - SGPRStartIdx = 2, - VGPRStartIdx = 13 + SGPRStartIdx = 1, + VGPRStartIdx = 12, + AGPRStartIdx = 26 }; const RegisterBankInfo::ValueMapping *getValueMapping(unsigned BankID, @@ -130,21 +153,38 @@ const RegisterBankInfo::ValueMapping *getValueMapping(unsigned BankID, unsigned Idx; switch (Size) { case 1: - if (BankID == AMDGPU::SCCRegBankID) - return &ValMappings[0]; if (BankID == AMDGPU::VCCRegBankID) - return &ValMappings[1]; + return &ValMappings[0]; - // 1-bit values not from a compare etc. Idx = BankID == AMDGPU::SGPRRegBankID ? PM_SGPR1 : PM_VGPR1; break; case 96: - assert(BankID != AMDGPU::VCCRegBankID); - Idx = BankID == AMDGPU::SGPRRegBankID ? PM_SGPR96 : PM_VGPR96; + switch (BankID) { + case AMDGPU::VGPRRegBankID: + Idx = PM_VGPR96; + break; + case AMDGPU::SGPRRegBankID: + Idx = PM_SGPR96; + break; + case AMDGPU::AGPRRegBankID: + Idx = PM_AGPR96; + break; + default: llvm_unreachable("Invalid register bank"); + } break; default: - assert(BankID != AMDGPU::VCCRegBankID); - Idx = BankID == AMDGPU::VGPRRegBankID ? VGPRStartIdx : SGPRStartIdx; + switch (BankID) { + case AMDGPU::VGPRRegBankID: + Idx = VGPRStartIdx; + break; + case AMDGPU::SGPRRegBankID: + Idx = SGPRStartIdx; + break; + case AMDGPU::AGPRRegBankID: + Idx = AGPRStartIdx; + break; + default: llvm_unreachable("Invalid register bank"); + } Idx += Log2_32_Ceil(Size); break; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp new file mode 100644 index 000000000000..16d7f2c4f9e5 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.cpp @@ -0,0 +1,45 @@ +//===- AMDGPUGlobalISelUtils.cpp ---------------------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "AMDGPUGlobalISelUtils.h" +#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" +#include "llvm/IR/Constants.h" + +using namespace llvm; +using namespace MIPatternMatch; + +std::tuple<Register, unsigned, MachineInstr *> +AMDGPU::getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) { + MachineInstr *Def = getDefIgnoringCopies(Reg, MRI); + if (!Def) + return std::make_tuple(Reg, 0, nullptr); + + if (Def->getOpcode() == TargetOpcode::G_CONSTANT) { + unsigned Offset; + const MachineOperand &Op = Def->getOperand(1); + if (Op.isImm()) + Offset = Op.getImm(); + else + Offset = Op.getCImm()->getZExtValue(); + + return std::make_tuple(Register(), Offset, Def); + } + + int64_t Offset; + if (Def->getOpcode() == TargetOpcode::G_ADD) { + // TODO: Handle G_OR used for add case + if (mi_match(Def->getOperand(2).getReg(), MRI, m_ICst(Offset))) + return std::make_tuple(Def->getOperand(1).getReg(), Offset, Def); + + // FIXME: matcher should ignore copies + if (mi_match(Def->getOperand(2).getReg(), MRI, m_Copy(m_ICst(Offset)))) + return std::make_tuple(Def->getOperand(1).getReg(), Offset, Def); + } + + return std::make_tuple(Reg, 0, Def); +} diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h new file mode 100644 index 000000000000..1507ade79547 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h @@ -0,0 +1,29 @@ +//===- AMDGPUGlobalISelUtils -------------------------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H +#define LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H + +#include "llvm/CodeGen/Register.h" +#include <tuple> + +namespace llvm { + +class MachineInstr; +class MachineRegisterInfo; + +namespace AMDGPU { + +/// Returns Base register, constant offset, and offset def point. +std::tuple<Register, unsigned, MachineInstr *> +getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg); + +} +} + +#endif diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp index 9f5bcd8ff5f0..511d62943189 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp @@ -421,7 +421,12 @@ void MetadataStreamerV2::emitHiddenKernelArgs(const Function &Func) { if (HiddenArgNumBytes >= 32) { if (Func.getParent()->getNamedMetadata("llvm.printf.fmts")) emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenPrintfBuffer); - else + else if (Func.getParent()->getFunction("__ockl_hostcall_internal")) { + // The printf runtime binding pass should have ensured that hostcall and + // printf are not used in the same module. + assert(!Func.getParent()->getNamedMetadata("llvm.printf.fmts")); + emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenHostcallBuffer); + } else emitKernelArg(DL, Int8PtrTy, ValueKind::HiddenNone); } @@ -854,7 +859,12 @@ void MetadataStreamerV3::emitHiddenKernelArgs(const Function &Func, if (HiddenArgNumBytes >= 32) { if (Func.getParent()->getNamedMetadata("llvm.printf.fmts")) emitKernelArg(DL, Int8PtrTy, "hidden_printf_buffer", Offset, Args); - else + else if (Func.getParent()->getFunction("__ockl_hostcall_internal")) { + // The printf runtime binding pass should have ensured that hostcall and + // printf are not used in the same module. + assert(!Func.getParent()->getNamedMetadata("llvm.printf.fmts")); + emitKernelArg(DL, Int8PtrTy, "hidden_hostcall_buffer", Offset, Args); + } else emitKernelArg(DL, Int8PtrTy, "hidden_none", Offset, Args); } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index f330bd7ebcdd..2b6308dc1549 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -19,12 +19,12 @@ #include "AMDGPURegisterInfo.h" #include "AMDGPUSubtarget.h" #include "AMDGPUTargetMachine.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIDefines.h" #include "SIISelLowering.h" #include "SIInstrInfo.h" #include "SIMachineFunctionInfo.h" #include "SIRegisterInfo.h" -#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" @@ -39,6 +39,7 @@ #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/BasicBlock.h" +#include "llvm/InitializePasses.h" #ifdef EXPENSIVE_CHECKS #include "llvm/IR/Dominators.h" #endif @@ -127,6 +128,10 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel { // Subtarget - Keep a pointer to the AMDGPU Subtarget around so that we can // make the right decision when generating code for different targets. const GCNSubtarget *Subtarget; + + // Default FP mode for the current function. + AMDGPU::SIModeRegisterDefaults Mode; + bool EnableLateStructurizeCFG; public: @@ -166,6 +171,22 @@ private: return isInlineImmediate(N, true); } + bool isInlineImmediate16(int64_t Imm) const { + return AMDGPU::isInlinableLiteral16(Imm, Subtarget->hasInv2PiInlineImm()); + } + + bool isInlineImmediate32(int64_t Imm) const { + return AMDGPU::isInlinableLiteral32(Imm, Subtarget->hasInv2PiInlineImm()); + } + + bool isInlineImmediate64(int64_t Imm) const { + return AMDGPU::isInlinableLiteral64(Imm, Subtarget->hasInv2PiInlineImm()); + } + + bool isInlineImmediate(const APFloat &Imm) const { + return Subtarget->getInstrInfo()->isInlineConstant(Imm); + } + bool isVGPRImm(const SDNode *N) const; bool isUniformLoad(const SDNode *N) const; bool isUniformBr(const SDNode *N) const; @@ -240,10 +261,6 @@ private: bool SelectVOP3NoMods0(SDValue In, SDValue &Src, SDValue &SrcMods, SDValue &Clamp, SDValue &Omod) const; - bool SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, SDValue &SrcMods, - SDValue &Clamp, - SDValue &Omod) const; - bool SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp, SDValue &Omod) const; @@ -392,6 +409,7 @@ bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { } #endif Subtarget = &MF.getSubtarget<GCNSubtarget>(); + Mode = AMDGPU::SIModeRegisterDefaults(MF.getFunction(), *Subtarget); return SelectionDAGISel::runOnMachineFunction(MF); } @@ -2103,7 +2121,7 @@ void AMDGPUDAGToDAGISel::SelectFMAD_FMA(SDNode *N) { bool Sel1 = SelectVOP3PMadMixModsImpl(Src1, Src1, Src1Mods); bool Sel2 = SelectVOP3PMadMixModsImpl(Src2, Src2, Src2Mods); - assert((IsFMA || !Subtarget->hasFP32Denormals()) && + assert((IsFMA || !Mode.FP32Denormals) && "fmad selected with denormals enabled"); // TODO: We can select this with f32 denormals enabled if all the sources are // converted from f16 (in which case fmad isn't legal). @@ -2437,14 +2455,6 @@ bool AMDGPUDAGToDAGISel::SelectVOP3Mods0(SDValue In, SDValue &Src, return SelectVOP3Mods(In, Src, SrcMods); } -bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, - SDValue &SrcMods, - SDValue &Clamp, - SDValue &Omod) const { - Clamp = Omod = CurDAG->getTargetConstant(0, SDLoc(In), MVT::i32); - return SelectVOP3Mods(In, Src, SrcMods); -} - bool AMDGPUDAGToDAGISel::SelectVOP3OMods(SDValue In, SDValue &Src, SDValue &Clamp, SDValue &Omod) const { Src = In; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 1115d8c23620..23cc9404532d 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -451,9 +451,6 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I) setLibcallName(static_cast<RTLIB::Libcall>(I), nullptr); - setBooleanContents(ZeroOrNegativeOneBooleanContent); - setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); - setSchedulingPreference(Sched::RegPressure); setJumpIsExpensive(true); @@ -1399,16 +1396,19 @@ SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op, SelectionDAG &DAG) const { LoadSDNode *Load = cast<LoadSDNode>(Op); EVT VT = Op.getValueType(); + SDLoc SL(Op); // If this is a 2 element vector, we really want to scalarize and not create // weird 1 element vectors. - if (VT.getVectorNumElements() == 2) - return scalarizeVectorLoad(Load, DAG); + if (VT.getVectorNumElements() == 2) { + SDValue Ops[2]; + std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG); + return DAG.getMergeValues(Ops, SL); + } SDValue BasePtr = Load->getBasePtr(); EVT MemVT = Load->getMemoryVT(); - SDLoc SL(Op); const MachinePointerInfo &SrcValue = Load->getMemOperand()->getPointerInfo(); @@ -1584,8 +1584,11 @@ SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG, // float fqneg = -fq; SDValue fqneg = DAG.getNode(ISD::FNEG, DL, FltVT, fq); + MachineFunction &MF = DAG.getMachineFunction(); + const AMDGPUMachineFunction *MFI = MF.getInfo<AMDGPUMachineFunction>(); + // float fr = mad(fqneg, fb, fa); - unsigned OpCode = Subtarget->hasFP32Denormals() ? + unsigned OpCode = MFI->getMode().FP32Denormals ? (unsigned)AMDGPUISD::FMAD_FTZ : (unsigned)ISD::FMAD; SDValue fr = DAG.getNode(OpCode, DL, FltVT, fqneg, fb, fa); @@ -1666,8 +1669,11 @@ void AMDGPUTargetLowering::LowerUDIVREM64(SDValue Op, } if (isTypeLegal(MVT::i64)) { + MachineFunction &MF = DAG.getMachineFunction(); + const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); + // Compute denominator reciprocal. - unsigned FMAD = Subtarget->hasFP32Denormals() ? + unsigned FMAD = MFI->getMode().FP32Denormals ? (unsigned)AMDGPUISD::FMAD_FTZ : (unsigned)ISD::FMAD; @@ -2158,7 +2164,8 @@ SDValue AMDGPUTargetLowering::LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) con // Don't handle v2f16. The extra instructions to scalarize and repack around the // compare and vselect end up producing worse code than scalarizing the whole // operation. -SDValue AMDGPUTargetLowering::LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const { +SDValue AMDGPUTargetLowering::LowerFROUND_LegalFTRUNC(SDValue Op, + SelectionDAG &DAG) const { SDLoc SL(Op); SDValue X = Op.getOperand(0); EVT VT = Op.getValueType(); @@ -2247,8 +2254,8 @@ SDValue AMDGPUTargetLowering::LowerFROUND64(SDValue Op, SelectionDAG &DAG) const SDValue AMDGPUTargetLowering::LowerFROUND(SDValue Op, SelectionDAG &DAG) const { EVT VT = Op.getValueType(); - if (VT == MVT::f32 || VT == MVT::f16) - return LowerFROUND32_16(Op, DAG); + if (isOperationLegal(ISD::FTRUNC, VT)) + return LowerFROUND_LegalFTRUNC(Op, DAG); if (VT == MVT::f64) return LowerFROUND64(Op, DAG); @@ -2496,15 +2503,25 @@ SDValue AMDGPUTargetLowering::LowerINT_TO_FP64(SDValue Op, SelectionDAG &DAG, SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const { - assert(Op.getOperand(0).getValueType() == MVT::i64 && - "operation should be legal"); - // TODO: Factor out code common with LowerSINT_TO_FP. - EVT DestVT = Op.getValueType(); + SDValue Src = Op.getOperand(0); + EVT SrcVT = Src.getValueType(); + + if (SrcVT == MVT::i16) { + if (DestVT == MVT::f16) + return Op; + SDLoc DL(Op); + + // Promote src to i32 + SDValue Ext = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Src); + return DAG.getNode(ISD::UINT_TO_FP, DL, DestVT, Ext); + } + + assert(SrcVT == MVT::i64 && "operation should be legal"); + if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { SDLoc DL(Op); - SDValue Src = Op.getOperand(0); SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op)); @@ -2523,12 +2540,25 @@ SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op, SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const { - assert(Op.getOperand(0).getValueType() == MVT::i64 && - "operation should be legal"); + EVT DestVT = Op.getValueType(); + + SDValue Src = Op.getOperand(0); + EVT SrcVT = Src.getValueType(); + + if (SrcVT == MVT::i16) { + if (DestVT == MVT::f16) + return Op; + + SDLoc DL(Op); + // Promote src to i32 + SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, DL, MVT::i32, Src); + return DAG.getNode(ISD::SINT_TO_FP, DL, DestVT, Ext); + } + + assert(SrcVT == MVT::i64 && "operation should be legal"); // TODO: Factor out code common with LowerUINT_TO_FP. - EVT DestVT = Op.getValueType(); if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { SDLoc DL(Op); SDValue Src = Op.getOperand(0); @@ -2865,11 +2895,13 @@ SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N, // the bytes again are not eliminated in the case of an unaligned copy. if (!allowsMisalignedMemoryAccesses( VT, AS, Align, LN->getMemOperand()->getFlags(), &IsFast)) { + SDValue Ops[2]; + if (VT.isVector()) - return scalarizeVectorLoad(LN, DAG); + std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(LN, DAG); + else + std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG); - SDValue Ops[2]; - std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG); return DAG.getMergeValues(Ops, SDLoc(N)); } @@ -3565,8 +3597,8 @@ SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N, // select (setcc x, y), k, x -> select (setccinv x, y), x, k SDLoc SL(N); - ISD::CondCode NewCC = getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), - LHS.getValueType().isInteger()); + ISD::CondCode NewCC = + getSetCCInverse(cast<CondCodeSDNode>(CC)->get(), LHS.getValueType()); SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC); return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True); @@ -4343,7 +4375,6 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(BUFFER_ATOMIC_CMPSWAP) NODE_NAME_CASE(BUFFER_ATOMIC_FADD) NODE_NAME_CASE(BUFFER_ATOMIC_PK_FADD) - NODE_NAME_CASE(ATOMIC_FADD) NODE_NAME_CASE(ATOMIC_PK_FADD) case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break; @@ -4435,12 +4466,14 @@ void AMDGPUTargetLowering::computeKnownBitsForTargetNode( unsigned TrailZ = LHSKnown.countMinTrailingZeros() + RHSKnown.countMinTrailingZeros(); Known.Zero.setLowBits(std::min(TrailZ, 32u)); + // Skip extra check if all bits are known zeros. + if (TrailZ >= 32) + break; // Truncate to 24 bits. LHSKnown = LHSKnown.trunc(24); RHSKnown = RHSKnown.trunc(24); - bool Negative = false; if (Opc == AMDGPUISD::MUL_I24) { unsigned LHSValBits = 24 - LHSKnown.countMinSignBits(); unsigned RHSValBits = 24 - RHSKnown.countMinSignBits(); @@ -4448,16 +4481,16 @@ void AMDGPUTargetLowering::computeKnownBitsForTargetNode( if (MaxValBits >= 32) break; bool LHSNegative = LHSKnown.isNegative(); - bool LHSPositive = LHSKnown.isNonNegative(); + bool LHSNonNegative = LHSKnown.isNonNegative(); + bool LHSPositive = LHSKnown.isStrictlyPositive(); bool RHSNegative = RHSKnown.isNegative(); - bool RHSPositive = RHSKnown.isNonNegative(); - if ((!LHSNegative && !LHSPositive) || (!RHSNegative && !RHSPositive)) - break; - Negative = (LHSNegative && RHSPositive) || (LHSPositive && RHSNegative); - if (Negative) - Known.One.setHighBits(32 - MaxValBits); - else + bool RHSNonNegative = RHSKnown.isNonNegative(); + bool RHSPositive = RHSKnown.isStrictlyPositive(); + + if ((LHSNonNegative && RHSNonNegative) || (LHSNegative && RHSNegative)) Known.Zero.setHighBits(32 - MaxValBits); + else if ((LHSNegative && RHSPositive) || (LHSPositive && RHSNegative)) + Known.One.setHighBits(32 - MaxValBits); } else { unsigned LHSValBits = 24 - LHSKnown.countMinLeadingZeros(); unsigned RHSValBits = 24 - RHSKnown.countMinLeadingZeros(); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h index dea0d1d4343a..a90b7f5653dc 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h @@ -52,7 +52,7 @@ protected: SDValue LowerFRINT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFNEARBYINT(SDValue Op, SelectionDAG &DAG) const; - SDValue LowerFROUND32_16(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerFROUND_LegalFTRUNC(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFROUND(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFFLOOR(SDValue Op, SelectionDAG &DAG) const; @@ -531,7 +531,6 @@ enum NodeType : unsigned { BUFFER_ATOMIC_CMPSWAP, BUFFER_ATOMIC_FADD, BUFFER_ATOMIC_PK_FADD, - ATOMIC_FADD, ATOMIC_PK_FADD, LAST_AMDGPU_ISD_NUMBER diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp index a83ec23ec054..64d761997b0c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInline.cpp @@ -17,21 +17,21 @@ /// //===----------------------------------------------------------------------===// - #include "AMDGPU.h" -#include "llvm/Transforms/IPO.h" #include "llvm/Analysis/AssumptionCache.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/InlineCost.h" -#include "llvm/Analysis/ValueTracking.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" +#include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/Inliner.h" using namespace llvm; @@ -39,7 +39,7 @@ using namespace llvm; #define DEBUG_TYPE "inline" static cl::opt<int> -ArgAllocaCost("amdgpu-inline-arg-alloca-cost", cl::Hidden, cl::init(1500), +ArgAllocaCost("amdgpu-inline-arg-alloca-cost", cl::Hidden, cl::init(4000), cl::desc("Cost of alloca argument")); // If the amount of scratch memory to eliminate exceeds our ability to allocate diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td index cf0ce5659951..50c451be4b86 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td @@ -244,9 +244,9 @@ def AMDGPUdiv_fmas : SDNode<"AMDGPUISD::DIV_FMAS", AMDGPUFmasOp, // Single or double precision division fixup. // Special case divide fixup and flags(src0 = Quotient, src1 = // Denominator, src2 = Numerator). -def AMDGPUdiv_fixup : SDNode<"AMDGPUISD::DIV_FIXUP", SDTFPTernaryOp>; +def AMDGPUdiv_fixup_impl : SDNode<"AMDGPUISD::DIV_FIXUP", SDTFPTernaryOp>; -def AMDGPUfmad_ftz : SDNode<"AMDGPUISD::FMAD_FTZ", SDTFPTernaryOp>; +def AMDGPUfmad_ftz_impl : SDNode<"AMDGPUISD::FMAD_FTZ", SDTFPTernaryOp>; // Look Up 2.0 / pi src0 with segment select src1[4:0] def AMDGPUtrig_preop : SDNode<"AMDGPUISD::TRIG_PREOP", AMDGPUTrigPreOp>; @@ -290,10 +290,10 @@ def AMDGPUffbl_b32 : SDNode<"AMDGPUISD::FFBL_B32", SDTIntUnaryOp>; // Signed and unsigned 24-bit multiply. The highest 8-bits are ignore // when performing the mulitply. The result is a 32-bit value. -def AMDGPUmul_u24 : SDNode<"AMDGPUISD::MUL_U24", SDTIntBinOp, +def AMDGPUmul_u24_impl : SDNode<"AMDGPUISD::MUL_U24", SDTIntBinOp, [SDNPCommutative, SDNPAssociative] >; -def AMDGPUmul_i24 : SDNode<"AMDGPUISD::MUL_I24", SDTIntBinOp, +def AMDGPUmul_i24_impl : SDNode<"AMDGPUISD::MUL_I24", SDTIntBinOp, [SDNPCommutative, SDNPAssociative] >; @@ -434,6 +434,10 @@ def AMDGPUfmed3 : PatFrags<(ops node:$src0, node:$src1, node:$src2), [(int_amdgcn_fmed3 node:$src0, node:$src1, node:$src2), (AMDGPUfmed3_impl node:$src0, node:$src1, node:$src2)]>; +def AMDGPUdiv_fixup : PatFrags<(ops node:$src0, node:$src1, node:$src2), + [(int_amdgcn_div_fixup node:$src0, node:$src1, node:$src2), + (AMDGPUdiv_fixup_impl node:$src0, node:$src1, node:$src2)]>; + def AMDGPUffbh_i32 : PatFrags<(ops node:$src), [(int_amdgcn_sffbh node:$src), (AMDGPUffbh_i32_impl node:$src)]>; @@ -457,3 +461,15 @@ def AMDGPUpk_i16_i32 : PatFrags<(ops node:$src0, node:$src1), def AMDGPUpk_u16_u32 : PatFrags<(ops node:$src0, node:$src1), [(int_amdgcn_cvt_pk_u16 node:$src0, node:$src1), (AMDGPUpk_u16_u32_impl node:$src0, node:$src1)]>; + +def AMDGPUfmad_ftz : PatFrags<(ops node:$src0, node:$src1, node:$src2), + [(int_amdgcn_fmad_ftz node:$src0, node:$src1, node:$src2), + (AMDGPUfmad_ftz_impl node:$src0, node:$src1, node:$src2)]>; + +def AMDGPUmul_u24 : PatFrags<(ops node:$src0, node:$src1), + [(int_amdgcn_mul_u24 node:$src0, node:$src1), + (AMDGPUmul_u24_impl node:$src0, node:$src1)]>; + +def AMDGPUmul_i24 : PatFrags<(ops node:$src0, node:$src1), + [(int_amdgcn_mul_i24 node:$src0, node:$src1), + (AMDGPUmul_i24_impl node:$src0, node:$src1)]>; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp index 3cfa9d57ec46..c0ea35817ec8 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp @@ -13,6 +13,7 @@ #include "AMDGPUInstructionSelector.h" #include "AMDGPUInstrInfo.h" +#include "AMDGPUGlobalISelUtils.h" #include "AMDGPURegisterBankInfo.h" #include "AMDGPURegisterInfo.h" #include "AMDGPUSubtarget.h" @@ -69,28 +70,6 @@ void AMDGPUInstructionSelector::setupMF(MachineFunction &MF, GISelKnownBits &KB, InstructionSelector::setupMF(MF, KB, CoverageInfo); } -static bool isSCC(Register Reg, const MachineRegisterInfo &MRI) { - if (Register::isPhysicalRegister(Reg)) - return Reg == AMDGPU::SCC; - - auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); - const TargetRegisterClass *RC = - RegClassOrBank.dyn_cast<const TargetRegisterClass*>(); - if (RC) { - // FIXME: This is ambiguous for wave32. This could be SCC or VCC, but the - // context of the register bank has been lost. - // Has a hack getRegClassForSizeOnBank uses exactly SGPR_32RegClass, which - // won't ever beconstrained any further. - if (RC != &AMDGPU::SGPR_32RegClass) - return false; - const LLT Ty = MRI.getType(Reg); - return Ty.isValid() && Ty.getSizeInBits() == 1; - } - - const RegisterBank *RB = RegClassOrBank.get<const RegisterBank *>(); - return RB->getID() == AMDGPU::SCCRegBankID; -} - bool AMDGPUInstructionSelector::isVCC(Register Reg, const MachineRegisterInfo &MRI) const { if (Register::isPhysicalRegister(Reg)) @@ -133,12 +112,26 @@ bool AMDGPUInstructionSelector::selectCOPY(MachineInstr &I) const { if (!RBI.constrainGenericRegister(DstReg, *TRI.getBoolRC(), *MRI)) return false; + const TargetRegisterClass *SrcRC + = TRI.getConstrainedRegClassForOperand(Src, *MRI); + + Register MaskedReg = MRI->createVirtualRegister(SrcRC); + + // We can't trust the high bits at this point, so clear them. + + // TODO: Skip masking high bits if def is known boolean. + + unsigned AndOpc = TRI.isSGPRClass(SrcRC) ? + AMDGPU::S_AND_B32 : AMDGPU::V_AND_B32_e32; + BuildMI(*BB, &I, DL, TII.get(AndOpc), MaskedReg) + .addImm(1) + .addReg(SrcReg); BuildMI(*BB, &I, DL, TII.get(AMDGPU::V_CMP_NE_U32_e64), DstReg) .addImm(0) - .addReg(SrcReg); + .addReg(MaskedReg); if (!MRI->getRegClassOrNull(SrcReg)) - MRI->setRegClass(SrcReg, TRI.getConstrainedRegClassForOperand(Src, *MRI)); + MRI->setRegClass(SrcReg, SrcRC); I.eraseFromParent(); return true; } @@ -195,11 +188,6 @@ bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const { } const RegisterBank &RB = *RegClassOrBank.get<const RegisterBank *>(); - if (RB.getID() == AMDGPU::SCCRegBankID) { - LLVM_DEBUG(dbgs() << "illegal scc phi\n"); - return false; - } - DefRC = TRI.getRegClassForTypeOnBank(DefTy, RB, *MRI); if (!DefRC) { LLVM_DEBUG(dbgs() << "PHI operand has unexpected size/bank\n"); @@ -207,6 +195,7 @@ bool AMDGPUInstructionSelector::selectPHI(MachineInstr &I) const { } } + // TODO: Verify that all registers have the same bank I.setDesc(TII.get(TargetOpcode::PHI)); return RBI.constrainGenericRegister(DefReg, *DefRC, *MRI); } @@ -290,6 +279,11 @@ bool AMDGPUInstructionSelector::selectG_AND_OR_XOR(MachineInstr &I) const { if (DstRB->getID() == AMDGPU::SGPRRegBankID) { unsigned InstOpc = getLogicalBitOpcode(I.getOpcode(), Size > 32); I.setDesc(TII.get(InstOpc)); + // Dead implicit-def of scc + I.addOperand(MachineOperand::CreateReg(AMDGPU::SCC, true, // isDef + true, // isImp + false, // isKill + true)); // isDead return constrainSelectedInstRegOperands(I, TII, TRI, RBI); } @@ -393,21 +387,25 @@ bool AMDGPUInstructionSelector::selectG_ADD_SUB(MachineInstr &I) const { return true; } -bool AMDGPUInstructionSelector::selectG_UADDO_USUBO(MachineInstr &I) const { +bool AMDGPUInstructionSelector::selectG_UADDO_USUBO_UADDE_USUBE( + MachineInstr &I) const { MachineBasicBlock *BB = I.getParent(); MachineFunction *MF = BB->getParent(); - MachineRegisterInfo &MRI = MF->getRegInfo(); const DebugLoc &DL = I.getDebugLoc(); Register Dst0Reg = I.getOperand(0).getReg(); Register Dst1Reg = I.getOperand(1).getReg(); - const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO; + const bool IsAdd = I.getOpcode() == AMDGPU::G_UADDO || + I.getOpcode() == AMDGPU::G_UADDE; + const bool HasCarryIn = I.getOpcode() == AMDGPU::G_UADDE || + I.getOpcode() == AMDGPU::G_USUBE; - if (!isSCC(Dst1Reg, MRI)) { - // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned - // carry out despite the _i32 name. These were renamed in VI to _U32. - // FIXME: We should probably rename the opcodes here. - unsigned NewOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64; - I.setDesc(TII.get(NewOpc)); + if (isVCC(Dst1Reg, *MRI)) { + // The name of the opcodes are misleading. v_add_i32/v_sub_i32 have unsigned + // carry out despite the _i32 name. These were renamed in VI to _U32. + // FIXME: We should probably rename the opcodes here. + unsigned NoCarryOpc = IsAdd ? AMDGPU::V_ADD_I32_e64 : AMDGPU::V_SUB_I32_e64; + unsigned CarryOpc = IsAdd ? AMDGPU::V_ADDC_U32_e64 : AMDGPU::V_SUBB_U32_e64; + I.setDesc(TII.get(HasCarryIn ? CarryOpc : NoCarryOpc)); I.addOperand(*MF, MachineOperand::CreateReg(AMDGPU::EXEC, false, true)); I.addOperand(*MF, MachineOperand::CreateImm(0)); return constrainSelectedInstRegOperands(I, TII, TRI, RBI); @@ -415,19 +413,32 @@ bool AMDGPUInstructionSelector::selectG_UADDO_USUBO(MachineInstr &I) const { Register Src0Reg = I.getOperand(2).getReg(); Register Src1Reg = I.getOperand(3).getReg(); - unsigned NewOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; - BuildMI(*BB, &I, DL, TII.get(NewOpc), Dst0Reg) + + if (HasCarryIn) { + BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) + .addReg(I.getOperand(4).getReg()); + } + + unsigned NoCarryOpc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; + unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; + + BuildMI(*BB, &I, DL, TII.get(HasCarryIn ? CarryOpc : NoCarryOpc), Dst0Reg) .add(I.getOperand(2)) .add(I.getOperand(3)); BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), Dst1Reg) .addReg(AMDGPU::SCC); - if (!MRI.getRegClassOrNull(Dst1Reg)) - MRI.setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); + if (!MRI->getRegClassOrNull(Dst1Reg)) + MRI->setRegClass(Dst1Reg, &AMDGPU::SReg_32RegClass); + + if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, *MRI) || + !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, *MRI) || + !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, *MRI)) + return false; - if (!RBI.constrainGenericRegister(Dst0Reg, AMDGPU::SReg_32RegClass, MRI) || - !RBI.constrainGenericRegister(Src0Reg, AMDGPU::SReg_32RegClass, MRI) || - !RBI.constrainGenericRegister(Src1Reg, AMDGPU::SReg_32RegClass, MRI)) + if (HasCarryIn && + !RBI.constrainGenericRegister(I.getOperand(4).getReg(), + AMDGPU::SReg_32RegClass, *MRI)) return false; I.eraseFromParent(); @@ -436,15 +447,29 @@ bool AMDGPUInstructionSelector::selectG_UADDO_USUBO(MachineInstr &I) const { bool AMDGPUInstructionSelector::selectG_EXTRACT(MachineInstr &I) const { MachineBasicBlock *BB = I.getParent(); + Register DstReg = I.getOperand(0).getReg(); + Register SrcReg = I.getOperand(1).getReg(); + LLT DstTy = MRI->getType(DstReg); + LLT SrcTy = MRI->getType(SrcReg); + const unsigned SrcSize = SrcTy.getSizeInBits(); + const unsigned DstSize = DstTy.getSizeInBits(); + + // TODO: Should handle any multiple of 32 offset. unsigned Offset = I.getOperand(2).getImm(); - if (Offset % 32 != 0) + if (Offset % DstSize != 0) return false; - unsigned SubReg = TRI.getSubRegFromChannel(Offset / 32); + const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); + const TargetRegisterClass *SrcRC = + TRI.getRegClassForSizeOnBank(SrcSize, *SrcBank, *MRI); + if (!SrcRC) + return false; + + ArrayRef<int16_t> SubRegs = TRI.getRegSplitParts(SrcRC, DstSize / 8); + const DebugLoc &DL = I.getDebugLoc(); - MachineInstr *Copy = BuildMI(*BB, &I, DL, TII.get(TargetOpcode::COPY), - I.getOperand(0).getReg()) - .addReg(I.getOperand(1).getReg(), 0, SubReg); + MachineInstr *Copy = BuildMI(*BB, &I, DL, TII.get(TargetOpcode::COPY), DstReg) + .addReg(SrcReg, 0, SubRegs[Offset / DstSize]); for (const MachineOperand &MO : Copy->operands()) { const TargetRegisterClass *RC = @@ -465,7 +490,7 @@ bool AMDGPUInstructionSelector::selectG_MERGE_VALUES(MachineInstr &MI) const { const unsigned SrcSize = SrcTy.getSizeInBits(); if (SrcSize < 32) - return false; + return selectImpl(MI, *CoverageInfo); const DebugLoc &DL = MI.getDebugLoc(); const RegisterBank *DstBank = RBI.getRegBank(DstReg, *MRI, TRI); @@ -538,7 +563,7 @@ bool AMDGPUInstructionSelector::selectG_UNMERGE_VALUES(MachineInstr &MI) const { return true; } -bool AMDGPUInstructionSelector::selectG_GEP(MachineInstr &I) const { +bool AMDGPUInstructionSelector::selectG_PTR_ADD(MachineInstr &I) const { return selectG_ADD_SUB(I); } @@ -723,7 +748,7 @@ bool AMDGPUInstructionSelector::selectG_ICMP(MachineInstr &I) const { auto Pred = (CmpInst::Predicate)I.getOperand(1).getPredicate(); Register CCReg = I.getOperand(0).getReg(); - if (isSCC(CCReg, *MRI)) { + if (!isVCC(CCReg, *MRI)) { int Opcode = getS_CMPOpcode(Pred, Size); if (Opcode == -1) return false; @@ -797,38 +822,6 @@ static unsigned extractSWZ(unsigned AuxiliaryData) { return (AuxiliaryData >> 3) & 1; } -// Returns Base register, constant offset, and offset def point. -static std::tuple<Register, unsigned, MachineInstr *> -getBaseWithConstantOffset(MachineRegisterInfo &MRI, Register Reg) { - MachineInstr *Def = getDefIgnoringCopies(Reg, MRI); - if (!Def) - return std::make_tuple(Reg, 0, nullptr); - - if (Def->getOpcode() == AMDGPU::G_CONSTANT) { - unsigned Offset; - const MachineOperand &Op = Def->getOperand(1); - if (Op.isImm()) - Offset = Op.getImm(); - else - Offset = Op.getCImm()->getZExtValue(); - - return std::make_tuple(Register(), Offset, Def); - } - - int64_t Offset; - if (Def->getOpcode() == AMDGPU::G_ADD) { - // TODO: Handle G_OR used for add case - if (mi_match(Def->getOperand(1).getReg(), MRI, m_ICst(Offset))) - return std::make_tuple(Def->getOperand(0).getReg(), Offset, Def); - - // FIXME: matcher should ignore copies - if (mi_match(Def->getOperand(1).getReg(), MRI, m_Copy(m_ICst(Offset)))) - return std::make_tuple(Def->getOperand(0).getReg(), Offset, Def); - } - - return std::make_tuple(Reg, 0, Def); -} - static unsigned getBufferStoreOpcode(LLT Ty, const unsigned MemSize, const bool Offen) { @@ -925,7 +918,7 @@ AMDGPUInstructionSelector::splitBufferOffsets(MachineIRBuilder &B, MachineInstr *OffsetDef; std::tie(BaseReg, TotalConstOffset, OffsetDef) - = getBaseWithConstantOffset(*MRI, OrigOffset); + = AMDGPU::getBaseWithConstantOffset(*MRI, OrigOffset); unsigned ImmOffset = TotalConstOffset; @@ -1029,6 +1022,90 @@ bool AMDGPUInstructionSelector::selectStoreIntrinsic(MachineInstr &MI, return constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI); } +static unsigned getDSShaderTypeValue(const MachineFunction &MF) { + switch (MF.getFunction().getCallingConv()) { + case CallingConv::AMDGPU_PS: + return 1; + case CallingConv::AMDGPU_VS: + return 2; + case CallingConv::AMDGPU_GS: + return 3; + case CallingConv::AMDGPU_HS: + case CallingConv::AMDGPU_LS: + case CallingConv::AMDGPU_ES: + report_fatal_error("ds_ordered_count unsupported for this calling conv"); + case CallingConv::AMDGPU_CS: + case CallingConv::AMDGPU_KERNEL: + case CallingConv::C: + case CallingConv::Fast: + default: + // Assume other calling conventions are various compute callable functions + return 0; + } +} + +bool AMDGPUInstructionSelector::selectDSOrderedIntrinsic( + MachineInstr &MI, Intrinsic::ID IntrID) const { + MachineBasicBlock *MBB = MI.getParent(); + MachineFunction *MF = MBB->getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + + unsigned IndexOperand = MI.getOperand(7).getImm(); + bool WaveRelease = MI.getOperand(8).getImm() != 0; + bool WaveDone = MI.getOperand(9).getImm() != 0; + + if (WaveDone && !WaveRelease) + report_fatal_error("ds_ordered_count: wave_done requires wave_release"); + + unsigned OrderedCountIndex = IndexOperand & 0x3f; + IndexOperand &= ~0x3f; + unsigned CountDw = 0; + + if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) { + CountDw = (IndexOperand >> 24) & 0xf; + IndexOperand &= ~(0xf << 24); + + if (CountDw < 1 || CountDw > 4) { + report_fatal_error( + "ds_ordered_count: dword count must be between 1 and 4"); + } + } + + if (IndexOperand) + report_fatal_error("ds_ordered_count: bad index operand"); + + unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1; + unsigned ShaderType = getDSShaderTypeValue(*MF); + + unsigned Offset0 = OrderedCountIndex << 2; + unsigned Offset1 = WaveRelease | (WaveDone << 1) | (ShaderType << 2) | + (Instruction << 4); + + if (STI.getGeneration() >= AMDGPUSubtarget::GFX10) + Offset1 |= (CountDw - 1) << 6; + + unsigned Offset = Offset0 | (Offset1 << 8); + + Register M0Val = MI.getOperand(2).getReg(); + BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) + .addReg(M0Val); + + Register DstReg = MI.getOperand(0).getReg(); + Register ValReg = MI.getOperand(3).getReg(); + MachineInstrBuilder DS = + BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::DS_ORDERED_COUNT), DstReg) + .addReg(ValReg) + .addImm(Offset) + .cloneMemRefs(MI); + + if (!RBI.constrainGenericRegister(M0Val, AMDGPU::SReg_32RegClass, *MRI)) + return false; + + bool Ret = constrainSelectedInstRegOperands(*DS, TII, TRI, RBI); + MI.eraseFromParent(); + return Ret; +} + bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS( MachineInstr &I) const { MachineBasicBlock *BB = I.getParent(); @@ -1084,6 +1161,9 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS( return selectStoreIntrinsic(I, false); case Intrinsic::amdgcn_raw_buffer_store_format: return selectStoreIntrinsic(I, true); + case Intrinsic::amdgcn_ds_ordered_add: + case Intrinsic::amdgcn_ds_ordered_swap: + return selectDSOrderedIntrinsic(I, IntrinsicID); default: return selectImpl(I, *CoverageInfo); } @@ -1098,7 +1178,7 @@ bool AMDGPUInstructionSelector::selectG_SELECT(MachineInstr &I) const { assert(Size <= 32 || Size == 64); const MachineOperand &CCOp = I.getOperand(1); Register CCReg = CCOp.getReg(); - if (isSCC(CCReg, *MRI)) { + if (!isVCC(CCReg, *MRI)) { unsigned SelectOpcode = Size == 64 ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32; MachineInstr *CopySCC = BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) @@ -1170,10 +1250,19 @@ bool AMDGPUInstructionSelector::selectG_TRUNC(MachineInstr &I) const { if (!DstTy.isScalar()) return false; - const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); + const LLT S1 = LLT::scalar(1); + const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); - if (SrcRB != DstRB) - return false; + const RegisterBank *DstRB; + if (DstTy == S1) { + // This is a special case. We don't treat s1 for legalization artifacts as + // vcc booleans. + DstRB = SrcRB; + } else { + DstRB = RBI.getRegBank(DstReg, *MRI, TRI); + if (SrcRB != DstRB) + return false; + } unsigned DstSize = DstTy.getSizeInBits(); unsigned SrcSize = SrcTy.getSizeInBits(); @@ -1214,6 +1303,20 @@ static bool shouldUseAndMask(unsigned Size, unsigned &Mask) { return SignedMask >= -16 && SignedMask <= 64; } +// Like RegisterBankInfo::getRegBank, but don't assume vcc for s1. +const RegisterBank *AMDGPUInstructionSelector::getArtifactRegBank( + Register Reg, const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const { + const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); + if (auto *RB = RegClassOrBank.dyn_cast<const RegisterBank *>()) + return RB; + + // Ignore the type, since we don't use vcc in artifacts. + if (auto *RC = RegClassOrBank.dyn_cast<const TargetRegisterClass *>()) + return &RBI.getRegBankFromRegClass(*RC, LLT()); + return nullptr; +} + bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const { bool Signed = I.getOpcode() == AMDGPU::G_SEXT; const DebugLoc &DL = I.getDebugLoc(); @@ -1223,57 +1326,17 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const { const LLT DstTy = MRI->getType(DstReg); const LLT SrcTy = MRI->getType(SrcReg); - const LLT S1 = LLT::scalar(1); const unsigned SrcSize = SrcTy.getSizeInBits(); const unsigned DstSize = DstTy.getSizeInBits(); if (!DstTy.isScalar()) return false; - const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, *MRI, TRI); - - if (SrcBank->getID() == AMDGPU::SCCRegBankID) { - if (SrcTy != S1 || DstSize > 64) // Invalid - return false; - - unsigned Opcode = - DstSize > 32 ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32; - const TargetRegisterClass *DstRC = - DstSize > 32 ? &AMDGPU::SReg_64RegClass : &AMDGPU::SReg_32RegClass; - - // FIXME: Create an extra copy to avoid incorrectly constraining the result - // of the scc producer. - Register TmpReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); - BuildMI(MBB, I, DL, TII.get(AMDGPU::COPY), TmpReg) - .addReg(SrcReg); - BuildMI(MBB, I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) - .addReg(TmpReg); - - // The instruction operands are backwards from what you would expect. - BuildMI(MBB, I, DL, TII.get(Opcode), DstReg) - .addImm(0) - .addImm(Signed ? -1 : 1); - I.eraseFromParent(); - return RBI.constrainGenericRegister(DstReg, *DstRC, *MRI); - } - - if (SrcBank->getID() == AMDGPU::VCCRegBankID && DstSize <= 32) { - if (SrcTy != S1) // Invalid - return false; - - MachineInstr *ExtI = - BuildMI(MBB, I, DL, TII.get(AMDGPU::V_CNDMASK_B32_e64), DstReg) - .addImm(0) // src0_modifiers - .addImm(0) // src0 - .addImm(0) // src1_modifiers - .addImm(Signed ? -1 : 1) // src1 - .addUse(SrcReg); - I.eraseFromParent(); - return constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI); - } - if (I.getOpcode() == AMDGPU::G_ANYEXT) return selectCOPY(I); + // Artifact casts should never use vcc. + const RegisterBank *SrcBank = getArtifactRegBank(SrcReg, *MRI, TRI); + if (SrcBank->getID() == AMDGPU::VGPRRegBankID && DstSize <= 32) { // 64-bit should have been split up in RegBankSelect @@ -1352,49 +1415,6 @@ bool AMDGPUInstructionSelector::selectG_SZA_EXT(MachineInstr &I) const { return false; } -static int64_t getFPTrueImmVal(unsigned Size, bool Signed) { - switch (Size) { - case 16: - return Signed ? 0xBC00 : 0x3C00; - case 32: - return Signed ? 0xbf800000 : 0x3f800000; - case 64: - return Signed ? 0xbff0000000000000 : 0x3ff0000000000000; - default: - llvm_unreachable("Invalid FP type size"); - } -} - -bool AMDGPUInstructionSelector::selectG_SITOFP_UITOFP(MachineInstr &I) const { - MachineBasicBlock *MBB = I.getParent(); - MachineFunction *MF = MBB->getParent(); - MachineRegisterInfo &MRI = MF->getRegInfo(); - Register Src = I.getOperand(1).getReg(); - if (!isSCC(Src, MRI)) - return selectImpl(I, *CoverageInfo); - - bool Signed = I.getOpcode() == AMDGPU::G_SITOFP; - Register DstReg = I.getOperand(0).getReg(); - const LLT DstTy = MRI.getType(DstReg); - const unsigned DstSize = DstTy.getSizeInBits(); - const DebugLoc &DL = I.getDebugLoc(); - - BuildMI(*MBB, I, DL, TII.get(AMDGPU::COPY), AMDGPU::SCC) - .addReg(Src); - - unsigned NewOpc = - DstSize > 32 ? AMDGPU::S_CSELECT_B64 : AMDGPU::S_CSELECT_B32; - auto MIB = BuildMI(*MBB, I, DL, TII.get(NewOpc), DstReg) - .addImm(0) - .addImm(getFPTrueImmVal(DstSize, Signed)); - - if (!constrainSelectedInstRegOperands(*MIB, TII, TRI, RBI)) - return false; - - I.eraseFromParent(); - return true; -} - bool AMDGPUInstructionSelector::selectG_CONSTANT(MachineInstr &I) const { MachineBasicBlock *BB = I.getParent(); MachineOperand &ImmOp = I.getOperand(1); @@ -1478,7 +1498,7 @@ void AMDGPUInstructionSelector::getAddrModeInfo(const MachineInstr &Load, assert(PtrMI); - if (PtrMI->getOpcode() != TargetOpcode::G_GEP) + if (PtrMI->getOpcode() != TargetOpcode::G_PTR_ADD) return; GEPInfo GEPInfo(*PtrMI); @@ -1568,12 +1588,15 @@ bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const { // GlobalISel, we should push that decision into RegBankSelect. Assume for now // RegBankSelect knows what it's doing if the branch condition is scc, even // though it currently does not. - if (isSCC(CondReg, *MRI)) { + if (!isVCC(CondReg, *MRI)) { + if (MRI->getType(CondReg) != LLT::scalar(32)) + return false; + CondPhysReg = AMDGPU::SCC; BrOpcode = AMDGPU::S_CBRANCH_SCC1; // FIXME: Hack for isSCC tests ConstrainRC = &AMDGPU::SGPR_32RegClass; - } else if (isVCC(CondReg, *MRI)) { + } else { // FIXME: Do we have to insert an and with exec here, like in SelectionDAG? // We sort of know that a VCC producer based on the register bank, that ands // inactive lanes with 0. What if there was a logical operation with vcc @@ -1582,8 +1605,7 @@ bool AMDGPUInstructionSelector::selectG_BRCOND(MachineInstr &I) const { CondPhysReg = TRI.getVCC(); BrOpcode = AMDGPU::S_CBRANCH_VCCNZ; ConstrainRC = TRI.getBoolRC(); - } else - return false; + } if (!MRI->getRegClassOrNull(CondReg)) MRI->setRegClass(CondReg, ConstrainRC); @@ -1670,6 +1692,80 @@ bool AMDGPUInstructionSelector::selectG_PTR_MASK(MachineInstr &I) const { return true; } +bool AMDGPUInstructionSelector::selectG_EXTRACT_VECTOR_ELT( + MachineInstr &MI) const { + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + Register IdxReg = MI.getOperand(2).getReg(); + + LLT DstTy = MRI->getType(DstReg); + LLT SrcTy = MRI->getType(SrcReg); + + const RegisterBank *DstRB = RBI.getRegBank(DstReg, *MRI, TRI); + const RegisterBank *SrcRB = RBI.getRegBank(SrcReg, *MRI, TRI); + const RegisterBank *IdxRB = RBI.getRegBank(IdxReg, *MRI, TRI); + + // The index must be scalar. If it wasn't RegBankSelect should have moved this + // into a waterfall loop. + if (IdxRB->getID() != AMDGPU::SGPRRegBankID) + return false; + + const TargetRegisterClass *SrcRC = TRI.getRegClassForTypeOnBank(SrcTy, *SrcRB, + *MRI); + const TargetRegisterClass *DstRC = TRI.getRegClassForTypeOnBank(DstTy, *DstRB, + *MRI); + if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, *MRI) || + !RBI.constrainGenericRegister(DstReg, *DstRC, *MRI) || + !RBI.constrainGenericRegister(IdxReg, AMDGPU::SReg_32RegClass, *MRI)) + return false; + + MachineBasicBlock *BB = MI.getParent(); + const DebugLoc &DL = MI.getDebugLoc(); + const bool Is64 = DstTy.getSizeInBits() == 64; + + unsigned SubReg = Is64 ? AMDGPU::sub0_sub1 : AMDGPU::sub0; + + if (SrcRB->getID() == AMDGPU::SGPRRegBankID) { + if (DstTy.getSizeInBits() != 32 && !Is64) + return false; + + BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) + .addReg(IdxReg); + + unsigned Opc = Is64 ? AMDGPU::S_MOVRELS_B64 : AMDGPU::S_MOVRELS_B32; + BuildMI(*BB, &MI, DL, TII.get(Opc), DstReg) + .addReg(SrcReg, 0, SubReg) + .addReg(SrcReg, RegState::Implicit); + MI.eraseFromParent(); + return true; + } + + if (SrcRB->getID() != AMDGPU::VGPRRegBankID || DstTy.getSizeInBits() != 32) + return false; + + if (!STI.useVGPRIndexMode()) { + BuildMI(*BB, &MI, DL, TII.get(AMDGPU::COPY), AMDGPU::M0) + .addReg(IdxReg); + BuildMI(*BB, &MI, DL, TII.get(AMDGPU::V_MOVRELS_B32_e32), DstReg) + .addReg(SrcReg, RegState::Undef, SubReg) + .addReg(SrcReg, RegState::Implicit); + MI.eraseFromParent(); + return true; + } + + BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_ON)) + .addReg(IdxReg) + .addImm(AMDGPU::VGPRIndexMode::SRC0_ENABLE); + BuildMI(*BB, MI, DL, TII.get(AMDGPU::V_MOV_B32_e32), DstReg) + .addReg(SrcReg, RegState::Undef, SubReg) + .addReg(SrcReg, RegState::Implicit) + .addReg(AMDGPU::M0, RegState::Implicit); + BuildMI(*BB, MI, DL, TII.get(AMDGPU::S_SET_GPR_IDX_OFF)); + + MI.eraseFromParent(); + return true; +} + bool AMDGPUInstructionSelector::select(MachineInstr &I) { if (I.isPHI()) return selectPHI(I); @@ -1694,7 +1790,9 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) { return selectG_ADD_SUB(I); case TargetOpcode::G_UADDO: case TargetOpcode::G_USUBO: - return selectG_UADDO_USUBO(I); + case TargetOpcode::G_UADDE: + case TargetOpcode::G_USUBE: + return selectG_UADDO_USUBO_UADDE_USUBE(I); case TargetOpcode::G_INTTOPTR: case TargetOpcode::G_BITCAST: case TargetOpcode::G_PTRTOINT: @@ -1710,8 +1808,8 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) { return selectG_MERGE_VALUES(I); case TargetOpcode::G_UNMERGE_VALUES: return selectG_UNMERGE_VALUES(I); - case TargetOpcode::G_GEP: - return selectG_GEP(I); + case TargetOpcode::G_PTR_ADD: + return selectG_PTR_ADD(I); case TargetOpcode::G_IMPLICIT_DEF: return selectG_IMPLICIT_DEF(I); case TargetOpcode::G_INSERT: @@ -1747,21 +1845,17 @@ bool AMDGPUInstructionSelector::select(MachineInstr &I) { case TargetOpcode::G_SEXT: case TargetOpcode::G_ZEXT: case TargetOpcode::G_ANYEXT: + if (selectImpl(I, *CoverageInfo)) + return true; return selectG_SZA_EXT(I); - case TargetOpcode::G_SITOFP: - case TargetOpcode::G_UITOFP: - return selectG_SITOFP_UITOFP(I); case TargetOpcode::G_BRCOND: return selectG_BRCOND(I); case TargetOpcode::G_FRAME_INDEX: return selectG_FRAME_INDEX(I); - case TargetOpcode::G_FENCE: - // FIXME: Tablegen importer doesn't handle the imm operands correctly, and - // is checking for G_CONSTANT - I.setDesc(TII.get(AMDGPU::ATOMIC_FENCE)); - return true; case TargetOpcode::G_PTR_MASK: return selectG_PTR_MASK(I); + case TargetOpcode::G_EXTRACT_VECTOR_ELT: + return selectG_EXTRACT_VECTOR_ELT(I); default: return selectImpl(I, *CoverageInfo); } @@ -1821,20 +1915,6 @@ AMDGPUInstructionSelector::selectVOP3Mods0(MachineOperand &Root) const { } InstructionSelector::ComplexRendererFns -AMDGPUInstructionSelector::selectVOP3Mods0Clamp0OMod(MachineOperand &Root) const { - Register Src; - unsigned Mods; - std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg()); - - return {{ - [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); }, - [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); }, // src0_mods - [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }, // clamp - [=](MachineInstrBuilder &MIB) { MIB.addImm(0); } // omod - }}; -} - -InstructionSelector::ComplexRendererFns AMDGPUInstructionSelector::selectVOP3OMods(MachineOperand &Root) const { return {{ [=](MachineInstrBuilder &MIB) { MIB.add(Root); }, @@ -1856,6 +1936,20 @@ AMDGPUInstructionSelector::selectVOP3Mods(MachineOperand &Root) const { } InstructionSelector::ComplexRendererFns +AMDGPUInstructionSelector::selectVOP3Mods_nnan(MachineOperand &Root) const { + Register Src; + unsigned Mods; + std::tie(Src, Mods) = selectVOP3ModsImpl(Root.getReg()); + if (!TM.Options.NoNaNsFPMath && !isKnownNeverNaN(Src, *MRI)) + return None; + + return {{ + [=](MachineInstrBuilder &MIB) { MIB.addReg(Src); }, + [=](MachineInstrBuilder &MIB) { MIB.addImm(Mods); } // src_mods + }}; +} + +InstructionSelector::ComplexRendererFns AMDGPUInstructionSelector::selectVOP3OpSelMods0(MachineOperand &Root) const { // FIXME: Handle clamp and op_sel return {{ @@ -1961,7 +2055,7 @@ AMDGPUInstructionSelector::selectFlatOffsetImpl(MachineOperand &Root) const { return Default; const MachineInstr *OpDef = MRI->getVRegDef(Root.getReg()); - if (!OpDef || OpDef->getOpcode() != AMDGPU::G_GEP) + if (!OpDef || OpDef->getOpcode() != AMDGPU::G_PTR_ADD) return Default; Optional<int64_t> Offset = @@ -2175,10 +2269,65 @@ AMDGPUInstructionSelector::selectDS1Addr1Offset(MachineOperand &Root) const { } void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB, - const MachineInstr &MI) const { - const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo(); - assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); - Optional<int64_t> CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), MRI); + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && + "Expected G_CONSTANT"); + Optional<int64_t> CstVal = getConstantVRegVal(MI.getOperand(0).getReg(), *MRI); assert(CstVal && "Expected constant value"); MIB.addImm(CstVal.getValue()); } + +void AMDGPUInstructionSelector::renderNegateImm(MachineInstrBuilder &MIB, + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && + "Expected G_CONSTANT"); + MIB.addImm(-MI.getOperand(1).getCImm()->getSExtValue()); +} + +void AMDGPUInstructionSelector::renderBitcastImm(MachineInstrBuilder &MIB, + const MachineInstr &MI, + int OpIdx) const { + assert(OpIdx == -1); + + const MachineOperand &Op = MI.getOperand(1); + if (MI.getOpcode() == TargetOpcode::G_FCONSTANT) + MIB.addImm(Op.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue()); + else { + assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && "Expected G_CONSTANT"); + MIB.addImm(Op.getCImm()->getSExtValue()); + } +} + +void AMDGPUInstructionSelector::renderPopcntImm(MachineInstrBuilder &MIB, + const MachineInstr &MI, + int OpIdx) const { + assert(MI.getOpcode() == TargetOpcode::G_CONSTANT && OpIdx == -1 && + "Expected G_CONSTANT"); + MIB.addImm(MI.getOperand(1).getCImm()->getValue().countPopulation()); +} + +/// This only really exists to satisfy DAG type checking machinery, so is a +/// no-op here. +void AMDGPUInstructionSelector::renderTruncTImm(MachineInstrBuilder &MIB, + const MachineInstr &MI, + int OpIdx) const { + MIB.addImm(MI.getOperand(OpIdx).getImm()); +} + +bool AMDGPUInstructionSelector::isInlineImmediate16(int64_t Imm) const { + return AMDGPU::isInlinableLiteral16(Imm, STI.hasInv2PiInlineImm()); +} + +bool AMDGPUInstructionSelector::isInlineImmediate32(int64_t Imm) const { + return AMDGPU::isInlinableLiteral32(Imm, STI.hasInv2PiInlineImm()); +} + +bool AMDGPUInstructionSelector::isInlineImmediate64(int64_t Imm) const { + return AMDGPU::isInlinableLiteral64(Imm, STI.hasInv2PiInlineImm()); +} + +bool AMDGPUInstructionSelector::isInlineImmediate(const APFloat &Imm) const { + return TII.isInlineConstant(Imm); +} diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h index d3c83a6a872a..38ca7fd4104b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h @@ -38,6 +38,7 @@ class MachineInstr; class MachineIRBuilder; class MachineOperand; class MachineRegisterInfo; +class RegisterBank; class SIInstrInfo; class SIMachineFunctionInfo; class SIRegisterInfo; @@ -69,6 +70,10 @@ private: bool isInstrUniform(const MachineInstr &MI) const; bool isVCC(Register Reg, const MachineRegisterInfo &MRI) const; + const RegisterBank *getArtifactRegBank( + Register Reg, const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const; + /// tblgen-erated 'select' implementation. bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; @@ -79,15 +84,14 @@ private: bool selectPHI(MachineInstr &I) const; bool selectG_TRUNC(MachineInstr &I) const; bool selectG_SZA_EXT(MachineInstr &I) const; - bool selectG_SITOFP_UITOFP(MachineInstr &I) const; bool selectG_CONSTANT(MachineInstr &I) const; bool selectG_AND_OR_XOR(MachineInstr &I) const; bool selectG_ADD_SUB(MachineInstr &I) const; - bool selectG_UADDO_USUBO(MachineInstr &I) const; + bool selectG_UADDO_USUBO_UADDE_USUBE(MachineInstr &I) const; bool selectG_EXTRACT(MachineInstr &I) const; bool selectG_MERGE_VALUES(MachineInstr &I) const; bool selectG_UNMERGE_VALUES(MachineInstr &I) const; - bool selectG_GEP(MachineInstr &I) const; + bool selectG_PTR_ADD(MachineInstr &I) const; bool selectG_IMPLICIT_DEF(MachineInstr &I) const; bool selectG_INSERT(MachineInstr &I) const; bool selectG_INTRINSIC(MachineInstr &I) const; @@ -96,6 +100,7 @@ private: splitBufferOffsets(MachineIRBuilder &B, Register OrigOffset) const; bool selectStoreIntrinsic(MachineInstr &MI, bool IsFormat) const; + bool selectDSOrderedIntrinsic(MachineInstr &MI, Intrinsic::ID IID) const; bool selectG_INTRINSIC_W_SIDE_EFFECTS(MachineInstr &I) const; int getS_CMPOpcode(CmpInst::Predicate P, unsigned Size) const; @@ -112,6 +117,7 @@ private: bool selectG_BRCOND(MachineInstr &I) const; bool selectG_FRAME_INDEX(MachineInstr &I) const; bool selectG_PTR_MASK(MachineInstr &I) const; + bool selectG_EXTRACT_VECTOR_ELT(MachineInstr &I) const; std::pair<Register, unsigned> selectVOP3ModsImpl(Register Src) const; @@ -125,11 +131,11 @@ private: InstructionSelector::ComplexRendererFns selectVOP3Mods0(MachineOperand &Root) const; InstructionSelector::ComplexRendererFns - selectVOP3Mods0Clamp0OMod(MachineOperand &Root) const; - InstructionSelector::ComplexRendererFns selectVOP3OMods(MachineOperand &Root) const; InstructionSelector::ComplexRendererFns selectVOP3Mods(MachineOperand &Root) const; + InstructionSelector::ComplexRendererFns + selectVOP3Mods_nnan(MachineOperand &Root) const; InstructionSelector::ComplexRendererFns selectVOP3OpSelMods0(MachineOperand &Root) const; @@ -164,8 +170,25 @@ private: InstructionSelector::ComplexRendererFns selectDS1Addr1Offset(MachineOperand &Root) const; - void renderTruncImm32(MachineInstrBuilder &MIB, - const MachineInstr &MI) const; + void renderTruncImm32(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx = -1) const; + + void renderTruncTImm(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; + + void renderNegateImm(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; + + void renderBitcastImm(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; + + void renderPopcntImm(MachineInstrBuilder &MIB, const MachineInstr &MI, + int OpIdx) const; + + bool isInlineImmediate16(int64_t Imm) const; + bool isInlineImmediate32(int64_t Imm) const; + bool isInlineImmediate64(int64_t Imm) const; + bool isInlineImmediate(const APFloat &Imm) const; const SIInstrInfo &TII; const SIRegisterInfo &TRI; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td index 846e7f577a28..7e71dbdd1240 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -77,28 +77,39 @@ class ILFormat<dag outs, dag ins, string asmstr, list<dag> pattern> def TruePredicate : Predicate<"">; +// Add a predicate to the list if does not already exist to deduplicate it. +class PredConcat<list<Predicate> lst, Predicate pred> { + list<Predicate> ret = + !foldl([pred], lst, acc, cur, + !listconcat(acc, !if(!eq(!cast<string>(cur),!cast<string>(pred)), + [], [cur]))); +} + class PredicateControl { Predicate SubtargetPredicate = TruePredicate; - list<Predicate> AssemblerPredicates = []; Predicate AssemblerPredicate = TruePredicate; Predicate WaveSizePredicate = TruePredicate; list<Predicate> OtherPredicates = []; - list<Predicate> Predicates = !listconcat([SubtargetPredicate, - AssemblerPredicate, - WaveSizePredicate], - AssemblerPredicates, - OtherPredicates); + list<Predicate> Predicates = PredConcat< + PredConcat<PredConcat<OtherPredicates, + SubtargetPredicate>.ret, + AssemblerPredicate>.ret, + WaveSizePredicate>.ret; } + class AMDGPUPat<dag pattern, dag result> : Pat<pattern, result>, PredicateControl; -def FP16Denormals : Predicate<"Subtarget->hasFP16Denormals()">; -def FP32Denormals : Predicate<"Subtarget->hasFP32Denormals()">; -def FP64Denormals : Predicate<"Subtarget->hasFP64Denormals()">; -def NoFP16Denormals : Predicate<"!Subtarget->hasFP16Denormals()">; -def NoFP32Denormals : Predicate<"!Subtarget->hasFP32Denormals()">; -def NoFP64Denormals : Predicate<"!Subtarget->hasFP64Denormals()">; +let RecomputePerFunction = 1 in { +def FP16Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals">; +def FP32Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP32Denormals">; +def FP64Denormals : Predicate<"MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals">; +def NoFP16Denormals : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals">; +def NoFP32Denormals : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().FP32Denormals">; +def NoFP64Denormals : Predicate<"!MF->getInfo<SIMachineFunctionInfo>()->getMode().FP64FP16Denormals">; def UnsafeFPMath : Predicate<"TM.Options.UnsafeFPMath">; +} + def FMA : Predicate<"Subtarget->hasFMA()">; def InstFlag : OperandWithDefaultOps <i32, (ops (i32 0))>; @@ -147,20 +158,30 @@ def brtarget : Operand<OtherVT>; class HasOneUseUnaryOp<SDPatternOperator op> : PatFrag< (ops node:$src0), (op $src0), - [{ return N->hasOneUse(); }] ->; + [{ return N->hasOneUse(); }]> { + + let GISelPredicateCode = [{ + return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg()); + }]; +} class HasOneUseBinOp<SDPatternOperator op> : PatFrag< (ops node:$src0, node:$src1), (op $src0, $src1), - [{ return N->hasOneUse(); }] ->; + [{ return N->hasOneUse(); }]> { + let GISelPredicateCode = [{ + return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg()); + }]; +} class HasOneUseTernaryOp<SDPatternOperator op> : PatFrag< (ops node:$src0, node:$src1, node:$src2), (op $src0, $src1, $src2), - [{ return N->hasOneUse(); }] ->; + [{ return N->hasOneUse(); }]> { + let GISelPredicateCode = [{ + return MRI.hasOneNonDBGUse(MI.getOperand(0).getReg()); + }]; +} let Properties = [SDNPCommutative, SDNPAssociative] in { def smax_oneuse : HasOneUseBinOp<smax>; @@ -315,15 +336,10 @@ class Aligned<int Bytes> { int MinAlignment = Bytes; } -class LoadFrag <SDPatternOperator op> : PatFrag<(ops node:$ptr), (op node:$ptr)>; - -class StoreFrag<SDPatternOperator op> : PatFrag < - (ops node:$value, node:$ptr), (op node:$value, node:$ptr) ->; - class StoreHi16<SDPatternOperator op> : PatFrag < - (ops node:$value, node:$ptr), (op (srl node:$value, (i32 16)), node:$ptr) ->; + (ops node:$value, node:$ptr), (op (srl node:$value, (i32 16)), node:$ptr)> { + let IsStore = 1; +} def LoadAddress_constant : AddressSpaceList<[ AddrSpaces.Constant ]>; def LoadAddress_global : AddressSpaceList<[ AddrSpaces.Global, AddrSpaces.Constant ]>; @@ -345,48 +361,6 @@ def StoreAddress_region : AddressSpaceList<[ AddrSpaces.Region ]>; -class GlobalLoadAddress : CodePatPred<[{ - auto AS = cast<MemSDNode>(N)->getAddressSpace(); - return AS == AMDGPUAS::GLOBAL_ADDRESS || AS == AMDGPUAS::CONSTANT_ADDRESS; -}]>; - -class FlatLoadAddress : CodePatPred<[{ - const auto AS = cast<MemSDNode>(N)->getAddressSpace(); - return AS == AMDGPUAS::FLAT_ADDRESS || - AS == AMDGPUAS::GLOBAL_ADDRESS || - AS == AMDGPUAS::CONSTANT_ADDRESS; -}]>; - -class GlobalAddress : CodePatPred<[{ - return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; -}]>; - -class PrivateAddress : CodePatPred<[{ - return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS; -}]>; - -class LocalAddress : CodePatPred<[{ - return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; -}]>; - -class RegionAddress : CodePatPred<[{ - return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::REGION_ADDRESS; -}]>; - -class FlatStoreAddress : CodePatPred<[{ - const auto AS = cast<MemSDNode>(N)->getAddressSpace(); - return AS == AMDGPUAS::FLAT_ADDRESS || - AS == AMDGPUAS::GLOBAL_ADDRESS; -}]>; - -// TODO: Remove these when stores to new PatFrag format. -class PrivateStore <SDPatternOperator op> : StoreFrag <op>, PrivateAddress; -class LocalStore <SDPatternOperator op> : StoreFrag <op>, LocalAddress; -class RegionStore <SDPatternOperator op> : StoreFrag <op>, RegionAddress; -class GlobalStore <SDPatternOperator op> : StoreFrag<op>, GlobalAddress; -class FlatStore <SDPatternOperator op> : StoreFrag <op>, FlatStoreAddress; - - foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in { let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in { @@ -464,6 +438,10 @@ def truncstorei16_#as : PatFrag<(ops node:$val, node:$ptr), let MemoryVT = i16; } +def store_hi16_#as : StoreHi16 <truncstorei16>; +def truncstorei8_hi16_#as : StoreHi16<truncstorei8>; +def truncstorei16_hi16_#as : StoreHi16<truncstorei16>; + defm atomic_store_#as : binary_atomic_op<atomic_store>; } // End let AddressSpaces = ... @@ -497,18 +475,7 @@ defm atomic_load_umax : ret_noret_binary_atomic_op<atomic_load_umax>; defm atomic_load_umin : ret_noret_binary_atomic_op<atomic_load_umin>; defm atomic_load_xor : ret_noret_binary_atomic_op<atomic_load_xor>; defm atomic_load_fadd : ret_noret_binary_atomic_op<atomic_load_fadd, 0>; - - -def store_hi16_private : StoreHi16 <truncstorei16>, PrivateAddress; -def truncstorei8_hi16_private : StoreHi16<truncstorei8>, PrivateAddress; - -def store_atomic_global : GlobalStore<atomic_store>; -def truncstorei8_hi16_global : StoreHi16 <truncstorei8>, GlobalAddress; -def truncstorei16_hi16_global : StoreHi16 <truncstorei16>, GlobalAddress; - -def store_local_hi16 : StoreHi16 <truncstorei16>, LocalAddress; -def truncstorei8_local_hi16 : StoreHi16<truncstorei8>, LocalAddress; -def atomic_store_local : LocalStore <atomic_store>; +defm AMDGPUatomic_cmp_swap : ret_noret_binary_atomic_op<AMDGPUatomic_cmp_swap>; def load_align8_local : PatFrag <(ops node:$ptr), (load_local node:$ptr)> { @@ -535,30 +502,6 @@ def store_align16_local: PatFrag<(ops node:$val, node:$ptr), let IsTruncStore = 0; } - -def atomic_store_flat : FlatStore <atomic_store>; -def truncstorei8_hi16_flat : StoreHi16<truncstorei8>, FlatStoreAddress; -def truncstorei16_hi16_flat : StoreHi16<truncstorei16>, FlatStoreAddress; - - -class local_binary_atomic_op<SDNode atomic_op> : - PatFrag<(ops node:$ptr, node:$value), - (atomic_op node:$ptr, node:$value), [{ - return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; -}]>; - -class region_binary_atomic_op<SDNode atomic_op> : - PatFrag<(ops node:$ptr, node:$value), - (atomic_op node:$ptr, node:$value), [{ - return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::REGION_ADDRESS; -}]>; - - -def mskor_global : PatFrag<(ops node:$val, node:$ptr), - (AMDGPUstore_mskor node:$val, node:$ptr), [{ - return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; -}]>; - let AddressSpaces = StoreAddress_local.AddrSpaces in { defm atomic_cmp_swap_local : ternary_atomic_op<atomic_cmp_swap>; defm atomic_cmp_swap_local_m0 : ternary_atomic_op<atomic_cmp_swap_glue>; @@ -569,31 +512,6 @@ defm atomic_cmp_swap_region : ternary_atomic_op<atomic_cmp_swap>; defm atomic_cmp_swap_region_m0 : ternary_atomic_op<atomic_cmp_swap_glue>; } -class global_binary_atomic_op_frag<SDNode atomic_op> : PatFrag< - (ops node:$ptr, node:$value), - (atomic_op node:$ptr, node:$value), - [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;}]>; - -// Legacy. -def AMDGPUatomic_cmp_swap_global : PatFrag< - (ops node:$ptr, node:$value), - (AMDGPUatomic_cmp_swap node:$ptr, node:$value)>, GlobalAddress; - -def atomic_cmp_swap_global : PatFrag< - (ops node:$ptr, node:$cmp, node:$value), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$value)>, GlobalAddress; - - -def atomic_cmp_swap_global_noret : PatFrag< - (ops node:$ptr, node:$cmp, node:$value), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$value), - [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (SDValue(N, 0).use_empty());}]>; - -def atomic_cmp_swap_global_ret : PatFrag< - (ops node:$ptr, node:$cmp, node:$value), - (atomic_cmp_swap node:$ptr, node:$cmp, node:$value), - [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (!SDValue(N, 0).use_empty());}]>; - //===----------------------------------------------------------------------===// // Misc Pattern Fragments //===----------------------------------------------------------------------===// @@ -686,12 +604,12 @@ multiclass BFIPatterns <Instruction BFI_INT, def : AMDGPUPat < (or (and i64:$y, i64:$x), (and i64:$z, (not i64:$x))), (REG_SEQUENCE RC64, - (BFI_INT (i32 (EXTRACT_SUBREG $x, sub0)), - (i32 (EXTRACT_SUBREG $y, sub0)), - (i32 (EXTRACT_SUBREG $z, sub0))), sub0, - (BFI_INT (i32 (EXTRACT_SUBREG $x, sub1)), - (i32 (EXTRACT_SUBREG $y, sub1)), - (i32 (EXTRACT_SUBREG $z, sub1))), sub1) + (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub0)), + (i32 (EXTRACT_SUBREG RC64:$y, sub0)), + (i32 (EXTRACT_SUBREG RC64:$z, sub0))), sub0, + (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub1)), + (i32 (EXTRACT_SUBREG RC64:$y, sub1)), + (i32 (EXTRACT_SUBREG RC64:$z, sub1))), sub1) >; // SHA-256 Ch function @@ -705,12 +623,12 @@ multiclass BFIPatterns <Instruction BFI_INT, def : AMDGPUPat < (xor i64:$z, (and i64:$x, (xor i64:$y, i64:$z))), (REG_SEQUENCE RC64, - (BFI_INT (i32 (EXTRACT_SUBREG $x, sub0)), - (i32 (EXTRACT_SUBREG $y, sub0)), - (i32 (EXTRACT_SUBREG $z, sub0))), sub0, - (BFI_INT (i32 (EXTRACT_SUBREG $x, sub1)), - (i32 (EXTRACT_SUBREG $y, sub1)), - (i32 (EXTRACT_SUBREG $z, sub1))), sub1) + (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub0)), + (i32 (EXTRACT_SUBREG RC64:$y, sub0)), + (i32 (EXTRACT_SUBREG RC64:$z, sub0))), sub0, + (BFI_INT (i32 (EXTRACT_SUBREG RC64:$x, sub1)), + (i32 (EXTRACT_SUBREG RC64:$y, sub1)), + (i32 (EXTRACT_SUBREG RC64:$z, sub1))), sub1) >; def : AMDGPUPat < @@ -721,7 +639,7 @@ multiclass BFIPatterns <Instruction BFI_INT, def : AMDGPUPat < (f32 (fcopysign f32:$src0, f64:$src1)), (BFI_INT (LoadImm32 (i32 0x7fffffff)), $src0, - (i32 (EXTRACT_SUBREG $src1, sub1))) + (i32 (EXTRACT_SUBREG RC64:$src1, sub1))) >; def : AMDGPUPat < @@ -729,8 +647,8 @@ multiclass BFIPatterns <Instruction BFI_INT, (REG_SEQUENCE RC64, (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, (BFI_INT (LoadImm32 (i32 0x7fffffff)), - (i32 (EXTRACT_SUBREG $src0, sub1)), - (i32 (EXTRACT_SUBREG $src1, sub1))), sub1) + (i32 (EXTRACT_SUBREG RC64:$src0, sub1)), + (i32 (EXTRACT_SUBREG RC64:$src1, sub1))), sub1) >; def : AMDGPUPat < @@ -738,7 +656,7 @@ multiclass BFIPatterns <Instruction BFI_INT, (REG_SEQUENCE RC64, (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, (BFI_INT (LoadImm32 (i32 0x7fffffff)), - (i32 (EXTRACT_SUBREG $src0, sub1)), + (i32 (EXTRACT_SUBREG RC64:$src0, sub1)), $src1), sub1) >; } @@ -755,21 +673,21 @@ multiclass SHA256MaPattern <Instruction BFI_INT, Instruction XOR, RegisterClass def : AMDGPUPat < (or (and i64:$x, i64:$z), (and i64:$y, (or i64:$x, i64:$z))), (REG_SEQUENCE RC64, - (BFI_INT (XOR (i32 (EXTRACT_SUBREG $x, sub0)), - (i32 (EXTRACT_SUBREG $y, sub0))), - (i32 (EXTRACT_SUBREG $z, sub0)), - (i32 (EXTRACT_SUBREG $y, sub0))), sub0, - (BFI_INT (XOR (i32 (EXTRACT_SUBREG $x, sub1)), - (i32 (EXTRACT_SUBREG $y, sub1))), - (i32 (EXTRACT_SUBREG $z, sub1)), - (i32 (EXTRACT_SUBREG $y, sub1))), sub1) + (BFI_INT (XOR (i32 (EXTRACT_SUBREG RC64:$x, sub0)), + (i32 (EXTRACT_SUBREG RC64:$y, sub0))), + (i32 (EXTRACT_SUBREG RC64:$z, sub0)), + (i32 (EXTRACT_SUBREG RC64:$y, sub0))), sub0, + (BFI_INT (XOR (i32 (EXTRACT_SUBREG RC64:$x, sub1)), + (i32 (EXTRACT_SUBREG RC64:$y, sub1))), + (i32 (EXTRACT_SUBREG RC64:$z, sub1)), + (i32 (EXTRACT_SUBREG RC64:$y, sub1))), sub1) >; } // Bitfield extract patterns -def IMMZeroBasedBitfieldMask : PatLeaf <(imm), [{ - return isMask_32(N->getZExtValue()); +def IMMZeroBasedBitfieldMask : ImmLeaf <i32, [{ + return isMask_32(Imm); }]>; def IMMPopCount : SDNodeXForm<imm, [{ @@ -819,30 +737,6 @@ class ROTRPattern <Instruction BIT_ALIGN> : AMDGPUPat < (BIT_ALIGN $src0, $src0, $src1) >; -multiclass IntMed3Pat<Instruction med3Inst, - SDPatternOperator min, - SDPatternOperator max, - SDPatternOperator min_oneuse, - SDPatternOperator max_oneuse, - ValueType vt = i32> { - - // This matches 16 permutations of - // min(max(a, b), max(min(a, b), c)) - def : AMDGPUPat < - (min (max_oneuse vt:$src0, vt:$src1), - (max_oneuse (min_oneuse vt:$src0, vt:$src1), vt:$src2)), - (med3Inst vt:$src0, vt:$src1, vt:$src2) ->; - - // This matches 16 permutations of - // max(min(x, y), min(max(x, y), z)) - def : AMDGPUPat < - (max (min_oneuse vt:$src0, vt:$src1), - (min_oneuse (max_oneuse vt:$src0, vt:$src1), vt:$src2)), - (med3Inst $src0, $src1, $src2) ->; -} - // Special conversion patterns def cvt_rpi_i32_f32 : PatFrag < diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp index 5aba35a19ced..3f99d5cfb7f9 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp @@ -244,7 +244,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, S32, S64, S16, V2S16 }; - setAction({G_BRCOND, S1}, Legal); + setAction({G_BRCOND, S1}, Legal); // VCC branches + setAction({G_BRCOND, S32}, Legal); // SCC branches // TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more // elements for v3s16 @@ -272,6 +273,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .scalarize(0); } + // FIXME: Not really legal. Placeholder for custom lowering. + getActionDefinitionsBuilder({G_SDIV, G_UDIV, G_SREM, G_UREM}) + .legalFor({S32, S64}) + .clampScalar(0, S32, S64) + .widenScalarToNextPow2(0, 32) + .scalarize(0); + getActionDefinitionsBuilder({G_UMULH, G_SMULH}) .legalFor({S32}) .clampScalar(0, S32, S32) @@ -289,7 +297,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder({G_UADDO, G_USUBO, G_UADDE, G_SADDE, G_USUBE, G_SSUBE}) - .legalFor({{S32, S1}}) + .legalFor({{S32, S1}, {S32, S32}}) .clampScalar(0, S32, S32) .scalarize(0); // TODO: Implement. @@ -354,6 +362,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, if (ST.hasVOP3PInsts()) { MinNumMaxNum.customFor(FPTypesPK16) + .moreElementsIf(isSmallOddVector(0), oneMoreElement(0)) .clampMaxNumElements(0, S16, 2) .clampScalar(0, S16, S64) .scalarize(0); @@ -438,13 +447,15 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, {S96, S32}, // FIXME: Hack {S64, LLT::scalar(33)}, - {S32, S8}, {S128, S32}, {S128, S64}, {S32, LLT::scalar(24)}}) - .scalarize(0); + {S32, S8}, {S32, LLT::scalar(24)}}) + .scalarize(0) + .clampScalar(0, S32, S64); // TODO: Split s1->s64 during regbankselect for VALU. auto &IToFP = getActionDefinitionsBuilder({G_SITOFP, G_UITOFP}) - .legalFor({{S32, S32}, {S64, S32}, {S16, S32}, {S32, S1}, {S16, S1}, {S64, S1}}) + .legalFor({{S32, S32}, {S64, S32}, {S16, S32}}) .lowerFor({{S32, S64}}) + .lowerIf(typeIs(1, S1)) .customFor({{S64, S64}}); if (ST.has16BitInsts()) IToFP.legalFor({{S16, S16}}); @@ -462,10 +473,15 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .scalarize(0); getActionDefinitionsBuilder(G_INTRINSIC_ROUND) - .legalFor({S32, S64}) - .scalarize(0); + .scalarize(0) + .lower(); - if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) { + if (ST.has16BitInsts()) { + getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT}) + .legalFor({S16, S32, S64}) + .clampScalar(0, S16, S64) + .scalarize(0); + } else if (ST.getGeneration() >= AMDGPUSubtarget::SEA_ISLANDS) { getActionDefinitionsBuilder({G_INTRINSIC_TRUNC, G_FCEIL, G_FRINT}) .legalFor({S32, S64}) .clampScalar(0, S32, S64) @@ -478,7 +494,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .scalarize(0); } - getActionDefinitionsBuilder(G_GEP) + getActionDefinitionsBuilder(G_PTR_ADD) .legalForCartesianProduct(AddrSpaces64, {S64}) .legalForCartesianProduct(AddrSpaces32, {S32}) .scalarize(0); @@ -491,9 +507,20 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, auto &CmpBuilder = getActionDefinitionsBuilder(G_ICMP) + // The compare output type differs based on the register bank of the output, + // so make both s1 and s32 legal. + // + // Scalar compares producing output in scc will be promoted to s32, as that + // is the allocatable register type that will be needed for the copy from + // scc. This will be promoted during RegBankSelect, and we assume something + // before that won't try to use s32 result types. + // + // Vector compares producing an output in vcc/SGPR will use s1 in VCC reg + // bank. .legalForCartesianProduct( {S1}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr}) - .legalFor({{S1, S32}, {S1, S64}}); + .legalForCartesianProduct( + {S32}, {S32, S64, GlobalPtr, LocalPtr, ConstantPtr, PrivatePtr, FlatPtr}); if (ST.has16BitInsts()) { CmpBuilder.legalFor({{S1, S16}}); } @@ -502,7 +529,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .widenScalarToNextPow2(1) .clampScalar(1, S32, S64) .scalarize(0) - .legalIf(all(typeIs(0, S1), isPointer(1))); + .legalIf(all(typeInSet(0, {S1, S32}), isPointer(1))); getActionDefinitionsBuilder(G_FCMP) .legalForCartesianProduct({S1}, ST.has16BitInsts() ? FPTypes16 : FPTypesBase) @@ -639,6 +666,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, // Split vector extloads. unsigned MemSize = Query.MMODescrs[0].SizeInBits; + unsigned Align = Query.MMODescrs[0].AlignInBits; + + if (MemSize < DstTy.getSizeInBits()) + MemSize = std::max(MemSize, Align); + if (DstTy.isVector() && DstTy.getSizeInBits() > MemSize) return true; @@ -653,7 +685,6 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, if (NumRegs == 3 && !ST.hasDwordx3LoadStores()) return true; - unsigned Align = Query.MMODescrs[0].AlignInBits; if (Align < MemSize) { const SITargetLowering *TLI = ST.getTargetLowering(); return !TLI->allowsMisalignedMemoryAccessesImpl(MemSize, AS, Align / 8); @@ -795,14 +826,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, unsigned MemSize = Query.MMODescrs[0].SizeInBits; unsigned Align = Query.MMODescrs[0].AlignInBits; - // No extending vector loads. - if (Size > MemSize && Ty0.isVector()) - return false; - // FIXME: Widening store from alignment not valid. if (MemSize < Size) MemSize = std::max(MemSize, Align); + // No extending vector loads. + if (Size > MemSize && Ty0.isVector()) + return false; + switch (MemSize) { case 8: case 16: @@ -848,7 +879,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MAX, G_ATOMICRMW_MIN, G_ATOMICRMW_UMAX, - G_ATOMICRMW_UMIN, G_ATOMIC_CMPXCHG}) + G_ATOMICRMW_UMIN}) .legalFor({{S32, GlobalPtr}, {S32, LocalPtr}, {S64, GlobalPtr}, {S64, LocalPtr}}); if (ST.hasFlatAddressSpace()) { @@ -858,14 +889,24 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder(G_ATOMICRMW_FADD) .legalFor({{S32, LocalPtr}}); + // BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output + // demarshalling + getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG) + .customFor({{S32, GlobalPtr}, {S64, GlobalPtr}, + {S32, FlatPtr}, {S64, FlatPtr}}) + .legalFor({{S32, LocalPtr}, {S64, LocalPtr}, + {S32, RegionPtr}, {S64, RegionPtr}}); + getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS) .lower(); // TODO: Pointer types, any 32-bit or 64-bit vector + + // Condition should be s32 for scalar, s1 for vector. getActionDefinitionsBuilder(G_SELECT) .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16, GlobalPtr, LocalPtr, FlatPtr, PrivatePtr, - LLT::vector(2, LocalPtr), LLT::vector(2, PrivatePtr)}, {S1}) + LLT::vector(2, LocalPtr), LLT::vector(2, PrivatePtr)}, {S1, S32}) .clampScalar(0, S16, S64) .moreElementsIf(isSmallOddVector(0), oneMoreElement(0)) .fewerElementsIf(numElementsNotEven(0), scalarize(0)) @@ -875,7 +916,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, .clampMaxNumElements(0, PrivatePtr, 2) .scalarize(0) .widenScalarToNextPow2(0) - .legalIf(all(isPointer(0), typeIs(1, S1))); + .legalIf(all(isPointer(0), typeInSet(1, {S1, S32}))); // TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can // be more flexible with the shift amount type. @@ -888,7 +929,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, } else Shifts.legalFor({{S16, S32}, {S16, S16}}); - Shifts.clampScalar(1, S16, S32); + // TODO: Support 16-bit shift amounts + Shifts.clampScalar(1, S32, S32); Shifts.clampScalar(0, S16, S64); Shifts.widenScalarToNextPow2(0, 16); } else { @@ -1075,6 +1117,16 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_, getActionDefinitionsBuilder(G_SEXT_INREG).lower(); + getActionDefinitionsBuilder({G_READ_REGISTER, G_WRITE_REGISTER}).lower(); + + getActionDefinitionsBuilder(G_READCYCLECOUNTER) + .legalFor({S64}); + + getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE, + G_DYN_STACKALLOC, G_INDEXED_LOAD, G_INDEXED_SEXTLOAD, + G_INDEXED_ZEXTLOAD, G_INDEXED_STORE}) + .unsupported(); + computeTables(); verify(*ST.getInstrInfo()); } @@ -1116,6 +1168,8 @@ bool AMDGPULegalizerInfo::legalizeCustom(MachineInstr &MI, return legalizeFMad(MI, MRI, B); case TargetOpcode::G_FDIV: return legalizeFDIV(MI, MRI, B); + case TargetOpcode::G_ATOMIC_CMPXCHG: + return legalizeAtomicCmpXChg(MI, MRI, B); default: return false; } @@ -1175,12 +1229,8 @@ Register AMDGPULegalizerInfo::getSegmentAperture( // private_segment_aperture_base_hi. uint32_t StructOffset = (AS == AMDGPUAS::LOCAL_ADDRESS) ? 0x40 : 0x44; - // FIXME: Don't use undef - Value *V = UndefValue::get(PointerType::get( - Type::getInt8Ty(MF.getFunction().getContext()), - AMDGPUAS::CONSTANT_ADDRESS)); - - MachinePointerInfo PtrInfo(V, StructOffset); + // TODO: can we be smarter about machine pointer info? + MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); MachineMemOperand *MMO = MF.getMachineMemOperand( PtrInfo, MachineMemOperand::MOLoad | @@ -1192,7 +1242,7 @@ Register AMDGPULegalizerInfo::getSegmentAperture( Register LoadResult = MRI.createGenericVirtualRegister(S32); Register LoadAddr; - B.materializeGEP(LoadAddr, QueuePtr, LLT::scalar(64), StructOffset); + B.materializePtrAdd(LoadAddr, QueuePtr, LLT::scalar(64), StructOffset); B.buildLoad(LoadResult, LoadAddr, *MMO); return LoadResult; } @@ -1709,13 +1759,15 @@ bool AMDGPULegalizerInfo::legalizeFMad( LLT Ty = MRI.getType(MI.getOperand(0).getReg()); assert(Ty.isScalar()); + MachineFunction &MF = B.getMF(); + const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); + // TODO: Always legal with future ftz flag. - if (Ty == LLT::scalar(32) && !ST.hasFP32Denormals()) + if (Ty == LLT::scalar(32) && !MFI->getMode().FP32Denormals) return true; - if (Ty == LLT::scalar(16) && !ST.hasFP16Denormals()) + if (Ty == LLT::scalar(16) && !MFI->getMode().FP64FP16Denormals) return true; - MachineFunction &MF = B.getMF(); MachineIRBuilder HelperBuilder(MI); GISelObserverWrapper DummyObserver; @@ -1724,16 +1776,55 @@ bool AMDGPULegalizerInfo::legalizeFMad( return Helper.lowerFMad(MI) == LegalizerHelper::Legalized; } +bool AMDGPULegalizerInfo::legalizeAtomicCmpXChg( + MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const { + Register DstReg = MI.getOperand(0).getReg(); + Register PtrReg = MI.getOperand(1).getReg(); + Register CmpVal = MI.getOperand(2).getReg(); + Register NewVal = MI.getOperand(3).getReg(); + + assert(SITargetLowering::isFlatGlobalAddrSpace( + MRI.getType(PtrReg).getAddressSpace()) && + "this should not have been custom lowered"); + + LLT ValTy = MRI.getType(CmpVal); + LLT VecTy = LLT::vector(2, ValTy); + + B.setInstr(MI); + Register PackedVal = B.buildBuildVector(VecTy, { NewVal, CmpVal }).getReg(0); + + B.buildInstr(AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG) + .addDef(DstReg) + .addUse(PtrReg) + .addUse(PackedVal) + .setMemRefs(MI.memoperands()); + + MI.eraseFromParent(); + return true; +} + // Return the use branch instruction, otherwise null if the usage is invalid. static MachineInstr *verifyCFIntrinsic(MachineInstr &MI, - MachineRegisterInfo &MRI) { + MachineRegisterInfo &MRI, + MachineInstr *&Br) { Register CondDef = MI.getOperand(0).getReg(); if (!MRI.hasOneNonDBGUse(CondDef)) return nullptr; MachineInstr &UseMI = *MRI.use_instr_nodbg_begin(CondDef); - return UseMI.getParent() == MI.getParent() && - UseMI.getOpcode() == AMDGPU::G_BRCOND ? &UseMI : nullptr; + if (UseMI.getParent() != MI.getParent() || + UseMI.getOpcode() != AMDGPU::G_BRCOND) + return nullptr; + + // Make sure the cond br is followed by a G_BR + MachineBasicBlock::iterator Next = std::next(UseMI.getIterator()); + if (Next != MI.getParent()->end()) { + if (Next->getOpcode() != AMDGPU::G_BR) + return nullptr; + Br = &*Next; + } + + return &UseMI; } Register AMDGPULegalizerInfo::getLiveInRegister(MachineRegisterInfo &MRI, @@ -1823,10 +1914,22 @@ bool AMDGPULegalizerInfo::legalizeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const { B.setInstr(MI); + Register Dst = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(Dst); + LLT S16 = LLT::scalar(16); + LLT S32 = LLT::scalar(32); + LLT S64 = LLT::scalar(64); if (legalizeFastUnsafeFDIV(MI, MRI, B)) return true; + if (DstTy == S16) + return legalizeFDIV16(MI, MRI, B); + if (DstTy == S32) + return legalizeFDIV32(MI, MRI, B); + if (DstTy == S64) + return legalizeFDIV64(MI, MRI, B); + return false; } @@ -1850,7 +1953,8 @@ bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI, if (!MF.getTarget().Options.UnsafeFPMath && ResTy == S64) return false; - if (!Unsafe && ResTy == S32 && ST.hasFP32Denormals()) + if (!Unsafe && ResTy == S32 && + MF.getInfo<SIMachineFunctionInfo>()->getMode().FP32Denormals) return false; if (auto CLHS = getConstantFPVRegVal(LHS, MRI)) { @@ -1890,6 +1994,219 @@ bool AMDGPULegalizerInfo::legalizeFastUnsafeFDIV(MachineInstr &MI, return false; } +bool AMDGPULegalizerInfo::legalizeFDIV16(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { + B.setInstr(MI); + Register Res = MI.getOperand(0).getReg(); + Register LHS = MI.getOperand(1).getReg(); + Register RHS = MI.getOperand(2).getReg(); + + uint16_t Flags = MI.getFlags(); + + LLT S16 = LLT::scalar(16); + LLT S32 = LLT::scalar(32); + + auto LHSExt = B.buildFPExt(S32, LHS, Flags); + auto RHSExt = B.buildFPExt(S32, RHS, Flags); + + auto RCP = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}, false) + .addUse(RHSExt.getReg(0)) + .setMIFlags(Flags); + + auto QUOT = B.buildFMul(S32, LHSExt, RCP, Flags); + auto RDst = B.buildFPTrunc(S16, QUOT, Flags); + + B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res, false) + .addUse(RDst.getReg(0)) + .addUse(RHS) + .addUse(LHS) + .setMIFlags(Flags); + + MI.eraseFromParent(); + return true; +} + +// Enable or disable FP32 denorm mode. When 'Enable' is true, emit instructions +// to enable denorm mode. When 'Enable' is false, disable denorm mode. +static void toggleSPDenormMode(bool Enable, + MachineIRBuilder &B, + const GCNSubtarget &ST, + AMDGPU::SIModeRegisterDefaults Mode) { + // Set SP denorm mode to this value. + unsigned SPDenormMode = + Enable ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; + + if (ST.hasDenormModeInst()) { + // Preserve default FP64FP16 denorm mode while updating FP32 mode. + unsigned DPDenormModeDefault = Mode.FP64FP16Denormals + ? FP_DENORM_FLUSH_NONE + : FP_DENORM_FLUSH_IN_FLUSH_OUT; + + unsigned NewDenormModeValue = SPDenormMode | (DPDenormModeDefault << 2); + B.buildInstr(AMDGPU::S_DENORM_MODE) + .addImm(NewDenormModeValue); + + } else { + // Select FP32 bit field in mode register. + unsigned SPDenormModeBitField = AMDGPU::Hwreg::ID_MODE | + (4 << AMDGPU::Hwreg::OFFSET_SHIFT_) | + (1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_); + + B.buildInstr(AMDGPU::S_SETREG_IMM32_B32) + .addImm(SPDenormMode) + .addImm(SPDenormModeBitField); + } +} + +bool AMDGPULegalizerInfo::legalizeFDIV32(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { + B.setInstr(MI); + Register Res = MI.getOperand(0).getReg(); + Register LHS = MI.getOperand(1).getReg(); + Register RHS = MI.getOperand(2).getReg(); + const SIMachineFunctionInfo *MFI = B.getMF().getInfo<SIMachineFunctionInfo>(); + AMDGPU::SIModeRegisterDefaults Mode = MFI->getMode(); + + uint16_t Flags = MI.getFlags(); + + LLT S32 = LLT::scalar(32); + LLT S1 = LLT::scalar(1); + + auto One = B.buildFConstant(S32, 1.0f); + + auto DenominatorScaled = + B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1}, false) + .addUse(RHS) + .addUse(LHS) + .addImm(1) + .setMIFlags(Flags); + auto NumeratorScaled = + B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S32, S1}, false) + .addUse(LHS) + .addUse(RHS) + .addImm(0) + .setMIFlags(Flags); + + auto ApproxRcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S32}, false) + .addUse(DenominatorScaled.getReg(0)) + .setMIFlags(Flags); + auto NegDivScale0 = B.buildFNeg(S32, DenominatorScaled, Flags); + + // FIXME: Doesn't correctly model the FP mode switch, and the FP operations + // aren't modeled as reading it. + if (!Mode.FP32Denormals) + toggleSPDenormMode(true, B, ST, Mode); + + auto Fma0 = B.buildFMA(S32, NegDivScale0, ApproxRcp, One, Flags); + auto Fma1 = B.buildFMA(S32, Fma0, ApproxRcp, ApproxRcp, Flags); + auto Mul = B.buildFMul(S32, NumeratorScaled, Fma1, Flags); + auto Fma2 = B.buildFMA(S32, NegDivScale0, Mul, NumeratorScaled, Flags); + auto Fma3 = B.buildFMA(S32, Fma2, Fma1, Mul, Flags); + auto Fma4 = B.buildFMA(S32, NegDivScale0, Fma3, NumeratorScaled, Flags); + + if (!Mode.FP32Denormals) + toggleSPDenormMode(false, B, ST, Mode); + + auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {S32}, false) + .addUse(Fma4.getReg(0)) + .addUse(Fma1.getReg(0)) + .addUse(Fma3.getReg(0)) + .addUse(NumeratorScaled.getReg(1)) + .setMIFlags(Flags); + + B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, Res, false) + .addUse(Fmas.getReg(0)) + .addUse(RHS) + .addUse(LHS) + .setMIFlags(Flags); + + MI.eraseFromParent(); + return true; +} + +bool AMDGPULegalizerInfo::legalizeFDIV64(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &B) const { + B.setInstr(MI); + Register Res = MI.getOperand(0).getReg(); + Register LHS = MI.getOperand(1).getReg(); + Register RHS = MI.getOperand(2).getReg(); + + uint16_t Flags = MI.getFlags(); + + LLT S64 = LLT::scalar(64); + LLT S1 = LLT::scalar(1); + + auto One = B.buildFConstant(S64, 1.0); + + auto DivScale0 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S64, S1}, false) + .addUse(LHS) + .addUse(RHS) + .addImm(1) + .setMIFlags(Flags); + + auto NegDivScale0 = B.buildFNeg(S64, DivScale0.getReg(0), Flags); + + auto Rcp = B.buildIntrinsic(Intrinsic::amdgcn_rcp, {S64}, false) + .addUse(DivScale0.getReg(0)) + .setMIFlags(Flags); + + auto Fma0 = B.buildFMA(S64, NegDivScale0, Rcp, One, Flags); + auto Fma1 = B.buildFMA(S64, Rcp, Fma0, Rcp, Flags); + auto Fma2 = B.buildFMA(S64, NegDivScale0, Fma1, One, Flags); + + auto DivScale1 = B.buildIntrinsic(Intrinsic::amdgcn_div_scale, {S64, S1}, false) + .addUse(LHS) + .addUse(RHS) + .addImm(0) + .setMIFlags(Flags); + + auto Fma3 = B.buildFMA(S64, Fma1, Fma2, Fma1, Flags); + auto Mul = B.buildMul(S64, DivScale1.getReg(0), Fma3, Flags); + auto Fma4 = B.buildFMA(S64, NegDivScale0, Mul, DivScale1.getReg(0), Flags); + + Register Scale; + if (!ST.hasUsableDivScaleConditionOutput()) { + // Workaround a hardware bug on SI where the condition output from div_scale + // is not usable. + + Scale = MRI.createGenericVirtualRegister(S1); + + LLT S32 = LLT::scalar(32); + + auto NumUnmerge = B.buildUnmerge(S32, LHS); + auto DenUnmerge = B.buildUnmerge(S32, RHS); + auto Scale0Unmerge = B.buildUnmerge(S32, DivScale0); + auto Scale1Unmerge = B.buildUnmerge(S32, DivScale1); + + auto CmpNum = B.buildICmp(ICmpInst::ICMP_EQ, S1, NumUnmerge.getReg(1), + Scale1Unmerge.getReg(1)); + auto CmpDen = B.buildICmp(ICmpInst::ICMP_EQ, S1, DenUnmerge.getReg(1), + Scale0Unmerge.getReg(1)); + B.buildXor(Scale, CmpNum, CmpDen); + } else { + Scale = DivScale1.getReg(1); + } + + auto Fmas = B.buildIntrinsic(Intrinsic::amdgcn_div_fmas, {S64}, false) + .addUse(Fma4.getReg(0)) + .addUse(Fma3.getReg(0)) + .addUse(Mul.getReg(0)) + .addUse(Scale) + .setMIFlags(Flags); + + B.buildIntrinsic(Intrinsic::amdgcn_div_fixup, makeArrayRef(Res), false) + .addUse(Fmas.getReg(0)) + .addUse(RHS) + .addUse(LHS) + .setMIFlags(Flags); + + MI.eraseFromParent(); + return true; +} + bool AMDGPULegalizerInfo::legalizeFDIVFastIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const { @@ -1955,7 +2272,7 @@ bool AMDGPULegalizerInfo::legalizeImplicitArgPtr(MachineInstr &MI, if (!loadInputValue(KernargPtrReg, B, Arg)) return false; - B.buildGEP(DstReg, KernargPtrReg, B.buildConstant(IdxTy, Offset).getReg(0)); + B.buildPtrAdd(DstReg, KernargPtrReg, B.buildConstant(IdxTy, Offset).getReg(0)); MI.eraseFromParent(); return true; } @@ -2032,19 +2349,38 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const { // Replace the use G_BRCOND with the exec manipulate and branch pseudos. - switch (MI.getIntrinsicID()) { - case Intrinsic::amdgcn_if: { - if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI)) { + auto IntrID = MI.getIntrinsicID(); + switch (IntrID) { + case Intrinsic::amdgcn_if: + case Intrinsic::amdgcn_else: { + MachineInstr *Br = nullptr; + if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI, Br)) { const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo()); B.setInstr(*BrCond); Register Def = MI.getOperand(1).getReg(); Register Use = MI.getOperand(3).getReg(); - B.buildInstr(AMDGPU::SI_IF) - .addDef(Def) - .addUse(Use) - .addMBB(BrCond->getOperand(1).getMBB()); + + MachineBasicBlock *BrTarget = BrCond->getOperand(1).getMBB(); + if (Br) + BrTarget = Br->getOperand(0).getMBB(); + + if (IntrID == Intrinsic::amdgcn_if) { + B.buildInstr(AMDGPU::SI_IF) + .addDef(Def) + .addUse(Use) + .addMBB(BrTarget); + } else { + B.buildInstr(AMDGPU::SI_ELSE) + .addDef(Def) + .addUse(Use) + .addMBB(BrTarget) + .addImm(0); + } + + if (Br) + Br->getOperand(0).setMBB(BrCond->getOperand(1).getMBB()); MRI.setRegClass(Def, TRI->getWaveMaskRegClass()); MRI.setRegClass(Use, TRI->getWaveMaskRegClass()); @@ -2056,11 +2392,14 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(MachineInstr &MI, return false; } case Intrinsic::amdgcn_loop: { - if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI)) { + MachineInstr *Br = nullptr; + if (MachineInstr *BrCond = verifyCFIntrinsic(MI, MRI, Br)) { const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(MRI.getTargetRegisterInfo()); B.setInstr(*BrCond); + + // FIXME: Need to adjust branch targets based on unconditional branch. Register Reg = MI.getOperand(2).getReg(); B.buildInstr(AMDGPU::SI_LOOP) .addUse(Reg) diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h index d0fba23a8686..4b1405a92787 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.h @@ -72,6 +72,9 @@ public: bool legalizeFMad(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; + bool legalizeAtomicCmpXChg(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; + Register getLiveInRegister(MachineRegisterInfo &MRI, Register Reg, LLT Ty) const; @@ -83,6 +86,12 @@ public: bool legalizeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; + bool legalizeFDIV16(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; + bool legalizeFDIV32(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; + bool legalizeFDIV64(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &B) const; bool legalizeFastUnsafeFDIV(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const; bool legalizeFDIVFastIntrin(MachineInstr &MI, MachineRegisterInfo &MRI, diff --git a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp index 2c94e0046651..0c56927dea02 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPULibCalls.cpp @@ -11,31 +11,32 @@ // //===----------------------------------------------------------------------===// -#define DEBUG_TYPE "amdgpu-simplifylib" - #include "AMDGPU.h" #include "AMDGPULibFunc.h" #include "AMDGPUSubtarget.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/StringSet.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/Loads.h" -#include "llvm/ADT/StringSet.h" -#include "llvm/ADT/StringRef.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/IRBuilder.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" -#include "llvm/IR/IRBuilder.h" -#include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/ValueSymbolTable.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" -#include <vector> #include <cmath> +#include <vector> + +#define DEBUG_TYPE "amdgpu-simplifylib" using namespace llvm; @@ -1167,8 +1168,6 @@ bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B, return true; } if (ci_opr1 == 2) { // rootn(x, 2) = sqrt(x) - std::vector<const Type*> ParamsTys; - ParamsTys.push_back(opr0->getType()); Module *M = CI->getModule(); if (FunctionCallee FPExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_SQRT, FInfo))) { @@ -1194,8 +1193,6 @@ bool AMDGPULibCalls::fold_rootn(CallInst *CI, IRBuilder<> &B, replaceCall(nval); return true; } else if (ci_opr1 == -2) { // rootn(x, -2) = rsqrt(x) - std::vector<const Type*> ParamsTys; - ParamsTys.push_back(opr0->getType()); Module *M = CI->getModule(); if (FunctionCallee FPExpr = getFunction(M, AMDGPULibFunc(AMDGPULibFunc::EI_RSQRT, FInfo))) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp index 3760aed87a43..ce7286dabcc8 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMCInstLower.cpp @@ -344,7 +344,7 @@ void AMDGPUAsmPrinter::EmitInstruction(const MachineInstr *MI) { AMDGPUInstPrinter InstPrinter(*TM.getMCAsmInfo(), *STI.getInstrInfo(), *STI.getRegisterInfo()); - InstPrinter.printInst(&TmpInst, DisasmStream, StringRef(), STI); + InstPrinter.printInst(&TmpInst, 0, StringRef(), STI, DisasmStream); // Disassemble instruction/operands to hex representation. SmallVector<MCFixup, 4> Fixups; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp index ba72f71f4322..f61af5a27943 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp @@ -32,6 +32,7 @@ #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/DebugLoc.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/Debug.h" @@ -861,7 +862,7 @@ void LinearizedRegion::storeLiveOuts(RegionMRT *Region, void LinearizedRegion::print(raw_ostream &OS, const TargetRegisterInfo *TRI) { OS << "Linearized Region {"; bool IsFirst = true; - for (const auto &MBB : MBBs) { + for (auto MBB : MBBs) { if (IsFirst) { IsFirst = false; } else { @@ -995,7 +996,7 @@ MachineBasicBlock *LinearizedRegion::getExit() { return Exit; } void LinearizedRegion::addMBB(MachineBasicBlock *MBB) { MBBs.insert(MBB); } void LinearizedRegion::addMBBs(LinearizedRegion *InnerRegion) { - for (const auto &MBB : InnerRegion->MBBs) { + for (auto MBB : InnerRegion->MBBs) { addMBB(MBB); } } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp index 89ca702f577d..940ddff85d73 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.cpp @@ -18,6 +18,7 @@ AMDGPUMachineFunction::AMDGPUMachineFunction(const MachineFunction &MF) : LocalMemoryObjects(), ExplicitKernArgSize(0), LDSSize(0), + Mode(MF.getFunction(), MF.getSubtarget<GCNSubtarget>()), IsEntryFunction(AMDGPU::isEntryFunctionCC(MF.getFunction().getCallingConv())), NoSignedZerosFPMath(MF.getTarget().Options.NoSignedZerosFPMath), MemoryBound(false), diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h index 9818ab1ef148..1933e41c66f3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h @@ -11,6 +11,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/MachineFunction.h" +#include "Utils/AMDGPUBaseInfo.h" namespace llvm { @@ -28,6 +29,9 @@ protected: /// Number of bytes in the LDS that are being used. unsigned LDSSize; + // State of MODE register, assumed FP mode. + AMDGPU::SIModeRegisterDefaults Mode; + // Kernels + shaders. i.e. functions called by the driver and not called // by other functions. bool IsEntryFunction; @@ -53,6 +57,10 @@ public: return LDSSize; } + AMDGPU::SIModeRegisterDefaults getMode() const { + return Mode; + } + bool isEntryFunction() const { return IsEntryFunction; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp index 5250bf455d71..511de96b5f7c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUPrintfRuntimeBinding.cpp @@ -33,6 +33,7 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -581,6 +582,15 @@ bool AMDGPUPrintfRuntimeBinding::runOnModule(Module &M) { if (Printfs.empty()) return false; + if (auto HostcallFunction = M.getFunction("__ockl_hostcall_internal")) { + for (auto &U : HostcallFunction->uses()) { + if (auto *CI = dyn_cast<CallInst>(U.getUser())) { + M.getContext().emitError( + CI, "Cannot use both printf and hostcall in the same module"); + } + } + } + TD = &M.getDataLayout(); auto DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>(); DT = DTWP ? &DTWP->getDomTree() : nullptr; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp index 3e9dcca114a3..14958a180ce3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp @@ -37,6 +37,8 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicsAMDGPU.h" +#include "llvm/IR/IntrinsicsR600.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" @@ -268,21 +270,21 @@ AMDGPUPromoteAlloca::getLocalSizeYZ(IRBuilder<> &Builder) { Value *AMDGPUPromoteAlloca::getWorkitemID(IRBuilder<> &Builder, unsigned N) { const AMDGPUSubtarget &ST = AMDGPUSubtarget::get(*TM, *Builder.GetInsertBlock()->getParent()); - Intrinsic::ID IntrID = Intrinsic::ID::not_intrinsic; + Intrinsic::ID IntrID = Intrinsic::not_intrinsic; switch (N) { case 0: - IntrID = IsAMDGCN ? Intrinsic::amdgcn_workitem_id_x - : Intrinsic::r600_read_tidig_x; + IntrID = IsAMDGCN ? (Intrinsic::ID)Intrinsic::amdgcn_workitem_id_x + : (Intrinsic::ID)Intrinsic::r600_read_tidig_x; break; case 1: - IntrID = IsAMDGCN ? Intrinsic::amdgcn_workitem_id_y - : Intrinsic::r600_read_tidig_y; + IntrID = IsAMDGCN ? (Intrinsic::ID)Intrinsic::amdgcn_workitem_id_y + : (Intrinsic::ID)Intrinsic::r600_read_tidig_y; break; case 2: - IntrID = IsAMDGCN ? Intrinsic::amdgcn_workitem_id_z - : Intrinsic::r600_read_tidig_z; + IntrID = IsAMDGCN ? (Intrinsic::ID)Intrinsic::amdgcn_workitem_id_z + : (Intrinsic::ID)Intrinsic::r600_read_tidig_z; break; default: llvm_unreachable("invalid dimension"); @@ -648,7 +650,7 @@ bool AMDGPUPromoteAlloca::hasSufficientLocalMem(const Function &F) { // Check how much local memory is being used by global objects CurrentLocalMemUsage = 0; for (GlobalVariable &GV : Mod->globals()) { - if (GV.getType()->getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) + if (GV.getAddressSpace() != AMDGPUAS::LOCAL_ADDRESS) continue; for (const User *U : GV.users()) { @@ -882,25 +884,25 @@ bool AMDGPUPromoteAlloca::handleAlloca(AllocaInst &I, bool SufficientLDS) { continue; case Intrinsic::memcpy: { MemCpyInst *MemCpy = cast<MemCpyInst>(Intr); - Builder.CreateMemCpy(MemCpy->getRawDest(), MemCpy->getDestAlignment(), - MemCpy->getRawSource(), MemCpy->getSourceAlignment(), + Builder.CreateMemCpy(MemCpy->getRawDest(), MemCpy->getDestAlign(), + MemCpy->getRawSource(), MemCpy->getSourceAlign(), MemCpy->getLength(), MemCpy->isVolatile()); Intr->eraseFromParent(); continue; } case Intrinsic::memmove: { MemMoveInst *MemMove = cast<MemMoveInst>(Intr); - Builder.CreateMemMove(MemMove->getRawDest(), MemMove->getDestAlignment(), - MemMove->getRawSource(), MemMove->getSourceAlignment(), + Builder.CreateMemMove(MemMove->getRawDest(), MemMove->getDestAlign(), + MemMove->getRawSource(), MemMove->getSourceAlign(), MemMove->getLength(), MemMove->isVolatile()); Intr->eraseFromParent(); continue; } case Intrinsic::memset: { MemSetInst *MemSet = cast<MemSetInst>(Intr); - Builder.CreateMemSet(MemSet->getRawDest(), MemSet->getValue(), - MemSet->getLength(), MemSet->getDestAlignment(), - MemSet->isVolatile()); + Builder.CreateMemSet( + MemSet->getRawDest(), MemSet->getValue(), MemSet->getLength(), + MaybeAlign(MemSet->getDestAlignment()), MemSet->isVolatile()); Intr->eraseFromParent(); continue; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp index 4d78188b3dc3..1bb01dc8fa11 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp @@ -17,6 +17,7 @@ #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIMachineFunctionInfo.h" #include "SIRegisterInfo.h" +#include "llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h" #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" @@ -40,13 +41,15 @@ namespace { // Observer to apply a register bank to new registers created by LegalizerHelper. class ApplyRegBankMapping final : public GISelChangeObserver { private: + const AMDGPURegisterBankInfo &RBI; MachineRegisterInfo &MRI; const RegisterBank *NewBank; SmallVector<MachineInstr *, 4> NewInsts; public: - ApplyRegBankMapping(MachineRegisterInfo &MRI_, const RegisterBank *RB) - : MRI(MRI_), NewBank(RB) {} + ApplyRegBankMapping(const AMDGPURegisterBankInfo &RBI_, + MachineRegisterInfo &MRI_, const RegisterBank *RB) + : RBI(RBI_), MRI(MRI_), NewBank(RB) {} ~ApplyRegBankMapping() { for (MachineInstr *MI : NewInsts) @@ -55,6 +58,45 @@ public: /// Set any registers that don't have a set register class or bank to SALU. void applyBank(MachineInstr &MI) { + const unsigned Opc = MI.getOpcode(); + if (Opc == AMDGPU::G_ANYEXT || Opc == AMDGPU::G_ZEXT || + Opc == AMDGPU::G_SEXT) { + // LegalizerHelper wants to use the basic legalization artifacts when + // widening etc. We don't handle selection with vcc in artifact sources, + // so we need to use a sslect instead to handle these properly. + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + const RegisterBank *SrcBank = RBI.getRegBank(SrcReg, MRI, *RBI.TRI); + if (SrcBank == &AMDGPU::VCCRegBank) { + const LLT S32 = LLT::scalar(32); + assert(MRI.getType(SrcReg) == LLT::scalar(1)); + assert(MRI.getType(DstReg) == S32); + assert(NewBank == &AMDGPU::VGPRRegBank); + + // Replace the extension with a select, which really uses the boolean + // source. + MachineIRBuilder B(MI); + auto True = B.buildConstant(S32, Opc == AMDGPU::G_SEXT ? -1 : 1); + auto False = B.buildConstant(S32, 0); + B.buildSelect(DstReg, SrcReg, True, False); + MRI.setRegBank(True.getReg(0), *NewBank); + MRI.setRegBank(False.getReg(0), *NewBank); + MI.eraseFromParent(); + } + + assert(!MRI.getRegClassOrRegBank(DstReg)); + MRI.setRegBank(DstReg, *NewBank); + return; + } + +#ifndef NDEBUG + if (Opc == AMDGPU::G_TRUNC) { + Register DstReg = MI.getOperand(0).getReg(); + const RegisterBank *DstBank = RBI.getRegBank(DstReg, MRI, *RBI.TRI); + assert(DstBank != &AMDGPU::VCCRegBank); + } +#endif + for (MachineOperand &Op : MI.operands()) { if (!Op.isReg()) continue; @@ -64,10 +106,14 @@ public: continue; const RegisterBank *RB = NewBank; - // FIXME: This might not be enough to detect when SCC should be used. - if (MRI.getType(Reg) == LLT::scalar(1)) - RB = (NewBank == &AMDGPU::SGPRRegBank ? - &AMDGPU::SCCRegBank : &AMDGPU::VCCRegBank); + if (MRI.getType(Reg) == LLT::scalar(1)) { + assert(NewBank == &AMDGPU::VGPRRegBank && + "s1 operands should only be used for vector bools"); + assert((MI.getOpcode() != AMDGPU::G_TRUNC && + MI.getOpcode() != AMDGPU::G_ANYEXT) && + "not expecting legalization artifacts here"); + RB = &AMDGPU::VCCRegBank; + } MRI.setRegBank(Reg, *RB); } @@ -98,14 +144,14 @@ AMDGPURegisterBankInfo::AMDGPURegisterBankInfo(const GCNSubtarget &ST) AlreadyInit = true; - const RegisterBank &RBSGPR = getRegBank(AMDGPU::SGPRRegBankID); - (void)RBSGPR; - assert(&RBSGPR == &AMDGPU::SGPRRegBank); - - const RegisterBank &RBVGPR = getRegBank(AMDGPU::VGPRRegBankID); - (void)RBVGPR; - assert(&RBVGPR == &AMDGPU::VGPRRegBank); + assert(&getRegBank(AMDGPU::SGPRRegBankID) == &AMDGPU::SGPRRegBank && + &getRegBank(AMDGPU::VGPRRegBankID) == &AMDGPU::VGPRRegBank && + &getRegBank(AMDGPU::AGPRRegBankID) == &AMDGPU::AGPRRegBank); +} +static bool isVectorRegisterBank(const RegisterBank &Bank) { + unsigned BankID = Bank.getID(); + return BankID == AMDGPU::VGPRRegBankID || BankID == AMDGPU::AGPRRegBankID; } unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst, @@ -113,7 +159,7 @@ unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst, unsigned Size) const { // TODO: Should there be a UniformVGPRRegBank which can use readfirstlane? if (Dst.getID() == AMDGPU::SGPRRegBankID && - Src.getID() == AMDGPU::VGPRRegBankID) { + isVectorRegisterBank(Src)) { return std::numeric_limits<unsigned>::max(); } @@ -125,17 +171,20 @@ unsigned AMDGPURegisterBankInfo::copyCost(const RegisterBank &Dst, // have been a truncate from an arbitrary value, in which case a copy (lowered // as a compare with 0) needs to be inserted. if (Size == 1 && - (Dst.getID() == AMDGPU::SCCRegBankID || - Dst.getID() == AMDGPU::SGPRRegBankID) && - (Src.getID() == AMDGPU::SGPRRegBankID || - Src.getID() == AMDGPU::VGPRRegBankID || + (Dst.getID() == AMDGPU::SGPRRegBankID) && + (isVectorRegisterBank(Src) || + Src.getID() == AMDGPU::SGPRRegBankID || Src.getID() == AMDGPU::VCCRegBankID)) return std::numeric_limits<unsigned>::max(); - if (Dst.getID() == AMDGPU::SCCRegBankID && - Src.getID() == AMDGPU::VCCRegBankID) + if (Src.getID() == AMDGPU::VCCRegBankID) return std::numeric_limits<unsigned>::max(); + // There is no direct copy between AGPRs. + if (Dst.getID() == AMDGPU::AGPRRegBankID && + Src.getID() == AMDGPU::AGPRRegBankID) + return 4; + return RegisterBankInfo::copyCost(Dst, Src, Size); } @@ -164,12 +213,25 @@ unsigned AMDGPURegisterBankInfo::getBreakDownCost( return 1; } -const RegisterBank &AMDGPURegisterBankInfo::getRegBankFromRegClass( - const TargetRegisterClass &RC) const { +const RegisterBank & +AMDGPURegisterBankInfo::getRegBankFromRegClass(const TargetRegisterClass &RC, + LLT Ty) const { if (&RC == &AMDGPU::SReg_1RegClass) return AMDGPU::VCCRegBank; - return TRI->isSGPRClass(&RC) ? AMDGPU::SGPRRegBank : AMDGPU::VGPRRegBank; + // We promote real scalar booleans to SReg_32. Any SGPR using s1 is really a + // VCC-like use. + if (TRI->isSGPRClass(&RC)) { + // FIXME: This probably came from a copy from a physical register, which + // should be inferrrable from the copied to-type. We don't have many boolean + // physical register constraints so just assume a normal SGPR for now. + if (!Ty.isValid()) + return AMDGPU::SGPRRegBank; + + return Ty == LLT::scalar(1) ? AMDGPU::VCCRegBank : AMDGPU::SGPRRegBank; + } + + return TRI->isAGPRClass(&RC) ? AMDGPU::AGPRRegBank : AMDGPU::VGPRRegBank; } template <unsigned NumOps> @@ -323,15 +385,32 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects( } } +static bool memOpHasNoClobbered(const MachineMemOperand *MMO) { + const Instruction *I = dyn_cast_or_null<Instruction>(MMO->getValue()); + return I && I->getMetadata("amdgpu.noclobber"); +} + // FIXME: Returns uniform if there's no source value information. This is // probably wrong. -static bool isInstrUniformNonExtLoadAlign4(const MachineInstr &MI) { +static bool isScalarLoadLegal(const MachineInstr &MI) { if (!MI.hasOneMemOperand()) return false; const MachineMemOperand *MMO = *MI.memoperands_begin(); + const unsigned AS = MMO->getAddrSpace(); + const bool IsConst = AS == AMDGPUAS::CONSTANT_ADDRESS || + AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; + + // There are no extending SMRD/SMEM loads, and they require 4-byte alignment. return MMO->getSize() >= 4 && MMO->getAlignment() >= 4 && - AMDGPUInstrInfo::isUniformMMO(MMO); + // Can't do a scalar atomic load. + !MMO->isAtomic() && + // Don't use scalar loads for volatile accesses to non-constant address + // spaces. + (IsConst || !MMO->isVolatile()) && + // Memory must be known constant, or not written before this load. + (IsConst || MMO->isInvariant() || memOpHasNoClobbered(MMO)) && + AMDGPUInstrInfo::isUniformMMO(MMO); } RegisterBankInfo::InstructionMappings @@ -347,11 +426,10 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( case TargetOpcode::G_CONSTANT: { unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); if (Size == 1) { - static const OpRegBankEntry<1> Table[4] = { + static const OpRegBankEntry<1> Table[3] = { { { AMDGPU::VGPRRegBankID }, 1 }, { { AMDGPU::SGPRRegBankID }, 1 }, - { { AMDGPU::VCCRegBankID }, 1 }, - { { AMDGPU::SCCRegBankID }, 1 } + { { AMDGPU::VCCRegBankID }, 1 } }; return addMappingFromTable<1>(MI, MRI, {{ 0 }}, Table); @@ -378,25 +456,17 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( // s_{and|or|xor}_b32 set scc when the result of the 32-bit op is not 0. const InstructionMapping &SCCMapping = getInstructionMapping( 1, 1, getOperandsMapping( - {AMDGPU::getValueMapping(AMDGPU::SCCRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), + {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32), + AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32), + AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 32)}), 3); // Num Operands AltMappings.push_back(&SCCMapping); - const InstructionMapping &SGPRMapping = getInstructionMapping( - 1, 1, getOperandsMapping( - {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), - 3); // Num Operands - AltMappings.push_back(&SGPRMapping); - const InstructionMapping &VCCMapping0 = getInstructionMapping( - 2, 10, getOperandsMapping( + 2, 1, getOperandsMapping( {AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size)}), + AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size), + AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, Size)}), 3); // Num Operands AltMappings.push_back(&VCCMapping0); return AltMappings; @@ -448,9 +518,10 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( unsigned PtrSize = PtrTy.getSizeInBits(); unsigned AS = PtrTy.getAddressSpace(); LLT LoadTy = MRI.getType(MI.getOperand(0).getReg()); + if ((AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS && AS != AMDGPUAS::PRIVATE_ADDRESS) && - isInstrUniformNonExtLoadAlign4(MI)) { + isScalarLoadLegal(MI)) { const InstructionMapping &SSMapping = getInstructionMapping( 1, 1, getOperandsMapping( {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), @@ -476,9 +547,10 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( } case TargetOpcode::G_ICMP: { + // TODO: Should report 32-bit for scalar output type. unsigned Size = getSizeInBits(MI.getOperand(2).getReg(), MRI, *TRI); const InstructionMapping &SSMapping = getInstructionMapping(1, 1, - getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SCCRegBankID, 1), + getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), nullptr, // Predicate operand. AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), @@ -515,7 +587,7 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, *TRI); const InstructionMapping &SSMapping = getInstructionMapping(1, 1, getOperandsMapping({AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::SCCRegBankID, 1), + AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size)}), 4); // Num Operands @@ -556,10 +628,10 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( const InstructionMapping &SSMapping = getInstructionMapping(1, 1, getOperandsMapping( {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::SCCRegBankID, 1), + AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size), - AMDGPU::getValueMapping(AMDGPU::SCCRegBankID, 1)}), + AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1)}), 5); // Num Operands AltMappings.push_back(&SSMapping); @@ -576,9 +648,10 @@ AMDGPURegisterBankInfo::getInstrAlternativeMappings( case AMDGPU::G_BRCOND: { assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1); + // TODO: Change type to 32 for scalar const InstructionMapping &SMapping = getInstructionMapping( 1, 1, getOperandsMapping( - {AMDGPU::getValueMapping(AMDGPU::SCCRegBankID, 1), nullptr}), + {AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, 1), nullptr}), 2); // Num Operands AltMappings.push_back(&SMapping); @@ -910,7 +983,7 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop( Op.setReg(Merge.getReg(0)); } - MRI.setRegBank(Op.getReg(), getRegBank(AMDGPU::SGPRRegBankID)); + MRI.setRegBank(Op.getReg(), AMDGPU::SGPRRegBank); } } } @@ -948,8 +1021,9 @@ bool AMDGPURegisterBankInfo::executeInWaterfallLoop( .addDef(ExecReg) .addReg(SaveExecReg); - // Restore the insert point before the original instruction. - B.setInsertPt(MBB, MBB.end()); + // Set the insert point after the original instruction, so any new + // instructions will be in the remainder. + B.setInsertPt(*RemainderBB, RemainderBB->begin()); return true; } @@ -1004,11 +1078,13 @@ void AMDGPURegisterBankInfo::constrainOpWithReadfirstlane( return; MachineIRBuilder B(MI); - Register SGPR = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); + Register SGPR = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); B.buildInstr(AMDGPU::V_READFIRSTLANE_B32) .addDef(SGPR) .addReg(Reg); + MRI.setType(SGPR, MRI.getType(Reg)); + const TargetRegisterClass *Constrained = constrainGenericRegister(Reg, AMDGPU::VGPR_32RegClass, MRI); (void)Constrained; @@ -1050,11 +1126,11 @@ bool AMDGPURegisterBankInfo::applyMappingWideLoad(MachineInstr &MI, // If the pointer is an SGPR, we have nothing to do. if (SrcRegs.empty()) { - Register PtrReg = MI.getOperand(1).getReg(); - const RegisterBank *PtrBank = getRegBank(PtrReg, MRI, *TRI); + const RegisterBank *PtrBank = + OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; if (PtrBank == &AMDGPU::SGPRRegBank) return false; - SrcRegs.push_back(PtrReg); + SrcRegs.push_back(MI.getOperand(1).getReg()); } assert(LoadSize % MaxNonSmrdLoadSize == 0); @@ -1075,7 +1151,7 @@ bool AMDGPURegisterBankInfo::applyMappingWideLoad(MachineInstr &MI, unsigned SplitElts = MaxNonSmrdLoadSize / LoadTy.getScalarType().getSizeInBits(); const LLT LoadSplitTy = LLT::vector(SplitElts, LoadTy.getScalarType()); - ApplyRegBankMapping O(MRI, &AMDGPU::VGPRRegBank); + ApplyRegBankMapping O(*this, MRI, &AMDGPU::VGPRRegBank); GISelObserverWrapper Observer(&O); B.setChangeObserver(Observer); LegalizerHelper Helper(B.getMF(), Observer, B); @@ -1099,11 +1175,11 @@ bool AMDGPURegisterBankInfo::applyMappingWideLoad(MachineInstr &MI, for (unsigned DefIdx = 0, e = DefRegs.size(); DefIdx != e; ++DefIdx) { Register IdxReg = MRI.createGenericVirtualRegister(LLT::scalar(32)); B.buildConstant(IdxReg, DefIdx); - MRI.setRegBank(IdxReg, getRegBank(AMDGPU::VGPRRegBankID)); + MRI.setRegBank(IdxReg, AMDGPU::VGPRRegBank); B.buildExtractVectorElement(DefRegs[DefIdx], TmpReg, IdxReg); } - MRI.setRegBank(DstReg, getRegBank(AMDGPU::VGPRRegBankID)); + MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); return true; } @@ -1134,6 +1210,39 @@ bool AMDGPURegisterBankInfo::applyMappingImage( return true; } +// FIXME: Duplicated from LegalizerHelper +static CmpInst::Predicate minMaxToCompare(unsigned Opc) { + switch (Opc) { + case TargetOpcode::G_SMIN: + return CmpInst::ICMP_SLT; + case TargetOpcode::G_SMAX: + return CmpInst::ICMP_SGT; + case TargetOpcode::G_UMIN: + return CmpInst::ICMP_ULT; + case TargetOpcode::G_UMAX: + return CmpInst::ICMP_UGT; + default: + llvm_unreachable("not in integer min/max"); + } +} + +// FIXME: Duplicated from LegalizerHelper, except changing the boolean type. +void AMDGPURegisterBankInfo::lowerScalarMinMax(MachineIRBuilder &B, + MachineInstr &MI) const { + Register Dst = MI.getOperand(0).getReg(); + Register Src0 = MI.getOperand(1).getReg(); + Register Src1 = MI.getOperand(2).getReg(); + + const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); + LLT CmpType = LLT::scalar(32); + + auto Cmp = B.buildICmp(Pred, CmpType, Src0, Src1); + B.buildSelect(Dst, Cmp, Src0, Src1); + + B.getMRI()->setRegBank(Cmp.getReg(0), AMDGPU::SGPRRegBank); + MI.eraseFromParent(); +} + // For cases where only a single copy is inserted for matching register banks. // Replace the register in the instruction operand static void substituteSimpleCopyRegs( @@ -1323,22 +1432,161 @@ AMDGPURegisterBankInfo::selectStoreIntrinsic(MachineIRBuilder &B, return MIB; } +bool AMDGPURegisterBankInfo::buildVCopy(MachineIRBuilder &B, Register DstReg, + Register SrcReg) const { + MachineRegisterInfo &MRI = *B.getMRI(); + LLT SrcTy = MRI.getType(SrcReg); + if (SrcTy.getSizeInBits() == 32) { + // Use a v_mov_b32 here to make the exec dependency explicit. + B.buildInstr(AMDGPU::V_MOV_B32_e32) + .addDef(DstReg) + .addUse(SrcReg); + return constrainGenericRegister(DstReg, AMDGPU::VGPR_32RegClass, MRI) && + constrainGenericRegister(SrcReg, AMDGPU::SReg_32RegClass, MRI); + } + + Register TmpReg0 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); + Register TmpReg1 = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass); + + B.buildInstr(AMDGPU::V_MOV_B32_e32) + .addDef(TmpReg0) + .addUse(SrcReg, 0, AMDGPU::sub0); + B.buildInstr(AMDGPU::V_MOV_B32_e32) + .addDef(TmpReg1) + .addUse(SrcReg, 0, AMDGPU::sub1); + B.buildInstr(AMDGPU::REG_SEQUENCE) + .addDef(DstReg) + .addUse(TmpReg0) + .addImm(AMDGPU::sub0) + .addUse(TmpReg1) + .addImm(AMDGPU::sub1); + + return constrainGenericRegister(SrcReg, AMDGPU::SReg_64RegClass, MRI) && + constrainGenericRegister(DstReg, AMDGPU::VReg_64RegClass, MRI); +} + void AMDGPURegisterBankInfo::applyMappingImpl( const OperandsMapper &OpdMapper) const { MachineInstr &MI = OpdMapper.getMI(); unsigned Opc = MI.getOpcode(); MachineRegisterInfo &MRI = OpdMapper.getMRI(); switch (Opc) { + case AMDGPU::G_PHI: { + Register DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + if (DstTy != LLT::scalar(1)) + break; + + const LLT S32 = LLT::scalar(32); + const RegisterBank *DstBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; + if (DstBank == &AMDGPU::VCCRegBank) { + applyDefaultMapping(OpdMapper); + // The standard handling only considers the result register bank for + // phis. For VCC, blindly inserting a copy when the phi is lowered will + // produce an invalid copy. We can only copy with some kind of compare to + // get a vector boolean result. Insert a regitser bank copy that will be + // correctly lowered to a compare. + MachineIRBuilder B(*MI.getParent()->getParent()); + + for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { + Register SrcReg = MI.getOperand(I).getReg(); + const RegisterBank *SrcBank = getRegBank(SrcReg, MRI, *TRI); + + if (SrcBank != &AMDGPU::VCCRegBank) { + MachineBasicBlock *SrcMBB = MI.getOperand(I + 1).getMBB(); + B.setInsertPt(*SrcMBB, SrcMBB->getFirstTerminator()); + + auto Copy = B.buildCopy(LLT::scalar(1), SrcReg); + MRI.setRegBank(Copy.getReg(0), AMDGPU::VCCRegBank); + MI.getOperand(I).setReg(Copy.getReg(0)); + } + } + + return; + } + + // Phi handling is strange and only considers the bank of the destination. + substituteSimpleCopyRegs(OpdMapper, 0); + + // Promote SGPR/VGPR booleans to s32 + MachineFunction *MF = MI.getParent()->getParent(); + ApplyRegBankMapping ApplyBank(*this, MRI, DstBank); + GISelObserverWrapper Observer(&ApplyBank); + MachineIRBuilder B(MI); + LegalizerHelper Helper(*MF, Observer, B); + + if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized) + llvm_unreachable("widen scalar should have succeeded"); + + return; + } + case AMDGPU::G_ICMP: + case AMDGPU::G_UADDO: + case AMDGPU::G_USUBO: + case AMDGPU::G_UADDE: + case AMDGPU::G_SADDE: + case AMDGPU::G_USUBE: + case AMDGPU::G_SSUBE: { + unsigned BoolDstOp = Opc == AMDGPU::G_ICMP ? 0 : 1; + Register DstReg = MI.getOperand(BoolDstOp).getReg(); + + const RegisterBank *DstBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; + if (DstBank != &AMDGPU::SGPRRegBank) + break; + + const bool HasCarryIn = MI.getNumOperands() == 5; + + // If this is a scalar compare, promote the result to s32, as the selection + // will end up using a copy to a 32-bit vreg. + const LLT S32 = LLT::scalar(32); + Register NewDstReg = MRI.createGenericVirtualRegister(S32); + MRI.setRegBank(NewDstReg, AMDGPU::SGPRRegBank); + MI.getOperand(BoolDstOp).setReg(NewDstReg); + MachineIRBuilder B(MI); + + if (HasCarryIn) { + Register NewSrcReg = MRI.createGenericVirtualRegister(S32); + MRI.setRegBank(NewSrcReg, AMDGPU::SGPRRegBank); + B.buildZExt(NewSrcReg, MI.getOperand(4).getReg()); + MI.getOperand(4).setReg(NewSrcReg); + } + + MachineBasicBlock *MBB = MI.getParent(); + B.setInsertPt(*MBB, std::next(MI.getIterator())); + B.buildTrunc(DstReg, NewDstReg); + return; + } case AMDGPU::G_SELECT: { Register DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); + + SmallVector<Register, 1> CondRegs(OpdMapper.getVRegs(1)); + if (CondRegs.empty()) + CondRegs.push_back(MI.getOperand(1).getReg()); + else { + assert(CondRegs.size() == 1); + } + + const RegisterBank *CondBank = getRegBank(CondRegs[0], MRI, *TRI); + if (CondBank == &AMDGPU::SGPRRegBank) { + MachineIRBuilder B(MI); + const LLT S32 = LLT::scalar(32); + Register NewCondReg = MRI.createGenericVirtualRegister(S32); + MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank); + + MI.getOperand(1).setReg(NewCondReg); + B.buildZExt(NewCondReg, CondRegs[0]); + } + if (DstTy.getSizeInBits() != 64) break; + MachineIRBuilder B(MI); LLT HalfTy = getHalfSizedType(DstTy); SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0)); - SmallVector<Register, 1> Src0Regs(OpdMapper.getVRegs(1)); SmallVector<Register, 2> Src1Regs(OpdMapper.getVRegs(2)); SmallVector<Register, 2> Src2Regs(OpdMapper.getVRegs(3)); @@ -1348,13 +1596,6 @@ void AMDGPURegisterBankInfo::applyMappingImpl( break; } - MachineIRBuilder B(MI); - if (Src0Regs.empty()) - Src0Regs.push_back(MI.getOperand(1).getReg()); - else { - assert(Src0Regs.size() == 1); - } - if (Src1Regs.empty()) split64BitValueForMapping(B, Src1Regs, HalfTy, MI.getOperand(2).getReg()); else { @@ -1368,13 +1609,32 @@ void AMDGPURegisterBankInfo::applyMappingImpl( setRegsToType(MRI, DefRegs, HalfTy); - B.buildSelect(DefRegs[0], Src0Regs[0], Src1Regs[0], Src2Regs[0]); - B.buildSelect(DefRegs[1], Src0Regs[0], Src1Regs[1], Src2Regs[1]); + B.buildSelect(DefRegs[0], CondRegs[0], Src1Regs[0], Src2Regs[0]); + B.buildSelect(DefRegs[1], CondRegs[0], Src1Regs[1], Src2Regs[1]); - MRI.setRegBank(DstReg, getRegBank(AMDGPU::VGPRRegBankID)); + MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); MI.eraseFromParent(); return; } + case AMDGPU::G_BRCOND: { + Register CondReg = MI.getOperand(0).getReg(); + // FIXME: Should use legalizer helper, but should change bool ext type. + const RegisterBank *CondBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; + + if (CondBank == &AMDGPU::SGPRRegBank) { + MachineIRBuilder B(MI); + const LLT S32 = LLT::scalar(32); + Register NewCondReg = MRI.createGenericVirtualRegister(S32); + MRI.setRegBank(NewCondReg, AMDGPU::SGPRRegBank); + + MI.getOperand(0).setReg(NewCondReg); + B.buildZExt(NewCondReg, CondReg); + return; + } + + break; + } case AMDGPU::G_AND: case AMDGPU::G_OR: case AMDGPU::G_XOR: { @@ -1382,6 +1642,25 @@ void AMDGPURegisterBankInfo::applyMappingImpl( // there is a VGPR input. Register DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); + + if (DstTy.getSizeInBits() == 1) { + const RegisterBank *DstBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; + if (DstBank == &AMDGPU::VCCRegBank) + break; + + MachineFunction *MF = MI.getParent()->getParent(); + ApplyRegBankMapping ApplyBank(*this, MRI, DstBank); + GISelObserverWrapper Observer(&ApplyBank); + MachineIRBuilder B(MI); + LegalizerHelper Helper(*MF, Observer, B); + + if (Helper.widenScalar(MI, 0, LLT::scalar(32)) != + LegalizerHelper::Legalized) + llvm_unreachable("widen scalar should have succeeded"); + return; + } + if (DstTy.getSizeInBits() != 64) break; @@ -1427,7 +1706,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl( .addUse(Src0Regs[1]) .addUse(Src1Regs[1]); - MRI.setRegBank(DstReg, getRegBank(AMDGPU::VGPRRegBankID)); + MRI.setRegBank(DstReg, AMDGPU::VGPRRegBank); MI.eraseFromParent(); return; } @@ -1439,14 +1718,15 @@ void AMDGPURegisterBankInfo::applyMappingImpl( if (DstTy != LLT::scalar(16)) break; - const RegisterBank *DstBank = getRegBank(DstReg, MRI, *TRI); + const RegisterBank *DstBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; if (DstBank == &AMDGPU::VGPRRegBank) break; // 16-bit operations are VALU only, but can be promoted to 32-bit SALU. MachineFunction *MF = MI.getParent()->getParent(); MachineIRBuilder B(MI); - ApplyRegBankMapping ApplySALU(MRI, &AMDGPU::SGPRRegBank); + ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank); GISelObserverWrapper Observer(&ApplySALU); LegalizerHelper Helper(*MF, Observer, B); @@ -1460,15 +1740,13 @@ void AMDGPURegisterBankInfo::applyMappingImpl( case AMDGPU::G_UMIN: case AMDGPU::G_UMAX: { Register DstReg = MI.getOperand(0).getReg(); - const RegisterBank *DstBank = getRegBank(DstReg, MRI, *TRI); + const RegisterBank *DstBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; if (DstBank == &AMDGPU::VGPRRegBank) break; MachineFunction *MF = MI.getParent()->getParent(); MachineIRBuilder B(MI); - ApplyRegBankMapping ApplySALU(MRI, &AMDGPU::SGPRRegBank); - GISelObserverWrapper Observer(&ApplySALU); - LegalizerHelper Helper(*MF, Observer, B); // Turn scalar min/max into a compare and select. LLT Ty = MRI.getType(DstReg); @@ -1476,17 +1754,18 @@ void AMDGPURegisterBankInfo::applyMappingImpl( LLT S16 = LLT::scalar(16); if (Ty == S16) { + ApplyRegBankMapping ApplySALU(*this, MRI, &AMDGPU::SGPRRegBank); + GISelObserverWrapper Observer(&ApplySALU); + LegalizerHelper Helper(*MF, Observer, B); + // Need to widen to s32, and expand as cmp + select. if (Helper.widenScalar(MI, 0, S32) != LegalizerHelper::Legalized) llvm_unreachable("widenScalar should have succeeded"); // FIXME: This is relying on widenScalar leaving MI in place. - if (Helper.lower(MI, 0, S32) != LegalizerHelper::Legalized) - llvm_unreachable("lower should have succeeded"); - } else { - if (Helper.lower(MI, 0, Ty) != LegalizerHelper::Legalized) - llvm_unreachable("lower should have succeeded"); - } + lowerScalarMinMax(B, MI); + } else + lowerScalarMinMax(B, MI); return; } @@ -1497,13 +1776,13 @@ void AMDGPURegisterBankInfo::applyMappingImpl( bool Signed = Opc == AMDGPU::G_SEXT; MachineIRBuilder B(MI); - const RegisterBank *SrcBank = getRegBank(SrcReg, MRI, *TRI); + const RegisterBank *SrcBank = + OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; Register DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); if (DstTy.isScalar() && SrcBank != &AMDGPU::SGPRRegBank && - SrcBank != &AMDGPU::SCCRegBank && SrcBank != &AMDGPU::VCCRegBank && // FIXME: Should handle any type that round to s64 when irregular // breakdowns supported. @@ -1534,16 +1813,15 @@ void AMDGPURegisterBankInfo::applyMappingImpl( if (SrcTy != LLT::scalar(1)) return; - if (SrcBank == &AMDGPU::SCCRegBank || SrcBank == &AMDGPU::VCCRegBank) { + if (SrcBank == &AMDGPU::VCCRegBank) { SmallVector<Register, 2> DefRegs(OpdMapper.getVRegs(0)); - const RegisterBank *DstBank = SrcBank == &AMDGPU::SCCRegBank ? - &AMDGPU::SGPRRegBank : &AMDGPU::VGPRRegBank; + const RegisterBank *DstBank = &AMDGPU::VGPRRegBank; unsigned DstSize = DstTy.getSizeInBits(); // 64-bit select is SGPR only const bool UseSel64 = DstSize > 32 && - SrcBank->getID() == AMDGPU::SCCRegBankID; + SrcBank->getID() == AMDGPU::SGPRRegBankID; // TODO: Should s16 select be legal? LLT SelType = UseSel64 ? LLT::scalar(64) : LLT::scalar(32); @@ -1554,7 +1832,7 @@ void AMDGPURegisterBankInfo::applyMappingImpl( MRI.setRegBank(False.getReg(0), *DstBank); MRI.setRegBank(DstReg, *DstBank); - if (DstSize > 32 && SrcBank->getID() != AMDGPU::SCCRegBankID) { + if (DstSize > 32) { B.buildSelect(DefRegs[0], SrcReg, True, False); B.buildCopy(DefRegs[1], DefRegs[0]); } else if (DstSize < 32) { @@ -1599,7 +1877,8 @@ void AMDGPURegisterBankInfo::applyMappingImpl( substituteSimpleCopyRegs(OpdMapper, 1); substituteSimpleCopyRegs(OpdMapper, 2); - const RegisterBank *DstBank = getRegBank(DstReg, MRI, *TRI); + const RegisterBank *DstBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; if (DstBank == &AMDGPU::SGPRRegBank) break; // Can use S_PACK_* instructions. @@ -1609,8 +1888,10 @@ void AMDGPURegisterBankInfo::applyMappingImpl( Register Hi = MI.getOperand(2).getReg(); const LLT S32 = LLT::scalar(32); - const RegisterBank *BankLo = getRegBank(Lo, MRI, *TRI); - const RegisterBank *BankHi = getRegBank(Hi, MRI, *TRI); + const RegisterBank *BankLo = + OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; + const RegisterBank *BankHi = + OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; Register ZextLo; Register ShiftHi; @@ -1653,25 +1934,50 @@ void AMDGPURegisterBankInfo::applyMappingImpl( assert(OpdMapper.getVRegs(1).empty() && OpdMapper.getVRegs(2).empty()); + LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); + MachineIRBuilder B(MI); + + const ValueMapping &DstMapping + = OpdMapper.getInstrMapping().getOperandMapping(0); + const RegisterBank *DstBank = DstMapping.BreakDown[0].RegBank; + const RegisterBank *SrcBank = + OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; + + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + Register IdxReg = MI.getOperand(2).getReg(); + + // If this is a VGPR result only because the index was a VGPR result, the + // actual indexing will be done on the SGPR source vector, which will + // produce a scalar result. We need to copy to the VGPR result inside the + // waterfall loop. + const bool NeedCopyToVGPR = DstBank == &AMDGPU::VGPRRegBank && + SrcBank == &AMDGPU::SGPRRegBank; if (DstRegs.empty()) { applyDefaultMapping(OpdMapper); + executeInWaterfallLoop(MI, MRI, { 2 }); + + if (NeedCopyToVGPR) { + // We don't want a phi for this temporary reg. + Register TmpReg = MRI.createGenericVirtualRegister(DstTy); + MRI.setRegBank(TmpReg, AMDGPU::SGPRRegBank); + MI.getOperand(0).setReg(TmpReg); + B.setInsertPt(*MI.getParent(), ++MI.getIterator()); + + // Use a v_mov_b32 here to make the exec dependency explicit. + buildVCopy(B, DstReg, TmpReg); + } + return; } - Register DstReg = MI.getOperand(0).getReg(); - Register SrcReg = MI.getOperand(1).getReg(); - Register IdxReg = MI.getOperand(2).getReg(); - LLT DstTy = MRI.getType(DstReg); - (void)DstTy; - assert(DstTy.getSizeInBits() == 64); LLT SrcTy = MRI.getType(SrcReg); const LLT S32 = LLT::scalar(32); LLT Vec32 = LLT::vector(2 * SrcTy.getNumElements(), 32); - MachineIRBuilder B(MI); auto CastSrc = B.buildBitcast(Vec32, SrcReg); auto One = B.buildConstant(S32, 1); @@ -1684,15 +1990,11 @@ void AMDGPURegisterBankInfo::applyMappingImpl( // Compute 32-bit element indices, (2 * OrigIdx, 2 * OrigIdx + 1). auto IdxLo = B.buildShl(S32, IdxReg, One); auto IdxHi = B.buildAdd(S32, IdxLo, One); - B.buildExtractVectorElement(DstRegs[0], CastSrc, IdxLo); - B.buildExtractVectorElement(DstRegs[1], CastSrc, IdxHi); - const ValueMapping &DstMapping - = OpdMapper.getInstrMapping().getOperandMapping(0); + auto Extract0 = B.buildExtractVectorElement(DstRegs[0], CastSrc, IdxLo); + auto Extract1 = B.buildExtractVectorElement(DstRegs[1], CastSrc, IdxHi); - // FIXME: Should be getting from mapping or not? - const RegisterBank *SrcBank = getRegBank(SrcReg, MRI, *TRI); - MRI.setRegBank(DstReg, *DstMapping.BreakDown[0].RegBank); + MRI.setRegBank(DstReg, *DstBank); MRI.setRegBank(CastSrc.getReg(0), *SrcBank); MRI.setRegBank(One.getReg(0), AMDGPU::SGPRRegBank); MRI.setRegBank(IdxLo.getReg(0), AMDGPU::SGPRRegBank); @@ -1710,6 +2012,23 @@ void AMDGPURegisterBankInfo::applyMappingImpl( MI.eraseFromParent(); executeInWaterfallLoop(B, make_range(Span.begin(), Span.end()), OpsToWaterfall, MRI); + + if (NeedCopyToVGPR) { + MachineBasicBlock *LoopBB = Extract1->getParent(); + Register TmpReg0 = MRI.createGenericVirtualRegister(S32); + Register TmpReg1 = MRI.createGenericVirtualRegister(S32); + MRI.setRegBank(TmpReg0, AMDGPU::SGPRRegBank); + MRI.setRegBank(TmpReg1, AMDGPU::SGPRRegBank); + + Extract0->getOperand(0).setReg(TmpReg0); + Extract1->getOperand(0).setReg(TmpReg1); + + B.setInsertPt(*LoopBB, ++Extract1->getIterator()); + + buildVCopy(B, DstRegs[0], TmpReg0); + buildVCopy(B, DstRegs[1], TmpReg1); + } + return; } case AMDGPU::G_INSERT_VECTOR_ELT: { @@ -1756,9 +2075,12 @@ void AMDGPURegisterBankInfo::applyMappingImpl( auto InsHi = B.buildInsertVectorElement(Vec32, InsLo, InsRegs[1], IdxHi); B.buildBitcast(DstReg, InsHi); - const RegisterBank *DstBank = getRegBank(DstReg, MRI, *TRI); - const RegisterBank *SrcBank = getRegBank(SrcReg, MRI, *TRI); - const RegisterBank *InsSrcBank = getRegBank(InsReg, MRI, *TRI); + const RegisterBank *DstBank = + OpdMapper.getInstrMapping().getOperandMapping(0).BreakDown[0].RegBank; + const RegisterBank *SrcBank = + OpdMapper.getInstrMapping().getOperandMapping(1).BreakDown[0].RegBank; + const RegisterBank *InsSrcBank = + OpdMapper.getInstrMapping().getOperandMapping(2).BreakDown[0].RegBank; MRI.setRegBank(InsReg, *InsSrcBank); MRI.setRegBank(CastSrc.getReg(0), *SrcBank); @@ -1908,11 +2230,8 @@ bool AMDGPURegisterBankInfo::isSALUMapping(const MachineInstr &MI) const { continue; Register Reg = MI.getOperand(i).getReg(); if (const RegisterBank *Bank = getRegBank(Reg, MRI, *TRI)) { - if (Bank->getID() == AMDGPU::VGPRRegBankID) + if (Bank->getID() != AMDGPU::SGPRRegBankID) return false; - - assert(Bank->getID() == AMDGPU::SGPRRegBankID || - Bank->getID() == AMDGPU::SCCRegBankID); } } return true; @@ -1926,8 +2245,7 @@ AMDGPURegisterBankInfo::getDefaultMappingSOP(const MachineInstr &MI) const { for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) { unsigned Size = getSizeInBits(MI.getOperand(i).getReg(), MRI, *TRI); - unsigned BankID = Size == 1 ? AMDGPU::SCCRegBankID : AMDGPU::SGPRRegBankID; - OpdsMapping[i] = AMDGPU::getValueMapping(BankID, Size); + OpdsMapping[i] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); } return getInstructionMapping(1, 1, getOperandsMapping(OpdsMapping), MI.getNumOperands()); @@ -2048,7 +2366,7 @@ AMDGPURegisterBankInfo::getInstrMappingForLoad(const MachineInstr &MI) const { if (PtrBank == &AMDGPU::SGPRRegBank && (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::REGION_ADDRESS && AS != AMDGPUAS::PRIVATE_ADDRESS) && - isInstrUniformNonExtLoadAlign4(MI)) { + isScalarLoadLegal(MI)) { // We have a uniform instruction so we want to use an SMRD load ValMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); PtrMapping = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, PtrSize); @@ -2072,7 +2390,6 @@ AMDGPURegisterBankInfo::getRegBankID(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, unsigned Default) const { - const RegisterBank *Bank = getRegBank(Reg, MRI, TRI); return Bank ? Bank->getID() : Default; } @@ -2083,6 +2400,22 @@ static unsigned regBankUnion(unsigned RB0, unsigned RB1) { AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; } +static int regBankBoolUnion(int RB0, int RB1) { + if (RB0 == -1) + return RB1; + if (RB1 == -1) + return RB0; + + // vcc, vcc -> vcc + // vcc, sgpr -> vcc + // vcc, vgpr -> vcc + if (RB0 == AMDGPU::VCCRegBankID || RB1 == AMDGPU::VCCRegBankID) + return AMDGPU::VCCRegBankID; + + // vcc, vgpr -> vgpr + return regBankUnion(RB0, RB1); +} + const RegisterBankInfo::ValueMapping * AMDGPURegisterBankInfo::getSGPROpMapping(Register Reg, const MachineRegisterInfo &MRI, @@ -2102,6 +2435,14 @@ AMDGPURegisterBankInfo::getVGPROpMapping(Register Reg, return AMDGPU::getValueMapping(AMDGPU::VGPRRegBankID, Size); } +const RegisterBankInfo::ValueMapping * +AMDGPURegisterBankInfo::getAGPROpMapping(Register Reg, + const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const { + unsigned Size = getSizeInBits(Reg, MRI, TRI); + return AMDGPU::getValueMapping(AMDGPU::AGPRRegBankID, Size); +} + /// /// This function must return a legal mapping, because /// AMDGPURegisterBankInfo::getInstrAlternativeMappings() is not called @@ -2142,6 +2483,11 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { if (MI.getOpcode() == TargetOpcode::G_PHI) { // TODO: Generate proper invalid bank enum. int ResultBank = -1; + Register DstReg = MI.getOperand(0).getReg(); + + // Sometimes the result may have already been assigned a bank. + if (const RegisterBank *DstBank = getRegBank(DstReg, MRI, *TRI)) + ResultBank = DstBank->getID(); for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { Register Reg = MI.getOperand(I).getReg(); @@ -2153,26 +2499,14 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { break; } + // FIXME: Need to promote SGPR case to s32 unsigned OpBank = Bank->getID(); - // scc, scc -> sgpr - if (OpBank == AMDGPU::SCCRegBankID) { - // There's only one SCC register, so a phi requires copying to SGPR. - OpBank = AMDGPU::SGPRRegBankID; - } else if (OpBank == AMDGPU::VCCRegBankID) { - // vcc, vcc -> vcc - // vcc, sgpr -> vgpr - if (ResultBank != -1 && ResultBank != AMDGPU::VCCRegBankID) { - ResultBank = AMDGPU::VGPRRegBankID; - break; - } - } - - ResultBank = OpBank; + ResultBank = regBankBoolUnion(ResultBank, OpBank); } assert(ResultBank != -1); - unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); + unsigned Size = MRI.getType(DstReg).getSizeInBits(); const ValueMapping &ValMap = getValueMapping(0, Size, getRegBank(ResultBank)); @@ -2208,10 +2542,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { TargetBankID = AMDGPU::VCCRegBankID; BankLHS = AMDGPU::VCCRegBankID; BankRHS = AMDGPU::VCCRegBankID; - } else if (DstBank == &AMDGPU::SCCRegBank) { - TargetBankID = AMDGPU::SCCRegBankID; - BankLHS = AMDGPU::SGPRRegBankID; - BankRHS = AMDGPU::SGPRRegBankID; } else { BankLHS = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI, AMDGPU::SGPRRegBankID); @@ -2233,13 +2563,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { BankRHS = AMDGPU::VCCRegBankID; } else if (BankLHS == AMDGPU::SGPRRegBankID && BankRHS == AMDGPU::SGPRRegBankID) { TargetBankID = AMDGPU::SGPRRegBankID; - } else if (BankLHS == AMDGPU::SCCRegBankID || BankRHS == AMDGPU::SCCRegBankID) { - // The operation must be done on a 32-bit register, but it will set - // scc. The result type could interchangably be SCC or SGPR, since - // both values will be produced. - TargetBankID = AMDGPU::SCCRegBankID; - BankLHS = AMDGPU::SGPRRegBankID; - BankRHS = AMDGPU::SGPRRegBankID; } } @@ -2268,7 +2591,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { LLVM_FALLTHROUGH; } - case AMDGPU::G_GEP: + case AMDGPU::G_PTR_ADD: case AMDGPU::G_ADD: case AMDGPU::G_SUB: case AMDGPU::G_MUL: @@ -2312,7 +2635,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_FMAXNUM_IEEE: case AMDGPU::G_FCANONICALIZE: case AMDGPU::G_INTRINSIC_TRUNC: - case AMDGPU::G_INTRINSIC_ROUND: case AMDGPU::G_AMDGPU_FFBH_U32: return getDefaultMappingVOP(MI); case AMDGPU::G_UMULH: @@ -2329,7 +2651,8 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_FCONSTANT: case AMDGPU::G_CONSTANT: case AMDGPU::G_GLOBAL_VALUE: - case AMDGPU::G_BLOCK_ADDR: { + case AMDGPU::G_BLOCK_ADDR: + case AMDGPU::G_READCYCLECOUNTER: { unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); OpdsMapping[0] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); break; @@ -2416,7 +2739,9 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { unsigned Bank = getRegBankID(Src, MRI, *TRI); unsigned DstSize = getSizeInBits(Dst, MRI, *TRI); unsigned SrcSize = getSizeInBits(Src, MRI, *TRI); - OpdsMapping[0] = AMDGPU::getValueMapping(Bank, DstSize); + OpdsMapping[0] = DstSize == 1 && Bank != AMDGPU::SGPRRegBankID ? + AMDGPU::getValueMapping(AMDGPU::VCCRegBankID, DstSize) : + AMDGPU::getValueMapping(Bank, DstSize); OpdsMapping[1] = AMDGPU::getValueMapping(Bank, SrcSize); break; } @@ -2432,7 +2757,6 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { const RegisterBank *SrcBank = getRegBank(Src, MRI, *TRI); assert(SrcBank); switch (SrcBank->getID()) { - case AMDGPU::SCCRegBankID: case AMDGPU::SGPRRegBankID: DstBank = AMDGPU::SGPRRegBankID; break; @@ -2493,9 +2817,13 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) && Subtarget.hasScalarCompareEq64())); - unsigned Op0Bank = CanUseSCC ? AMDGPU::SCCRegBankID : AMDGPU::VCCRegBankID; + unsigned Op0Bank = CanUseSCC ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; + + // TODO: Use 32-bit for scalar output size. + // SCC results will need to be copied to a 32-bit SGPR virtual register. + const unsigned ResultSize = 1; - OpdsMapping[0] = AMDGPU::getValueMapping(Op0Bank, 1); + OpdsMapping[0] = AMDGPU::getValueMapping(Op0Bank, ResultSize); OpdsMapping[1] = nullptr; // Predicate Operand. OpdsMapping[2] = AMDGPU::getValueMapping(Op2Bank, Size); OpdsMapping[3] = AMDGPU::getValueMapping(Op3Bank, Size); @@ -2555,6 +2883,7 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { default: return getInvalidInstructionMapping(); case Intrinsic::amdgcn_div_fmas: + case Intrinsic::amdgcn_div_fixup: case Intrinsic::amdgcn_trig_preop: case Intrinsic::amdgcn_sin: case Intrinsic::amdgcn_cos: @@ -2725,6 +3054,38 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { OpdsMapping[3] = AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, Size); break; } + case Intrinsic::amdgcn_mfma_f32_4x4x1f32: + case Intrinsic::amdgcn_mfma_f32_4x4x4f16: + case Intrinsic::amdgcn_mfma_i32_4x4x4i8: + case Intrinsic::amdgcn_mfma_f32_4x4x2bf16: + case Intrinsic::amdgcn_mfma_f32_16x16x1f32: + case Intrinsic::amdgcn_mfma_f32_16x16x4f32: + case Intrinsic::amdgcn_mfma_f32_16x16x4f16: + case Intrinsic::amdgcn_mfma_f32_16x16x16f16: + case Intrinsic::amdgcn_mfma_i32_16x16x4i8: + case Intrinsic::amdgcn_mfma_i32_16x16x16i8: + case Intrinsic::amdgcn_mfma_f32_16x16x2bf16: + case Intrinsic::amdgcn_mfma_f32_16x16x8bf16: + case Intrinsic::amdgcn_mfma_f32_32x32x1f32: + case Intrinsic::amdgcn_mfma_f32_32x32x2f32: + case Intrinsic::amdgcn_mfma_f32_32x32x4f16: + case Intrinsic::amdgcn_mfma_f32_32x32x8f16: + case Intrinsic::amdgcn_mfma_i32_32x32x4i8: + case Intrinsic::amdgcn_mfma_i32_32x32x8i8: + case Intrinsic::amdgcn_mfma_f32_32x32x2bf16: + case Intrinsic::amdgcn_mfma_f32_32x32x4bf16: { + // Default for MAI intrinsics. + // srcC can also be an immediate which can be folded later. + // FIXME: Should we eventually add an alternative mapping with AGPR src + // for srcA/srcB? + // + // vdst, srcA, srcB, srcC + OpdsMapping[0] = getAGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI); + OpdsMapping[2] = getVGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI); + OpdsMapping[3] = getVGPROpMapping(MI.getOperand(3).getReg(), MRI, *TRI); + OpdsMapping[4] = getAGPROpMapping(MI.getOperand(4).getReg(), MRI, *TRI); + break; + } } break; } @@ -2913,19 +3274,20 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { Op3Bank == AMDGPU::SGPRRegBankID; unsigned CondBankDefault = SGPRSrcs ? - AMDGPU::SCCRegBankID : AMDGPU::VCCRegBankID; + AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; unsigned CondBank = getRegBankID(MI.getOperand(1).getReg(), MRI, *TRI, CondBankDefault); if (CondBank == AMDGPU::SGPRRegBankID) - CondBank = SGPRSrcs ? AMDGPU::SCCRegBankID : AMDGPU::VCCRegBankID; + CondBank = SGPRSrcs ? AMDGPU::SGPRRegBankID : AMDGPU::VCCRegBankID; else if (CondBank == AMDGPU::VGPRRegBankID) CondBank = AMDGPU::VCCRegBankID; - unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SCCRegBankID ? + unsigned Bank = SGPRSrcs && CondBank == AMDGPU::SGPRRegBankID ? AMDGPU::SGPRRegBankID : AMDGPU::VGPRRegBankID; - assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SCCRegBankID); + assert(CondBank == AMDGPU::VCCRegBankID || CondBank == AMDGPU::SGPRRegBankID); + // TODO: Should report 32-bit for scalar condition type. if (Size == 64) { OpdsMapping[0] = AMDGPU::getValueMappingSGPR64Only(Bank, Size); OpdsMapping[1] = AMDGPU::getValueMapping(CondBank, 1); @@ -2957,14 +3319,15 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { case AMDGPU::G_ATOMICRMW_UMAX: case AMDGPU::G_ATOMICRMW_UMIN: case AMDGPU::G_ATOMICRMW_FADD: - case AMDGPU::G_ATOMIC_CMPXCHG: { + case AMDGPU::G_ATOMIC_CMPXCHG: + case AMDGPU::G_AMDGPU_ATOMIC_CMPXCHG: { return getDefaultMappingAllVGPR(MI); } case AMDGPU::G_BRCOND: { unsigned Bank = getRegBankID(MI.getOperand(0).getReg(), MRI, *TRI, AMDGPU::SGPRRegBankID); assert(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits() == 1); - if (Bank != AMDGPU::SCCRegBankID) + if (Bank != AMDGPU::SGPRRegBankID) Bank = AMDGPU::VCCRegBankID; OpdsMapping[0] = AMDGPU::getValueMapping(Bank, 1); diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h index a14b74961118..1ac7d3652a8b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.h @@ -40,10 +40,13 @@ protected: #include "AMDGPUGenRegisterBank.inc" }; class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo { +public: const GCNSubtarget &Subtarget; const SIRegisterInfo *TRI; const SIInstrInfo *TII; + bool buildVCopy(MachineIRBuilder &B, Register DstReg, Register SrcReg) const; + bool collectWaterfallOperands( SmallSet<Register, 4> &SGPROperandRegs, MachineInstr &MI, @@ -74,6 +77,8 @@ class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo { const AMDGPURegisterBankInfo::OperandsMapper &OpdMapper, MachineRegisterInfo &MRI, int RSrcIdx) const; + void lowerScalarMinMax(MachineIRBuilder &B, MachineInstr &MI) const; + Register handleD16VData(MachineIRBuilder &B, MachineRegisterInfo &MRI, Register Reg) const; @@ -103,6 +108,11 @@ class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo { const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const; + // Return a value mapping for an operand that is required to be a AGPR. + const ValueMapping *getAGPROpMapping(Register Reg, + const MachineRegisterInfo &MRI, + const TargetRegisterInfo &TRI) const; + /// Split 64-bit value \p Reg into two 32-bit halves and populate them into \p /// Regs. This appropriately sets the regbank of the new registers. void split64BitValueForMapping(MachineIRBuilder &B, @@ -131,6 +141,7 @@ class AMDGPURegisterBankInfo : public AMDGPUGenRegisterBankInfo { const MachineInstr &MI, const MachineRegisterInfo &MRI) const; bool isSALUMapping(const MachineInstr &MI) const; + const InstructionMapping &getDefaultMappingSOP(const MachineInstr &MI) const; const InstructionMapping &getDefaultMappingVOP(const MachineInstr &MI) const; const InstructionMapping &getDefaultMappingAllVGPR( @@ -149,8 +160,8 @@ public: unsigned getBreakDownCost(const ValueMapping &ValMapping, const RegisterBank *CurBank = nullptr) const override; - const RegisterBank & - getRegBankFromRegClass(const TargetRegisterClass &RC) const override; + const RegisterBank &getRegBankFromRegClass(const TargetRegisterClass &RC, + LLT) const override; InstructionMappings getInstrAlternativeMappings(const MachineInstr &MI) const override; diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td b/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td index 00f53b157577..c495316c5bce 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td +++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBanks.td @@ -14,7 +14,9 @@ def VGPRRegBank : RegisterBank<"VGPR", [VGPR_32, VReg_64, VReg_96, VReg_128, VReg_256, VReg_512, VReg_1024] >; -def SCCRegBank : RegisterBank <"SCC", [SReg_32, SCC_CLASS]>; - // It is helpful to distinguish conditions from ordinary SGPRs. def VCCRegBank : RegisterBank <"VCC", [SReg_1]>; + +def AGPRRegBank : RegisterBank <"AGPR", + [AGPR_32, AReg_64, AReg_128, AReg_512, AReg_1024] +>; diff --git a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp index 4f095087a57f..9a1e2fc42ed5 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp @@ -43,12 +43,12 @@ #include "AMDGPU.h" #include "Utils/AMDGPUBaseInfo.h" -#include "llvm/Analysis/MemoryDependenceAnalysis.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/MemoryDependenceAnalysis.h" #include "llvm/Analysis/MemoryLocation.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" @@ -64,6 +64,7 @@ #include "llvm/IR/Use.h" #include "llvm/IR/User.h" #include "llvm/IR/Value.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td index 26b8b7840270..8d70536ec21c 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUSearchableTables.td @@ -30,6 +30,147 @@ foreach intr = !listconcat(AMDGPUBufferIntrinsics, def : RsrcIntrinsic<!cast<AMDGPURsrcIntrinsic>(intr)>; } +class GcnBufferFormatBase<bits<8> f, bits<8> bpc, bits<8> numc, bits<8> nfmt, bits<8> dfmt> { + bits<8> Format = f; + bits<8> BitsPerComp = bpc; + bits<8> NumComponents = numc; + bits<8> NumFormat = nfmt; + bits<8> DataFormat = dfmt; +} + +class Gfx9BufferFormat<bits<8> f, bits<8> bpc, bits<8> numc, bits<8> nfmt, bits<8> dfmt> : GcnBufferFormatBase<f, bpc, numc, nfmt, dfmt>; +class Gfx10PlusBufferFormat<bits<8> f, bits<8> bpc, bits<8> numc, bits<8> nfmt, bits<8> dfmt> : GcnBufferFormatBase<f, bpc, numc, nfmt, dfmt>; + +class GcnBufferFormatTable : GenericTable { + let CppTypeName = "GcnBufferFormatInfo"; + let Fields = ["Format", "BitsPerComp", "NumComponents", "NumFormat", "DataFormat"]; + let PrimaryKey = ["BitsPerComp", "NumComponents", "NumFormat"]; +} + +def Gfx9BufferFormat : GcnBufferFormatTable { + let FilterClass = "Gfx9BufferFormat"; + let PrimaryKeyName = "getGfx9BufferFormatInfo"; +} +def Gfx10PlusBufferFormat : GcnBufferFormatTable { + let FilterClass = "Gfx10PlusBufferFormat"; + let PrimaryKeyName = "getGfx10PlusBufferFormatInfo"; +} + +def getGfx9BufferFormatInfo : SearchIndex { + let Table = Gfx9BufferFormat; + let Key = ["Format"]; +} +def getGfx10PlusBufferFormatInfo : SearchIndex { + let Table = Gfx10PlusBufferFormat; + let Key = ["Format"]; +} + +// Buffer formats with equal component sizes (GFX9 and earlier) +def : Gfx9BufferFormat< /*FORMAT_8_UNORM*/ 0x01, 8, 1, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_8*/ 1>; +def : Gfx9BufferFormat< /*FORMAT_8_SNORM*/ 0x11, 8, 1, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_8*/ 1>; +def : Gfx9BufferFormat< /*FORMAT_8_USCALED*/ 0x21, 8, 1, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_8*/ 1>; +def : Gfx9BufferFormat< /*FORMAT_8_SSCALED*/ 0x31, 8, 1, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_8*/ 1>; +def : Gfx9BufferFormat< /*FORMAT_8_UINT*/ 0x41, 8, 1, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_8*/ 1>; +def : Gfx9BufferFormat< /*FORMAT_8_SINT*/ 0x51, 8, 1, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_8*/ 1>; +def : Gfx9BufferFormat< /*FORMAT_16_UNORM*/ 0x02, 16, 1, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_16*/ 2>; +def : Gfx9BufferFormat< /*FORMAT_16_SNORM*/ 0x12, 16, 1, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_16*/ 2>; +def : Gfx9BufferFormat< /*FORMAT_16_USCALED*/ 0x22, 16, 1, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_16*/ 2>; +def : Gfx9BufferFormat< /*FORMAT_16_SSCALED*/ 0x32, 16, 1, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_16*/ 2>; +def : Gfx9BufferFormat< /*FORMAT_16_UINT*/ 0x42, 16, 1, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_16*/ 2>; +def : Gfx9BufferFormat< /*FORMAT_16_SINT*/ 0x52, 16, 1, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_16*/ 2>; +def : Gfx9BufferFormat< /*FORMAT_16_FLOAT*/ 0x72, 16, 1, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_16*/ 2>; +def : Gfx9BufferFormat< /*FORMAT_8_8_UNORM*/ 0x03, 8, 2, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx9BufferFormat< /*FORMAT_8_8_SNORM*/ 0x13, 8, 2, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx9BufferFormat< /*FORMAT_8_8_USCALED*/ 0x23, 8, 2, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx9BufferFormat< /*FORMAT_8_8_SSCALED*/ 0x33, 8, 2, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx9BufferFormat< /*FORMAT_8_8_UINT*/ 0x43, 8, 2, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx9BufferFormat< /*FORMAT_8_8_SINT*/ 0x53, 8, 2, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx9BufferFormat< /*FORMAT_32_UINT*/ 0x44, 32, 1, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_32*/ 4>; +def : Gfx9BufferFormat< /*FORMAT_32_SINT*/ 0x54, 32, 1, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_32*/ 4>; +def : Gfx9BufferFormat< /*FORMAT_32_FLOAT*/ 0x74, 32, 1, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_32*/ 4>; +def : Gfx9BufferFormat< /*FORMAT_16_16_UNORM*/ 0x05, 16, 2, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx9BufferFormat< /*FORMAT_16_16_SNORM*/ 0x15, 16, 2, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx9BufferFormat< /*FORMAT_16_16_USCALED*/ 0x25, 16, 2, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx9BufferFormat< /*FORMAT_16_16_SSCALED*/ 0x35, 16, 2, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx9BufferFormat< /*FORMAT_16_16_UINT*/ 0x45, 16, 2, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx9BufferFormat< /*FORMAT_16_16_SINT*/ 0x55, 16, 2, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx9BufferFormat< /*FORMAT_16_16_FLOAT*/ 0x75, 16, 2, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx9BufferFormat< /*FORMAT_8_8_8_8_UNORM*/ 0x0A, 8, 4, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx9BufferFormat< /*FORMAT_8_8_8_8_SNORM*/ 0x1A, 8, 4, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx9BufferFormat< /*FORMAT_8_8_8_8_USCALED*/ 0x2A, 8, 4, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx9BufferFormat< /*FORMAT_8_8_8_8_SSCALED*/ 0x3A, 8, 4, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx9BufferFormat< /*FORMAT_8_8_8_8_UINT*/ 0x4A, 8, 4, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx9BufferFormat< /*FORMAT_8_8_8_8_SINT*/ 0x5A, 8, 4, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx9BufferFormat< /*FORMAT_32_32_UINT*/ 0x4B, 32, 2, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_32_32*/ 11>; +def : Gfx9BufferFormat< /*FORMAT_32_32_SINT*/ 0x5B, 32, 2, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_32_32*/ 11>; +def : Gfx9BufferFormat< /*FORMAT_32_32_FLOAT*/ 0x7B, 32, 2, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_32_32*/ 11>; +def : Gfx9BufferFormat< /*FORMAT_16_16_16_16_UNORM*/ 0x0C, 16, 4, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx9BufferFormat< /*FORMAT_16_16_16_16_SNORM*/ 0x1C, 16, 4, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx9BufferFormat< /*FORMAT_16_16_16_16_USCALED*/ 0x2C, 16, 4, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx9BufferFormat< /*FORMAT_16_16_16_16_SSCALED*/ 0x3C, 16, 4, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx9BufferFormat< /*FORMAT_16_16_16_16_UINT*/ 0x4C, 16, 4, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx9BufferFormat< /*FORMAT_16_16_16_16_SINT*/ 0x5C, 16, 4, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx9BufferFormat< /*FORMAT_16_16_16_16_FLOAT*/ 0x7C, 16, 4, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx9BufferFormat< /*FORMAT_32_32_32_UINT*/ 0x4D, 32, 3, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_32_32_32*/ 13>; +def : Gfx9BufferFormat< /*FORMAT_32_32_32_SINT*/ 0x5D, 32, 3, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_32_32_32*/ 13>; +def : Gfx9BufferFormat< /*FORMAT_32_32_32_FLOAT*/ 0x7D, 32, 3, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_32_32_32*/ 13>; +def : Gfx9BufferFormat< /*FORMAT_32_32_32_32_UINT*/ 0x4E, 32, 4, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_32_32_32_32*/ 14>; +def : Gfx9BufferFormat< /*FORMAT_32_32_32_32_SINT*/ 0x5E, 32, 4, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_32_32_32_32*/ 14>; +def : Gfx9BufferFormat< /*FORMAT_32_32_32_32_FLOAT*/ 0x7E, 32, 4, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_32_32_32_32*/ 14>; + +// Buffer formats with equal component sizes (GFX10 and later) +def : Gfx10PlusBufferFormat< /*FORMAT_8_UNORM*/ 0x01, 8, 1, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_8*/ 1>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_SNORM*/ 0x02, 8, 1, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_8*/ 1>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_USCALED*/ 0x03, 8, 1, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_8*/ 1>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_SSCALED*/ 0x04, 8, 1, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_8*/ 1>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_UINT*/ 0x05, 8, 1, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_8*/ 1>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_SINT*/ 0x06, 8, 1, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_8*/ 1>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_UNORM*/ 0x07, 16, 1, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_16*/ 2>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_SNORM*/ 0x08, 16, 1, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_16*/ 2>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_USCALED*/ 0x09, 16, 1, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_16*/ 2>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_SSCALED*/ 0x0A, 16, 1, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_16*/ 2>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_UINT*/ 0x0B, 16, 1, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_16*/ 2>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_SINT*/ 0x0C, 16, 1, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_16*/ 2>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_FLOAT*/ 0x0D, 16, 1, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_16*/ 2>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_UNORM*/ 0x0E, 8, 2, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_SNORM*/ 0x0F, 8, 2, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_USCALED*/ 0x10, 8, 2, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_SSCALED*/ 0x11, 8, 2, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_UINT*/ 0x12, 8, 2, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_SINT*/ 0x13, 8, 2, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_8_8*/ 3>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_UINT*/ 0x14, 32, 1, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_32*/ 4>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_SINT*/ 0x15, 32, 1, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_32*/ 4>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_FLOAT*/ 0x16, 32, 1, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_32*/ 4>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_UNORM*/ 0x17, 16, 2, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_SNORM*/ 0x18, 16, 2, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_USCALED*/ 0x19, 16, 2, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_SSCALED*/ 0x1A, 16, 2, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_UINT*/ 0x1B, 16, 2, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_SINT*/ 0x1C, 16, 2, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_FLOAT*/ 0x1D, 16, 2, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_16_16*/ 5>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_8_8_UNORM*/ 0x38, 8, 4, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_8_8_SNORM*/ 0x39, 8, 4, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_8_8_USCALED*/ 0x3A, 8, 4, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_8_8_SSCALED*/ 0x3B, 8, 4, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_8_8_UINT*/ 0x3C, 8, 4, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx10PlusBufferFormat< /*FORMAT_8_8_8_8_SINT*/ 0x3D, 8, 4, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_8_8_8_8*/ 10>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_32_UINT*/ 0x3E, 32, 2, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_32_32*/ 11>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_32_SINT*/ 0x3F, 32, 2, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_32_32*/ 11>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_32_FLOAT*/ 0x40, 32, 2, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_32_32*/ 11>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_16_16_UNORM*/ 0x41, 16, 4, /*NUM_FORMAT_UNORM*/ 0, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_16_16_SNORM*/ 0x42, 16, 4, /*NUM_FORMAT_SNORM*/ 1, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_16_16_USCALED*/ 0x43, 16, 4, /*NUM_FORMAT_USCALED*/ 2, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_16_16_SSCALED*/ 0x44, 16, 4, /*NUM_FORMAT_SSCALED*/ 3, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_16_16_UINT*/ 0x45, 16, 4, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_16_16_SINT*/ 0x46, 16, 4, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx10PlusBufferFormat< /*FORMAT_16_16_16_16_FLOAT*/ 0x47, 16, 4, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_16_16_16_16*/ 12>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_32_32_UINT*/ 0x48, 32, 3, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_32_32_32*/ 13>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_32_32_SINT*/ 0x49, 32, 3, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_32_32_32*/ 13>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_32_32_FLOAT*/ 0x4A, 32, 3, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_32_32_32*/ 13>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_32_32_32_UINT*/ 0x4B, 32, 4, /*NUM_FORMAT_UINT*/ 4, /*DATA_FORMAT_32_32_32_32*/ 14>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_32_32_32_SINT*/ 0x4C, 32, 4, /*NUM_FORMAT_SINT*/ 5, /*DATA_FORMAT_32_32_32_32*/ 14>; +def : Gfx10PlusBufferFormat< /*FORMAT_32_32_32_32_FLOAT*/ 0x4D, 32, 4, /*NUM_FORMAT_FLOAT*/ 7, /*DATA_FORMAT_32_32_32_32*/ 14>; + class SourceOfDivergence<Intrinsic intr> { Intrinsic Intr = intr; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp index 3bb6dd4571c0..445e91092499 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp @@ -45,6 +45,11 @@ static cl::opt<bool> DisablePowerSched( cl::desc("Disable scheduling to minimize mAI power bursts"), cl::init(false)); +static cl::opt<bool> EnableVGPRIndexMode( + "amdgpu-vgpr-index-mode", + cl::desc("Use GPR indexing mode instead of movrel for vector indexing"), + cl::init(false)); + GCNSubtarget::~GCNSubtarget() = default; R600Subtarget & @@ -343,11 +348,6 @@ AMDGPUSubtarget::getOccupancyWithLocalMemSize(const MachineFunction &MF) const { std::pair<unsigned, unsigned> AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const { switch (CC) { - case CallingConv::AMDGPU_CS: - case CallingConv::AMDGPU_KERNEL: - case CallingConv::SPIR_KERNEL: - return std::make_pair(getWavefrontSize() * 2, - std::max(getWavefrontSize() * 4, 256u)); case CallingConv::AMDGPU_VS: case CallingConv::AMDGPU_LS: case CallingConv::AMDGPU_HS: @@ -356,13 +356,12 @@ AMDGPUSubtarget::getDefaultFlatWorkGroupSize(CallingConv::ID CC) const { case CallingConv::AMDGPU_PS: return std::make_pair(1, getWavefrontSize()); default: - return std::make_pair(1, 16 * getWavefrontSize()); + return std::make_pair(1u, getMaxFlatWorkGroupSize()); } } std::pair<unsigned, unsigned> AMDGPUSubtarget::getFlatWorkGroupSizes( const Function &F) const { - // FIXME: 1024 if function. // Default minimum/maximum flat work group sizes. std::pair<unsigned, unsigned> Default = getDefaultFlatWorkGroupSize(F.getCallingConv()); @@ -567,6 +566,10 @@ bool GCNSubtarget::hasMadF16() const { return InstrInfo.pseudoToMCOpcode(AMDGPU::V_MAD_F16) != -1; } +bool GCNSubtarget::useVGPRIndexMode() const { + return !hasMovrel() || (EnableVGPRIndexMode && hasVGPRIndexMode()); +} + unsigned GCNSubtarget::getOccupancyWithNumSGPRs(unsigned SGPRs) const { if (getGeneration() >= AMDGPUSubtarget::GFX10) return getMaxWavesPerEU(); @@ -713,6 +716,43 @@ unsigned GCNSubtarget::getMaxNumVGPRs(const MachineFunction &MF) const { return MaxNumVGPRs; } +void GCNSubtarget::adjustSchedDependency(SUnit *Src, SUnit *Dst, + SDep &Dep) const { + if (Dep.getKind() != SDep::Kind::Data || !Dep.getReg() || + !Src->isInstr() || !Dst->isInstr()) + return; + + MachineInstr *SrcI = Src->getInstr(); + MachineInstr *DstI = Dst->getInstr(); + + if (SrcI->isBundle()) { + const SIRegisterInfo *TRI = getRegisterInfo(); + auto Reg = Dep.getReg(); + MachineBasicBlock::const_instr_iterator I(SrcI->getIterator()); + MachineBasicBlock::const_instr_iterator E(SrcI->getParent()->instr_end()); + unsigned Lat = 0; + for (++I; I != E && I->isBundledWithPred(); ++I) { + if (I->modifiesRegister(Reg, TRI)) + Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *I); + else if (Lat) + --Lat; + } + Dep.setLatency(Lat); + } else if (DstI->isBundle()) { + const SIRegisterInfo *TRI = getRegisterInfo(); + auto Reg = Dep.getReg(); + MachineBasicBlock::const_instr_iterator I(DstI->getIterator()); + MachineBasicBlock::const_instr_iterator E(DstI->getParent()->instr_end()); + unsigned Lat = InstrInfo.getInstrLatency(getInstrItineraryData(), *SrcI); + for (++I; I != E && I->isBundledWithPred() && Lat; ++I) { + if (I->readsRegister(Reg, TRI)) + break; + --Lat; + } + Dep.setLatency(Lat); + } +} + namespace { struct MemOpClusterMutation : ScheduleDAGMutation { const SIInstrInfo *TII; @@ -773,6 +813,11 @@ struct FillMFMAShadowMutation : ScheduleDAGMutation { return MI && TII->isSALU(*MI) && !MI->isTerminator(); } + bool isVALU(const SUnit *SU) const { + const MachineInstr *MI = SU->getInstr(); + return MI && TII->isVALU(*MI); + } + bool canAddEdge(const SUnit *Succ, const SUnit *Pred) const { if (Pred->NodeNum < Succ->NodeNum) return true; @@ -821,7 +866,7 @@ struct FillMFMAShadowMutation : ScheduleDAGMutation { for (SDep &SI : From->Succs) { SUnit *SUv = SI.getSUnit(); - if (SUv != From && TII->isVALU(*SUv->getInstr()) && canAddEdge(SUv, SU)) + if (SUv != From && isVALU(SUv) && canAddEdge(SUv, SU)) SUv->addPred(SDep(SU, SDep::Artificial), false); } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h index 936feb00c62b..19a240800ba1 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.h @@ -148,7 +148,12 @@ public: return HasMadMixInsts; } - bool hasFP32Denormals() const { + bool hasFP32Denormals(const Function &F) const { + // FIXME: This should not be a property of the subtarget. This should be a + // property with a default set by the calling convention which can be + // overridden by attributes. For now, use the subtarget feature as a + // placeholder attribute. The function arguments only purpose is to + // discourage use without a function context until this is removed. return FP32Denormals; } @@ -612,11 +617,17 @@ public: unsigned getMaxLocalMemSizeWithWaveCount(unsigned WaveCount, const Function &) const; - bool hasFP16Denormals() const { + /// Alias for hasFP64FP16Denormals + bool hasFP16Denormals(const Function &F) const { return FP64FP16Denormals; } - bool hasFP64Denormals() const { + /// Alias for hasFP64FP16Denormals + bool hasFP64Denormals(const Function &F) const { + return FP64FP16Denormals; + } + + bool hasFP64FP16Denormals(const Function &F) const { return FP64FP16Denormals; } @@ -930,9 +941,7 @@ public: return HasVGPRIndexMode; } - bool useVGPRIndexMode(bool UserEnable) const { - return !hasMovrel() || (UserEnable && hasVGPRIndexMode()); - } + bool useVGPRIndexMode() const; bool hasScalarCompareEq64() const { return getGeneration() >= VOLCANIC_ISLANDS; @@ -1203,6 +1212,8 @@ public: unsigned getMinWavesPerEU() const override { return AMDGPU::IsaInfo::getMinWavesPerEU(this); } + + void adjustSchedDependency(SUnit *Src, SUnit *Dst, SDep &Dep) const override; }; class R600Subtarget final : public R600GenSubtargetInfo, diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp index e8cf77161a14..eb30d659bf0b 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp @@ -37,6 +37,7 @@ #include "llvm/IR/Attributes.h" #include "llvm/IR/Function.h" #include "llvm/IR/LegacyPassManager.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" @@ -182,7 +183,7 @@ static cl::opt<bool> EnableScalarIRPasses( cl::init(true), cl::Hidden); -extern "C" void LLVMInitializeAMDGPUTarget() { +extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() { // Register the target RegisterTargetMachine<R600TargetMachine> X(getTheAMDGPUTarget()); RegisterTargetMachine<GCNTargetMachine> Y(getTheGCNTarget()); @@ -227,6 +228,7 @@ extern "C" void LLVMInitializeAMDGPUTarget() { initializeSIModeRegisterPass(*PR); initializeSIWholeQuadModePass(*PR); initializeSILowerControlFlowPass(*PR); + initializeSIRemoveShortExecBranchesPass(*PR); initializeSIInsertSkipsPass(*PR); initializeSIMemoryLegalizerPass(*PR); initializeSIOptimizeExecMaskingPass(*PR); @@ -947,7 +949,7 @@ void GCNPassConfig::addOptimizedRegAlloc() { insertPass(&RegisterCoalescerID, &SIPreAllocateWWMRegsID, false); if (EnableDCEInRA) - insertPass(&RenameIndependentSubregsID, &DeadMachineInstructionElimID); + insertPass(&DetectDeadLanesID, &DeadMachineInstructionElimID); TargetPassConfig::addOptimizedRegAlloc(); } @@ -992,6 +994,7 @@ void GCNPassConfig::addPreEmitPass() { // be better for it to emit S_NOP <N> when possible. addPass(&PostRAHazardRecognizerID); + addPass(&SIRemoveShortExecBranchesID); addPass(&SIInsertSkipsPassID); addPass(&BranchRelaxationPassID); } @@ -1151,6 +1154,8 @@ bool GCNTargetMachine::parseMachineFunctionInfo( MFI->Mode.IEEE = YamlMFI.Mode.IEEE; MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp; + MFI->Mode.FP32Denormals = YamlMFI.Mode.FP32Denormals; + MFI->Mode.FP64FP16Denormals = YamlMFI.Mode.FP64FP16Denormals; return false; } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp index 616196ad5ba3..c4eeb81c5133 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp @@ -57,7 +57,7 @@ using namespace llvm; static cl::opt<unsigned> UnrollThresholdPrivate( "amdgpu-unroll-threshold-private", cl::desc("Unroll threshold for AMDGPU if private memory used in a loop"), - cl::init(2000), cl::Hidden); + cl::init(2700), cl::Hidden); static cl::opt<unsigned> UnrollThresholdLocal( "amdgpu-unroll-threshold-local", @@ -90,7 +90,8 @@ static bool dependsOnLocalPhi(const Loop *L, const Value *Cond, void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP) { - UP.Threshold = 300; // Twice the default. + const Function &F = *L->getHeader()->getParent(); + UP.Threshold = AMDGPU::getIntegerAttribute(F, "amdgpu-unroll-threshold", 300); UP.MaxCount = std::numeric_limits<unsigned>::max(); UP.Partial = true; @@ -337,10 +338,13 @@ bool GCNTTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, } } -int GCNTTIImpl::getArithmeticInstrCost( - unsigned Opcode, Type *Ty, TTI::OperandValueKind Opd1Info, - TTI::OperandValueKind Opd2Info, TTI::OperandValueProperties Opd1PropInfo, - TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args ) { +int GCNTTIImpl::getArithmeticInstrCost(unsigned Opcode, Type *Ty, + TTI::OperandValueKind Opd1Info, + TTI::OperandValueKind Opd2Info, + TTI::OperandValueProperties Opd1PropInfo, + TTI::OperandValueProperties Opd2PropInfo, + ArrayRef<const Value *> Args, + const Instruction *CxtI) { EVT OrigTy = TLI->getValueType(DL, Ty); if (!OrigTy.isSimple()) { return BaseT::getArithmeticInstrCost(Opcode, Ty, Opd1Info, Opd2Info, @@ -365,6 +369,9 @@ int GCNTTIImpl::getArithmeticInstrCost( if (SLT == MVT::i64) return get64BitInstrCost() * LT.first * NElts; + if (ST->has16BitInsts() && SLT == MVT::i16) + NElts = (NElts + 1) / 2; + // i32 return getFullRateInstrCost() * LT.first * NElts; case ISD::ADD: @@ -372,11 +379,14 @@ int GCNTTIImpl::getArithmeticInstrCost( case ISD::AND: case ISD::OR: case ISD::XOR: - if (SLT == MVT::i64){ + if (SLT == MVT::i64) { // and, or and xor are typically split into 2 VALU instructions. return 2 * getFullRateInstrCost() * LT.first * NElts; } + if (ST->has16BitInsts() && SLT == MVT::i16) + NElts = (NElts + 1) / 2; + return LT.first * NElts * getFullRateInstrCost(); case ISD::MUL: { const int QuarterRateCost = getQuarterRateInstrCost(); @@ -385,6 +395,9 @@ int GCNTTIImpl::getArithmeticInstrCost( return (4 * QuarterRateCost + (2 * 2) * FullRateCost) * LT.first * NElts; } + if (ST->has16BitInsts() && SLT == MVT::i16) + NElts = (NElts + 1) / 2; + // i32 return QuarterRateCost * NElts * LT.first; } @@ -394,6 +407,9 @@ int GCNTTIImpl::getArithmeticInstrCost( if (SLT == MVT::f64) return LT.first * NElts * get64BitInstrCost(); + if (ST->has16BitInsts() && SLT == MVT::f16) + NElts = (NElts + 1) / 2; + if (SLT == MVT::f32 || SLT == MVT::f16) return LT.first * NElts * getFullRateInstrCost(); break; @@ -412,7 +428,7 @@ int GCNTTIImpl::getArithmeticInstrCost( if (!Args.empty() && match(Args[0], PatternMatch::m_FPOne())) { // TODO: This is more complicated, unsafe flags etc. - if ((SLT == MVT::f32 && !ST->hasFP32Denormals()) || + if ((SLT == MVT::f32 && !HasFP32Denormals) || (SLT == MVT::f16 && ST->has16BitInsts())) { return LT.first * getQuarterRateInstrCost() * NElts; } @@ -431,7 +447,7 @@ int GCNTTIImpl::getArithmeticInstrCost( if (SLT == MVT::f32 || SLT == MVT::f16) { int Cost = 7 * getFullRateInstrCost() + 1 * getQuarterRateInstrCost(); - if (!ST->hasFP32Denormals()) { + if (!HasFP32Denormals) { // FP mode switches. Cost += 2 * getFullRateInstrCost(); } @@ -447,6 +463,49 @@ int GCNTTIImpl::getArithmeticInstrCost( Opd1PropInfo, Opd2PropInfo); } +template <typename T> +int GCNTTIImpl::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, + ArrayRef<T *> Args, + FastMathFlags FMF, unsigned VF) { + if (ID != Intrinsic::fma) + return BaseT::getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF); + + EVT OrigTy = TLI->getValueType(DL, RetTy); + if (!OrigTy.isSimple()) { + return BaseT::getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF); + } + + // Legalize the type. + std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, RetTy); + + unsigned NElts = LT.second.isVector() ? + LT.second.getVectorNumElements() : 1; + + MVT::SimpleValueType SLT = LT.second.getScalarType().SimpleTy; + + if (SLT == MVT::f64) + return LT.first * NElts * get64BitInstrCost(); + + if (ST->has16BitInsts() && SLT == MVT::f16) + NElts = (NElts + 1) / 2; + + return LT.first * NElts * (ST->hasFastFMAF32() ? getHalfRateInstrCost() + : getQuarterRateInstrCost()); +} + +int GCNTTIImpl::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, + ArrayRef<Value*> Args, FastMathFlags FMF, + unsigned VF) { + return getIntrinsicInstrCost<Value>(ID, RetTy, Args, FMF, VF); +} + +int GCNTTIImpl::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, + ArrayRef<Type *> Tys, FastMathFlags FMF, + unsigned ScalarizationCostPassed) { + return getIntrinsicInstrCost<Type>(ID, RetTy, Tys, FMF, + ScalarizationCostPassed); +} + unsigned GCNTTIImpl::getCFInstrCost(unsigned Opcode) { // XXX - For some reason this isn't called for switch. switch (Opcode) { @@ -671,10 +730,13 @@ unsigned GCNTTIImpl::getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index, bool GCNTTIImpl::areInlineCompatible(const Function *Caller, const Function *Callee) const { const TargetMachine &TM = getTLI()->getTargetMachine(); - const FeatureBitset &CallerBits = - TM.getSubtargetImpl(*Caller)->getFeatureBits(); - const FeatureBitset &CalleeBits = - TM.getSubtargetImpl(*Callee)->getFeatureBits(); + const GCNSubtarget *CallerST + = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Caller)); + const GCNSubtarget *CalleeST + = static_cast<const GCNSubtarget *>(TM.getSubtargetImpl(*Callee)); + + const FeatureBitset &CallerBits = CallerST->getFeatureBits(); + const FeatureBitset &CalleeBits = CalleeST->getFeatureBits(); FeatureBitset RealCallerBits = CallerBits & ~InlineFeatureIgnoreList; FeatureBitset RealCalleeBits = CalleeBits & ~InlineFeatureIgnoreList; @@ -683,8 +745,8 @@ bool GCNTTIImpl::areInlineCompatible(const Function *Caller, // FIXME: dx10_clamp can just take the caller setting, but there seems to be // no way to support merge for backend defined attributes. - AMDGPU::SIModeRegisterDefaults CallerMode(*Caller); - AMDGPU::SIModeRegisterDefaults CalleeMode(*Callee); + AMDGPU::SIModeRegisterDefaults CallerMode(*Caller, *CallerST); + AMDGPU::SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST); return CallerMode.isInlineCompatible(CalleeMode); } @@ -695,34 +757,114 @@ void GCNTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, unsigned GCNTTIImpl::getUserCost(const User *U, ArrayRef<const Value *> Operands) { - // Estimate extractelement elimination - if (const ExtractElementInst *EE = dyn_cast<ExtractElementInst>(U)) { - ConstantInt *CI = dyn_cast<ConstantInt>(EE->getOperand(1)); + const Instruction *I = dyn_cast<Instruction>(U); + if (!I) + return BaseT::getUserCost(U, Operands); + + // Estimate different operations to be optimized out + switch (I->getOpcode()) { + case Instruction::ExtractElement: { + ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1)); unsigned Idx = -1; if (CI) Idx = CI->getZExtValue(); - return getVectorInstrCost(EE->getOpcode(), EE->getOperand(0)->getType(), - Idx); + return getVectorInstrCost(I->getOpcode(), I->getOperand(0)->getType(), Idx); } - - // Estimate insertelement elimination - if (const InsertElementInst *IE = dyn_cast<InsertElementInst>(U)) { - ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2)); + case Instruction::InsertElement: { + ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(2)); unsigned Idx = -1; if (CI) Idx = CI->getZExtValue(); - return getVectorInstrCost(IE->getOpcode(), IE->getType(), Idx); + return getVectorInstrCost(I->getOpcode(), I->getType(), Idx); } + case Instruction::Call: { + if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) { + SmallVector<Value *, 4> Args(II->arg_operands()); + FastMathFlags FMF; + if (auto *FPMO = dyn_cast<FPMathOperator>(II)) + FMF = FPMO->getFastMathFlags(); + return getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(), Args, + FMF); + } else { + return BaseT::getUserCost(U, Operands); + } + } + case Instruction::ShuffleVector: { + const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I); + Type *Ty = Shuffle->getType(); + Type *SrcTy = Shuffle->getOperand(0)->getType(); + + // TODO: Identify and add costs for insert subvector, etc. + int SubIndex; + if (Shuffle->isExtractSubvectorMask(SubIndex)) + return getShuffleCost(TTI::SK_ExtractSubvector, SrcTy, SubIndex, Ty); + + if (Shuffle->changesLength()) + return BaseT::getUserCost(U, Operands); + + if (Shuffle->isIdentity()) + return 0; - // Estimate different intrinsics, e.g. llvm.fabs - if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) { - SmallVector<Value *, 4> Args(II->arg_operands()); - FastMathFlags FMF; - if (auto *FPMO = dyn_cast<FPMathOperator>(II)) - FMF = FPMO->getFastMathFlags(); - return getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(), Args, - FMF); + if (Shuffle->isReverse()) + return getShuffleCost(TTI::SK_Reverse, Ty, 0, nullptr); + + if (Shuffle->isSelect()) + return getShuffleCost(TTI::SK_Select, Ty, 0, nullptr); + + if (Shuffle->isTranspose()) + return getShuffleCost(TTI::SK_Transpose, Ty, 0, nullptr); + + if (Shuffle->isZeroEltSplat()) + return getShuffleCost(TTI::SK_Broadcast, Ty, 0, nullptr); + + if (Shuffle->isSingleSource()) + return getShuffleCost(TTI::SK_PermuteSingleSrc, Ty, 0, nullptr); + + return getShuffleCost(TTI::SK_PermuteTwoSrc, Ty, 0, nullptr); + } + case Instruction::ZExt: + case Instruction::SExt: + case Instruction::FPToUI: + case Instruction::FPToSI: + case Instruction::FPExt: + case Instruction::PtrToInt: + case Instruction::IntToPtr: + case Instruction::SIToFP: + case Instruction::UIToFP: + case Instruction::Trunc: + case Instruction::FPTrunc: + case Instruction::BitCast: + case Instruction::AddrSpaceCast: { + return getCastInstrCost(I->getOpcode(), I->getType(), + I->getOperand(0)->getType(), I); } + case Instruction::Add: + case Instruction::FAdd: + case Instruction::Sub: + case Instruction::FSub: + case Instruction::Mul: + case Instruction::FMul: + case Instruction::UDiv: + case Instruction::SDiv: + case Instruction::FDiv: + case Instruction::URem: + case Instruction::SRem: + case Instruction::FRem: + case Instruction::Shl: + case Instruction::LShr: + case Instruction::AShr: + case Instruction::And: + case Instruction::Or: + case Instruction::Xor: + case Instruction::FNeg: { + return getArithmeticInstrCost(I->getOpcode(), I->getType(), + TTI::OK_AnyValue, TTI::OK_AnyValue, + TTI::OP_None, TTI::OP_None, Operands, I); + } + default: + break; + } + return BaseT::getUserCost(U, Operands); } diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h index 67f7f9074f10..0b48f9f602b7 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h @@ -46,7 +46,7 @@ class AMDGPUTTIImpl final : public BasicTTIImplBase<AMDGPUTTIImpl> { Triple TargetTriple; - const TargetSubtargetInfo *ST; + const GCNSubtarget *ST; const TargetLoweringBase *TLI; const TargetSubtargetInfo *getST() const { return ST; } @@ -73,6 +73,7 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> { const AMDGPUTargetLowering *TLI; AMDGPUTTIImpl CommonTTI; bool IsGraphicsShader; + bool HasFP32Denormals; const FeatureBitset InlineFeatureIgnoreList = { // Codegen control options which don't matter. @@ -131,7 +132,8 @@ public: ST(static_cast<const GCNSubtarget*>(TM->getSubtargetImpl(F))), TLI(ST->getTargetLowering()), CommonTTI(TM, F), - IsGraphicsShader(AMDGPU::isShader(F.getCallingConv())) {} + IsGraphicsShader(AMDGPU::isShader(F.getCallingConv())), + HasFP32Denormals(ST->hasFP32Denormals(F)) { } bool hasBranchDivergence() { return true; } @@ -170,12 +172,13 @@ public: bool getTgtMemIntrinsic(IntrinsicInst *Inst, MemIntrinsicInfo &Info) const; int getArithmeticInstrCost( - unsigned Opcode, Type *Ty, - TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue, - TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue, - TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None, - TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None, - ArrayRef<const Value *> Args = ArrayRef<const Value *>()); + unsigned Opcode, Type *Ty, + TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue, + TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue, + TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None, + TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None, + ArrayRef<const Value *> Args = ArrayRef<const Value *>(), + const Instruction *CxtI = nullptr); unsigned getCFInstrCost(unsigned Opcode); @@ -204,13 +207,23 @@ public: bool areInlineCompatible(const Function *Caller, const Function *Callee) const; - unsigned getInliningThresholdMultiplier() { return 9; } + unsigned getInliningThresholdMultiplier() { return 11; } int getInlinerVectorBonusPercent() { return 0; } int getArithmeticReductionCost(unsigned Opcode, Type *Ty, bool IsPairwise); + template <typename T> + int getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<T *> Args, FastMathFlags FMF, + unsigned VF); + int getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<Type *> Tys, FastMathFlags FMF, + unsigned ScalarizationCostPassed = UINT_MAX); + int getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef<Value *> Args, FastMathFlags FMF, + unsigned VF = 1); int getMinMaxReductionCost(Type *Ty, Type *CondTy, bool IsPairwiseForm, bool IsUnsigned); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp index 396e0ed2e76c..191f603a66d6 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp @@ -27,7 +27,6 @@ #include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/Transforms/Utils/Local.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constants.h" @@ -36,10 +35,12 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/Type.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils.h" +#include "llvm/Transforms/Utils/Local.h" using namespace llvm; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp b/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp index d4401a22a1ad..281ae6d646e9 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp @@ -61,7 +61,7 @@ namespace { return false; MDNode *MaxMD = nullptr; auto MaxVer = 0U; - for (const auto &VersionMD : NamedMD->operands()) { + for (auto VersionMD : NamedMD->operands()) { assert(VersionMD->getNumOperands() == 2); auto CMajor = mdconst::extract<ConstantInt>(VersionMD->getOperand(0)); auto VersionMajor = CMajor->getZExtValue(); @@ -94,7 +94,7 @@ namespace { return false; SmallVector<Metadata *, 4> All; - for (const auto &MD : NamedMD->operands()) + for (auto MD : NamedMD->operands()) for (const auto &Op : MD->operands()) if (std::find(All.begin(), All.end(), Op.get()) == All.end()) All.push_back(Op.get()); diff --git a/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp b/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp index 101ecfc0c87c..1f28688a7296 100644 --- a/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp +++ b/llvm/lib/Target/AMDGPU/AMDILCFGStructurizer.cpp @@ -8,9 +8,9 @@ #include "AMDGPU.h" #include "AMDGPUSubtarget.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "R600InstrInfo.h" #include "R600RegisterInfo.h" -#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/SCCIterator.h" #include "llvm/ADT/SmallPtrSet.h" @@ -30,6 +30,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp index 9dd511fab57c..f3aa1a582368 100644 --- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp +++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp @@ -45,7 +45,7 @@ #include "llvm/Support/AMDHSAKernelDescriptor.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" -#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/Error.h" #include "llvm/Support/MachineValueType.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/SMLoc.h" @@ -1320,6 +1320,7 @@ private: bool validateIntClampSupported(const MCInst &Inst); bool validateMIMGAtomicDMask(const MCInst &Inst); bool validateMIMGGatherDMask(const MCInst &Inst); + bool validateMovrels(const MCInst &Inst); bool validateMIMGDataSize(const MCInst &Inst); bool validateMIMGAddrSize(const MCInst &Inst); bool validateMIMGD16(const MCInst &Inst); @@ -2362,7 +2363,7 @@ AMDGPUAsmParser::parseImm(OperandVector &Operands, bool HasSP3AbsModifier) { APFloat RealVal(APFloat::IEEEdouble()); auto roundMode = APFloat::rmNearestTiesToEven; - if (RealVal.convertFromString(Num, roundMode) == APFloat::opInvalidOp) { + if (errorToBool(RealVal.convertFromString(Num, roundMode).takeError())) { return MatchOperand_ParseFail; } if (Negate) @@ -3049,6 +3050,41 @@ bool AMDGPUAsmParser::validateMIMGGatherDMask(const MCInst &Inst) { return DMask == 0x1 || DMask == 0x2 || DMask == 0x4 || DMask == 0x8; } +static bool IsMovrelsSDWAOpcode(const unsigned Opcode) +{ + switch (Opcode) { + case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: + case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: + case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: + return true; + default: + return false; + } +} + +// movrels* opcodes should only allow VGPRS as src0. +// This is specified in .td description for vop1/vop3, +// but sdwa is handled differently. See isSDWAOperand. +bool AMDGPUAsmParser::validateMovrels(const MCInst &Inst) { + + const unsigned Opc = Inst.getOpcode(); + const MCInstrDesc &Desc = MII.get(Opc); + + if ((Desc.TSFlags & SIInstrFlags::SDWA) == 0 || !IsMovrelsSDWAOpcode(Opc)) + return true; + + const int Src0Idx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::src0); + assert(Src0Idx != -1); + + const MCOperand &Src0 = Inst.getOperand(Src0Idx); + if (!Src0.isReg()) + return false; + + auto Reg = Src0.getReg(); + const MCRegisterInfo *TRI = getContext().getRegisterInfo(); + return !isSGPR(mc2PseudoReg(Reg), TRI); +} + bool AMDGPUAsmParser::validateMIMGD16(const MCInst &Inst) { const unsigned Opc = Inst.getOpcode(); @@ -3469,6 +3505,10 @@ bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst, "invalid image_gather dmask: only one bit must be set"); return false; } + if (!validateMovrels(Inst)) { + Error(IDLoc, "source operand must be a VGPR"); + return false; + } if (!validateFlatOffset(Inst, Operands)) { return false; } @@ -6940,7 +6980,7 @@ AMDGPUOperand::Ptr AMDGPUAsmParser::defaultABID() const { } /// Force static initialization. -extern "C" void LLVMInitializeAMDGPUAsmParser() { +extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUAsmParser() { RegisterMCAsmParser<AMDGPUAsmParser> A(getTheAMDGPUTarget()); RegisterMCAsmParser<AMDGPUAsmParser> B(getTheGCNTarget()); } diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td index 1b12550aed88..691aff4ecbb8 100644 --- a/llvm/lib/Target/AMDGPU/BUFInstructions.td +++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td @@ -1621,8 +1621,8 @@ multiclass MUBUFStore_Atomic_Pattern <MUBUF_Pseudo Instr_ADDR64, MUBUF_Pseudo In >; } let SubtargetPredicate = isGFX6GFX7 in { -defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, store_atomic_global>; -defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, store_atomic_global>; +defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORD_ADDR64, BUFFER_STORE_DWORD_OFFSET, i32, atomic_store_global_32>; +defm : MUBUFStore_Atomic_Pattern <BUFFER_STORE_DWORDX2_ADDR64, BUFFER_STORE_DWORDX2_OFFSET, i64, atomic_store_global_64>; } // End Predicates = isGFX6GFX7 diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 816ec14a0e98..fe7faca8b157 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -58,6 +58,8 @@ class DS_Real <DS_Pseudo ds> : let isPseudo = 0; let isCodeGenOnly = 0; + let DS = 1; + let UseNamedOperandTable = 1; // copy relevant pseudo op flags let SubtargetPredicate = ds.SubtargetPredicate; @@ -617,7 +619,7 @@ def DS_ADD_SRC2_F32 : DS_1A<"ds_add_src2_f32">; def : GCNPat < (int_amdgcn_ds_swizzle i32:$src, timm:$offset16), - (DS_SWIZZLE_B32 $src, (as_i16imm $offset16), (i1 0)) + (DS_SWIZZLE_B32 VGPR_32:$src, (as_i16timm $offset16), (i1 0)) >; class DSReadPat <DS_Pseudo inst, ValueType vt, PatFrag frag, int gds=0> : GCNPat < @@ -731,8 +733,8 @@ defm : DSAtomicWritePat_mc <DS_WRITE_B32, i32, "atomic_store_local_32">; defm : DSAtomicWritePat_mc <DS_WRITE_B64, i64, "atomic_store_local_64">; let OtherPredicates = [D16PreservesUnusedBits] in { -def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_local_hi16>; -def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_local_hi16>; +def : DSWritePat <DS_WRITE_B16_D16_HI, i32, store_hi16_local>; +def : DSWritePat <DS_WRITE_B8_D16_HI, i32, truncstorei8_hi16_local>; } @@ -1075,7 +1077,7 @@ defm DS_MAX_SRC2_F64 : DS_Real_gfx6_gfx7_gfx10<0x0d3>; class DS_Real_vi <bits<8> op, DS_Pseudo ds> : DS_Real <ds>, SIMCInstr <ds.Mnemonic, SIEncodingFamily.VI> { - let AssemblerPredicates = [isGFX8GFX9]; + let AssemblerPredicate = isGFX8GFX9; let DecoderNamespace = "GFX8"; // encoding diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp index ec2e2c4e8b71..419513bdc248 100644 --- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp +++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp @@ -73,7 +73,7 @@ addOperand(MCInst &Inst, const MCOperand& Opnd) { Inst.addOperand(Opnd); return Opnd.isValid() ? MCDisassembler::Success : - MCDisassembler::SoftFail; + MCDisassembler::Fail; } static int insertNamedMCOperand(MCInst &MI, const MCOperand &Op, @@ -268,7 +268,6 @@ static bool isValidDPP8(const MCInst &MI) { DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size, ArrayRef<uint8_t> Bytes_, uint64_t Address, - raw_ostream &WS, raw_ostream &CS) const { CommentStream = &CS; bool IsSDWA = false; @@ -304,15 +303,6 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size, Res = tryDecodeInst(DecoderTableSDWA1064, MI, QW, Address); if (Res) { IsSDWA = true; break; } - // Some GFX9 subtargets repurposed the v_mad_mix_f32, v_mad_mixlo_f16 and - // v_mad_mixhi_f16 for FMA variants. Try to decode using this special - // table first so we print the correct name. - - if (STI.getFeatureBits()[AMDGPU::FeatureFmaMixInsts]) { - Res = tryDecodeInst(DecoderTableGFX9_DL64, MI, QW, Address); - if (Res) break; - } - if (STI.getFeatureBits()[AMDGPU::FeatureUnpackedD16VMem]) { Res = tryDecodeInst(DecoderTableGFX80_UNPACKED64, MI, QW, Address); if (Res) @@ -1262,7 +1252,7 @@ static MCDisassembler *createAMDGPUDisassembler(const Target &T, return new AMDGPUDisassembler(STI, Ctx, T.createMCInstrInfo()); } -extern "C" void LLVMInitializeAMDGPUDisassembler() { +extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUDisassembler() { TargetRegistry::RegisterMCDisassembler(getTheGCNTarget(), createAMDGPUDisassembler); TargetRegistry::RegisterMCSymbolizer(getTheGCNTarget(), diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h index c5eaba615c2a..f975af409a09 100644 --- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h +++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h @@ -53,7 +53,7 @@ public: DecodeStatus getInstruction(MCInst &MI, uint64_t &Size, ArrayRef<uint8_t> Bytes, uint64_t Address, - raw_ostream &WS, raw_ostream &CS) const override; + raw_ostream &CS) const override; const char* getRegClassName(unsigned RegClassID) const; diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td index 80ee17eba141..2057cac346d4 100644 --- a/llvm/lib/Target/AMDGPU/FLATInstructions.td +++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td @@ -364,32 +364,12 @@ multiclass FLAT_Global_Atomic_Pseudo< string opName, RegisterClass vdst_rc, ValueType vt, - SDPatternOperator atomic = null_frag, + SDPatternOperator atomic_rtn = null_frag, + SDPatternOperator atomic_no_rtn = null_frag, ValueType data_vt = vt, RegisterClass data_rc = vdst_rc> : - FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic, data_vt, data_rc>, - FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic, data_vt, data_rc>; - -class flat_binary_atomic_op<SDNode atomic_op> : PatFrag< - (ops node:$ptr, node:$value), - (atomic_op node:$ptr, node:$value), - [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::FLAT_ADDRESS;}] ->; - -def atomic_cmp_swap_flat : flat_binary_atomic_op<AMDGPUatomic_cmp_swap>; -def atomic_swap_flat : flat_binary_atomic_op<atomic_swap>; -def atomic_add_flat : flat_binary_atomic_op<atomic_load_add>; -def atomic_and_flat : flat_binary_atomic_op<atomic_load_and>; -def atomic_max_flat : flat_binary_atomic_op<atomic_load_max>; -def atomic_min_flat : flat_binary_atomic_op<atomic_load_min>; -def atomic_or_flat : flat_binary_atomic_op<atomic_load_or>; -def atomic_sub_flat : flat_binary_atomic_op<atomic_load_sub>; -def atomic_umax_flat : flat_binary_atomic_op<atomic_load_umax>; -def atomic_umin_flat : flat_binary_atomic_op<atomic_load_umin>; -def atomic_xor_flat : flat_binary_atomic_op<atomic_load_xor>; -def atomic_inc_flat : flat_binary_atomic_op<SIatomic_inc>; -def atomic_dec_flat : flat_binary_atomic_op<SIatomic_dec>; - + FLAT_Global_Atomic_Pseudo_NO_RTN<opName, vdst_rc, vt, atomic_no_rtn, data_vt, data_rc>, + FLAT_Global_Atomic_Pseudo_RTN<opName, vdst_rc, vt, atomic_rtn, data_vt, data_rc>; //===----------------------------------------------------------------------===// @@ -425,84 +405,84 @@ def FLAT_STORE_SHORT_D16_HI : FLAT_Store_Pseudo <"flat_store_short_d16_hi", VGPR } defm FLAT_ATOMIC_CMPSWAP : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap", - VGPR_32, i32, atomic_cmp_swap_flat, + VGPR_32, i32, AMDGPUatomic_cmp_swap_flat_32, v2i32, VReg_64>; defm FLAT_ATOMIC_CMPSWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_cmpswap_x2", - VReg_64, i64, atomic_cmp_swap_flat, + VReg_64, i64, AMDGPUatomic_cmp_swap_flat_64, v2i64, VReg_128>; defm FLAT_ATOMIC_SWAP : FLAT_Atomic_Pseudo <"flat_atomic_swap", - VGPR_32, i32, atomic_swap_flat>; + VGPR_32, i32, atomic_swap_flat_32>; defm FLAT_ATOMIC_SWAP_X2 : FLAT_Atomic_Pseudo <"flat_atomic_swap_x2", - VReg_64, i64, atomic_swap_flat>; + VReg_64, i64, atomic_swap_flat_64>; defm FLAT_ATOMIC_ADD : FLAT_Atomic_Pseudo <"flat_atomic_add", - VGPR_32, i32, atomic_add_flat>; + VGPR_32, i32, atomic_load_add_flat_32>; defm FLAT_ATOMIC_SUB : FLAT_Atomic_Pseudo <"flat_atomic_sub", - VGPR_32, i32, atomic_sub_flat>; + VGPR_32, i32, atomic_load_sub_flat_32>; defm FLAT_ATOMIC_SMIN : FLAT_Atomic_Pseudo <"flat_atomic_smin", - VGPR_32, i32, atomic_min_flat>; + VGPR_32, i32, atomic_load_min_flat_32>; defm FLAT_ATOMIC_UMIN : FLAT_Atomic_Pseudo <"flat_atomic_umin", - VGPR_32, i32, atomic_umin_flat>; + VGPR_32, i32, atomic_load_umin_flat_32>; defm FLAT_ATOMIC_SMAX : FLAT_Atomic_Pseudo <"flat_atomic_smax", - VGPR_32, i32, atomic_max_flat>; + VGPR_32, i32, atomic_load_max_flat_32>; defm FLAT_ATOMIC_UMAX : FLAT_Atomic_Pseudo <"flat_atomic_umax", - VGPR_32, i32, atomic_umax_flat>; + VGPR_32, i32, atomic_load_umax_flat_32>; defm FLAT_ATOMIC_AND : FLAT_Atomic_Pseudo <"flat_atomic_and", - VGPR_32, i32, atomic_and_flat>; + VGPR_32, i32, atomic_load_and_flat_32>; defm FLAT_ATOMIC_OR : FLAT_Atomic_Pseudo <"flat_atomic_or", - VGPR_32, i32, atomic_or_flat>; + VGPR_32, i32, atomic_load_or_flat_32>; defm FLAT_ATOMIC_XOR : FLAT_Atomic_Pseudo <"flat_atomic_xor", - VGPR_32, i32, atomic_xor_flat>; + VGPR_32, i32, atomic_load_xor_flat_32>; defm FLAT_ATOMIC_INC : FLAT_Atomic_Pseudo <"flat_atomic_inc", - VGPR_32, i32, atomic_inc_flat>; + VGPR_32, i32, atomic_inc_flat_32>; defm FLAT_ATOMIC_DEC : FLAT_Atomic_Pseudo <"flat_atomic_dec", - VGPR_32, i32, atomic_dec_flat>; + VGPR_32, i32, atomic_dec_flat_32>; defm FLAT_ATOMIC_ADD_X2 : FLAT_Atomic_Pseudo <"flat_atomic_add_x2", - VReg_64, i64, atomic_add_flat>; + VReg_64, i64, atomic_load_add_flat_64>; defm FLAT_ATOMIC_SUB_X2 : FLAT_Atomic_Pseudo <"flat_atomic_sub_x2", - VReg_64, i64, atomic_sub_flat>; + VReg_64, i64, atomic_load_sub_flat_64>; defm FLAT_ATOMIC_SMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smin_x2", - VReg_64, i64, atomic_min_flat>; + VReg_64, i64, atomic_load_min_flat_64>; defm FLAT_ATOMIC_UMIN_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umin_x2", - VReg_64, i64, atomic_umin_flat>; + VReg_64, i64, atomic_load_umin_flat_64>; defm FLAT_ATOMIC_SMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_smax_x2", - VReg_64, i64, atomic_max_flat>; + VReg_64, i64, atomic_load_max_flat_64>; defm FLAT_ATOMIC_UMAX_X2 : FLAT_Atomic_Pseudo <"flat_atomic_umax_x2", - VReg_64, i64, atomic_umax_flat>; + VReg_64, i64, atomic_load_umax_flat_64>; defm FLAT_ATOMIC_AND_X2 : FLAT_Atomic_Pseudo <"flat_atomic_and_x2", - VReg_64, i64, atomic_and_flat>; + VReg_64, i64, atomic_load_and_flat_64>; defm FLAT_ATOMIC_OR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_or_x2", - VReg_64, i64, atomic_or_flat>; + VReg_64, i64, atomic_load_or_flat_64>; defm FLAT_ATOMIC_XOR_X2 : FLAT_Atomic_Pseudo <"flat_atomic_xor_x2", - VReg_64, i64, atomic_xor_flat>; + VReg_64, i64, atomic_load_xor_flat_64>; defm FLAT_ATOMIC_INC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_inc_x2", - VReg_64, i64, atomic_inc_flat>; + VReg_64, i64, atomic_inc_flat_64>; defm FLAT_ATOMIC_DEC_X2 : FLAT_Atomic_Pseudo <"flat_atomic_dec_x2", - VReg_64, i64, atomic_dec_flat>; + VReg_64, i64, atomic_dec_flat_64>; // GFX7-, GFX10-only flat instructions. let SubtargetPredicate = isGFX7GFX10 in { @@ -556,11 +536,12 @@ defm GLOBAL_STORE_SHORT_D16_HI : FLAT_Global_Store_Pseudo <"global_store_short_d let is_flat_global = 1 in { defm GLOBAL_ATOMIC_CMPSWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap", - VGPR_32, i32, AMDGPUatomic_cmp_swap_global, + VGPR_32, i32, AMDGPUatomic_cmp_swap_global_32, null_frag, v2i32, VReg_64>; defm GLOBAL_ATOMIC_CMPSWAP_X2 : FLAT_Global_Atomic_Pseudo <"global_atomic_cmpswap_x2", - VReg_64, i64, AMDGPUatomic_cmp_swap_global, + VReg_64, i64, AMDGPUatomic_cmp_swap_global_64, + null_frag, v2i64, VReg_128>; defm GLOBAL_ATOMIC_SWAP : FLAT_Global_Atomic_Pseudo <"global_atomic_swap", @@ -778,7 +759,6 @@ def : FlatLoadPat <FLAT_LOAD_USHORT, zextloadi16_flat, i32>; def : FlatLoadPat <FLAT_LOAD_USHORT, load_flat, i16>; def : FlatLoadPat <FLAT_LOAD_SSHORT, sextloadi16_flat, i32>; def : FlatLoadPat <FLAT_LOAD_DWORDX3, load_flat, v3i32>; -def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, v4i32>; def : FlatLoadAtomicPat <FLAT_LOAD_DWORD, atomic_load_32_flat, i32>; def : FlatLoadAtomicPat <FLAT_LOAD_DWORDX2, atomic_load_64_flat, i64>; @@ -797,7 +777,11 @@ def : FlatLoadPat <FLAT_LOAD_DWORDX2, load_flat, vt>; } def : FlatStorePat <FLAT_STORE_DWORDX3, store_flat, v3i32, VReg_96>; -def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, v4i32, VReg_128>; + +foreach vt = VReg_128.RegTypes in { +def : FlatLoadPat <FLAT_LOAD_DWORDX4, load_flat, vt>; +def : FlatStorePat <FLAT_STORE_DWORDX4, store_flat, vt, VReg_128>; +} def : FlatStoreAtomicPat <FLAT_STORE_DWORD, atomic_store_flat_32, i32>; def : FlatStoreAtomicPat <FLAT_STORE_DWORDX2, atomic_store_flat_64, i64, VReg_64>; @@ -813,7 +797,7 @@ def : FlatAtomicPat <FLAT_ATOMIC_SMIN_RTN, atomic_load_min_global_32, i32>; def : FlatAtomicPat <FLAT_ATOMIC_UMIN_RTN, atomic_load_umin_global_32, i32>; def : FlatAtomicPat <FLAT_ATOMIC_OR_RTN, atomic_load_or_global_32, i32>; def : FlatAtomicPat <FLAT_ATOMIC_SWAP_RTN, atomic_swap_global_32, i32>; -def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global, i32, v2i32>; +def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; def : FlatAtomicPat <FLAT_ATOMIC_XOR_RTN, atomic_load_xor_global_32, i32>; def : FlatAtomicPat <FLAT_ATOMIC_ADD_X2_RTN, atomic_load_add_global_64, i64>; @@ -827,7 +811,7 @@ def : FlatAtomicPat <FLAT_ATOMIC_SMIN_X2_RTN, atomic_load_min_global_64, i64>; def : FlatAtomicPat <FLAT_ATOMIC_UMIN_X2_RTN, atomic_load_umin_global_64, i64>; def : FlatAtomicPat <FLAT_ATOMIC_OR_X2_RTN, atomic_load_or_global_64, i64>; def : FlatAtomicPat <FLAT_ATOMIC_SWAP_X2_RTN, atomic_swap_global_64, i64>; -def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global, i64, v2i64>; +def : FlatAtomicPat <FLAT_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; def : FlatAtomicPat <FLAT_ATOMIC_XOR_X2_RTN, atomic_load_xor_global_64, i64>; def : FlatStorePat <FLAT_STORE_BYTE, truncstorei8_flat, i16>; @@ -878,7 +862,11 @@ def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX2, store_global, vt, VReg_64>; } def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX3, load_global, v3i32>; -def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX4, load_global, v4i32>; + +foreach vt = VReg_128.RegTypes in { +def : FlatLoadSignedPat <GLOBAL_LOAD_DWORDX4, load_global, vt>; +def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX4, store_global, vt, VReg_128>; +} def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORD, atomic_load_32_global, i32>; def : FlatLoadAtomicPat <GLOBAL_LOAD_DWORDX2, atomic_load_64_global, i64>; @@ -888,7 +876,6 @@ def : FlatStoreSignedPat <GLOBAL_STORE_BYTE, truncstorei8_global, i16, VGPR_32>; def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, truncstorei16_global, i32, VGPR_32>; def : FlatStoreSignedPat <GLOBAL_STORE_SHORT, store_global, i16, VGPR_32>; def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX3, store_global, v3i32, VReg_96>; -def : FlatStoreSignedPat <GLOBAL_STORE_DWORDX4, store_global, v4i32, VReg_128>; let OtherPredicates = [D16PreservesUnusedBits] in { def : FlatStoreSignedPat <GLOBAL_STORE_SHORT_D16_HI, truncstorei16_hi16_global, i32>; @@ -909,8 +896,8 @@ def : FlatSignedLoadPat_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2i16>; def : FlatSignedLoadPat_D16 <GLOBAL_LOAD_SHORT_D16, load_d16_lo_global, v2f16>; } -def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORD, store_atomic_global, i32>; -def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORDX2, store_atomic_global, i64, VReg_64>; +def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORD, atomic_store_global_32, i32>; +def : FlatStoreSignedAtomicPat <GLOBAL_STORE_DWORDX2, atomic_store_global_64, i64, VReg_64>; def : FlatSignedAtomicPat <GLOBAL_ATOMIC_ADD_RTN, atomic_load_add_global_32, i32>; def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SUB_RTN, atomic_load_sub_global_32, i32>; @@ -923,7 +910,7 @@ def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMIN_RTN, atomic_load_min_global_32, i3 def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMIN_RTN, atomic_load_umin_global_32, i32>; def : FlatSignedAtomicPat <GLOBAL_ATOMIC_OR_RTN, atomic_load_or_global_32, i32>; def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SWAP_RTN, atomic_swap_global_32, i32>; -def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global, i32, v2i32>; +def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_RTN, AMDGPUatomic_cmp_swap_global_32, i32, v2i32>; def : FlatSignedAtomicPat <GLOBAL_ATOMIC_XOR_RTN, atomic_load_xor_global_32, i32>; def : FlatSignedAtomicPat <GLOBAL_ATOMIC_ADD_X2_RTN, atomic_load_add_global_64, i64>; @@ -937,7 +924,7 @@ def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SMIN_X2_RTN, atomic_load_min_global_64, def : FlatSignedAtomicPat <GLOBAL_ATOMIC_UMIN_X2_RTN, atomic_load_umin_global_64, i64>; def : FlatSignedAtomicPat <GLOBAL_ATOMIC_OR_X2_RTN, atomic_load_or_global_64, i64>; def : FlatSignedAtomicPat <GLOBAL_ATOMIC_SWAP_X2_RTN, atomic_swap_global_64, i64>; -def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global, i64, v2i64>; +def : FlatSignedAtomicPat <GLOBAL_ATOMIC_CMPSWAP_X2_RTN, AMDGPUatomic_cmp_swap_global_64, i64, v2i64>; def : FlatSignedAtomicPat <GLOBAL_ATOMIC_XOR_X2_RTN, atomic_load_xor_global_64, i64>; def : FlatAtomicPatNoRtn <GLOBAL_ATOMIC_ADD_F32, atomic_fadd_global_noret, f32>; diff --git a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp index 98678873e37c..10e2c3a263f1 100644 --- a/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp +++ b/llvm/lib/Target/AMDGPU/GCNDPPCombine.cpp @@ -104,6 +104,9 @@ public: AU.setPreservesCFG(); MachineFunctionPass::getAnalysisUsage(AU); } + +private: + int getDPPOp(unsigned Op) const; }; } // end anonymous namespace @@ -118,13 +121,13 @@ FunctionPass *llvm::createGCNDPPCombinePass() { return new GCNDPPCombine(); } -static int getDPPOp(unsigned Op) { +int GCNDPPCombine::getDPPOp(unsigned Op) const { auto DPP32 = AMDGPU::getDPPOp32(Op); - if (DPP32 != -1) - return DPP32; - - auto E32 = AMDGPU::getVOPe32(Op); - return E32 != -1 ? AMDGPU::getDPPOp32(E32) : -1; + if (DPP32 == -1) { + auto E32 = AMDGPU::getVOPe32(Op); + DPP32 = (E32 == -1)? -1 : AMDGPU::getDPPOp32(E32); + } + return (DPP32 == -1 || TII->pseudoToMCOpcode(DPP32) == -1) ? -1 : DPP32; } // tracks the register operand definition and returns: @@ -235,7 +238,8 @@ MachineInstr *GCNDPPCombine::createDPPInst(MachineInstr &OrigMI, } if (auto *Src2 = TII->getNamedOperand(OrigMI, AMDGPU::OpName::src2)) { - if (!TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src2)) { + if (!TII->getNamedOperand(*DPPInst.getInstr(), AMDGPU::OpName::src2) || + !TII->isOperandLegal(*DPPInst.getInstr(), NumOperands, Src2)) { LLVM_DEBUG(dbgs() << " failed: src2 is illegal\n"); Fail = true; break; diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp index 9528aee4c50e..3ef5a77af45e 100644 --- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp +++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp @@ -185,7 +185,7 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { if (SIInstrInfo::isMAI(*MI) && checkMAIHazards(MI) > 0) return NoopHazard; - if ((MI->mayLoad() || MI->mayStore()) && checkMAILdStHazards(MI) > 0) + if (MI->mayLoadOrStore() && checkMAILdStHazards(MI) > 0) return NoopHazard; if (MI->isInlineAsm() && checkInlineAsmHazards(MI) > 0) @@ -296,7 +296,7 @@ unsigned GCNHazardRecognizer::PreEmitNoopsCommon(MachineInstr *MI) { if (SIInstrInfo::isMAI(*MI)) return std::max(WaitStates, checkMAIHazards(MI)); - if (MI->mayLoad() || MI->mayStore()) + if (MI->mayLoadOrStore()) return std::max(WaitStates, checkMAILdStHazards(MI)); return WaitStates; diff --git a/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp b/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp index 36a8f74150f5..f6023f3a40a2 100644 --- a/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp +++ b/llvm/lib/Target/AMDGPU/GCNNSAReassign.cpp @@ -23,6 +23,7 @@ #include "llvm/CodeGen/LiveRegMatrix.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/MathExtras.h" #include <algorithm> diff --git a/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp b/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp index 2927d4eb745a..76593bc0e5ac 100644 --- a/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp +++ b/llvm/lib/Target/AMDGPU/GCNRegBankReassign.cpp @@ -32,9 +32,9 @@ #include "AMDGPU.h" #include "AMDGPUSubtarget.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIInstrInfo.h" #include "SIMachineFunctionInfo.h" -#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/LiveInterval.h" @@ -43,6 +43,7 @@ #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/MathExtras.h" using namespace llvm; diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index 973491a70d3c..e109eed5f607 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -16,6 +16,7 @@ #include "SIInstrInfo.h" #include "SIMachineFunctionInfo.h" #include "SIRegisterInfo.h" +#include "Utils/AMDGPUBaseInfo.h" #include "llvm/CodeGen/RegisterClassInfo.h" #include "llvm/Support/MathExtras.h" @@ -389,8 +390,16 @@ void GCNScheduleDAGMILive::schedule() { } if (WavesAfter >= MinOccupancy) { - Pressure[RegionIdx] = PressureAfter; - return; + unsigned TotalVGPRs = AMDGPU::IsaInfo::getAddressableNumVGPRs(&ST); + unsigned TotalSGPRs = AMDGPU::IsaInfo::getAddressableNumSGPRs(&ST); + if (WavesAfter > MFI.getMinWavesPerEU() || + PressureAfter.less(ST, PressureBefore) || + (TotalVGPRs >= PressureAfter.getVGPRNum() && + TotalSGPRs >= PressureAfter.getSGPRNum())) { + Pressure[RegionIdx] = PressureAfter; + return; + } + LLVM_DEBUG(dbgs() << "New pressure will result in more spilling.\n"); } LLVM_DEBUG(dbgs() << "Attempting to revert scheduling.\n"); diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp index a9888e6ed924..f65dc25d7eec 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp @@ -26,10 +26,11 @@ using namespace llvm; using namespace llvm::AMDGPU; -void AMDGPUInstPrinter::printInst(const MCInst *MI, raw_ostream &OS, - StringRef Annot, const MCSubtargetInfo &STI) { +void AMDGPUInstPrinter::printInst(const MCInst *MI, uint64_t Address, + StringRef Annot, const MCSubtargetInfo &STI, + raw_ostream &OS) { OS.flush(); - printInstruction(MI, STI, OS); + printInstruction(MI, Address, STI, OS); printAnnotation(OS, Annot); } @@ -1342,10 +1343,11 @@ void AMDGPUInstPrinter::printEndpgm(const MCInst *MI, unsigned OpNo, #include "AMDGPUGenAsmWriter.inc" -void R600InstPrinter::printInst(const MCInst *MI, raw_ostream &O, - StringRef Annot, const MCSubtargetInfo &STI) { +void R600InstPrinter::printInst(const MCInst *MI, uint64_t Address, + StringRef Annot, const MCSubtargetInfo &STI, + raw_ostream &O) { O.flush(); - printInstruction(MI, O); + printInstruction(MI, Address, O); printAnnotation(O, Annot); } diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h index 66b70831ff9e..ba53003e9041 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h @@ -23,12 +23,12 @@ public: : MCInstPrinter(MAI, MII, MRI) {} //Autogenerated by tblgen - void printInstruction(const MCInst *MI, const MCSubtargetInfo &STI, - raw_ostream &O); + void printInstruction(const MCInst *MI, uint64_t Address, + const MCSubtargetInfo &STI, raw_ostream &O); static const char *getRegisterName(unsigned RegNo); - void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot, - const MCSubtargetInfo &STI) override; + void printInst(const MCInst *MI, uint64_t Address, StringRef Annot, + const MCSubtargetInfo &STI, raw_ostream &O) override; static void printRegOperand(unsigned RegNo, raw_ostream &O, const MCRegisterInfo &MRI); @@ -240,9 +240,9 @@ public: const MCRegisterInfo &MRI) : MCInstPrinter(MAI, MII, MRI) {} - void printInst(const MCInst *MI, raw_ostream &O, StringRef Annot, - const MCSubtargetInfo &STI) override; - void printInstruction(const MCInst *MI, raw_ostream &O); + void printInst(const MCInst *MI, uint64_t Address, StringRef Annot, + const MCSubtargetInfo &STI, raw_ostream &O) override; + void printInstruction(const MCInst *MI, uint64_t Address, raw_ostream &O); static const char *getRegisterName(unsigned RegNo); void printAbs(const MCInst *MI, unsigned OpNo, raw_ostream &O); diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp index 9e04ab9bae93..9644e66fda4e 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp @@ -14,7 +14,9 @@ using namespace llvm; -AMDGPUMCAsmInfo::AMDGPUMCAsmInfo(const Triple &TT) : MCAsmInfoELF() { +AMDGPUMCAsmInfo::AMDGPUMCAsmInfo(const Triple &TT, + const MCTargetOptions &Options) + : MCAsmInfoELF() { CodePointerSize = (TT.getArch() == Triple::amdgcn) ? 8 : 4; StackGrowsUp = true; HasSingleParameterDotFile = false; diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h index 71e63ec27a8f..65c9b1917bf8 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h @@ -25,7 +25,7 @@ class Triple; // with 'L' as a local symbol. class AMDGPUMCAsmInfo : public MCAsmInfoELF { public: - explicit AMDGPUMCAsmInfo(const Triple &TT); + explicit AMDGPUMCAsmInfo(const Triple &TT, const MCTargetOptions &Options); bool shouldOmitSectionDirective(StringRef SectionName) const override; unsigned getMaxInstLength(const MCSubtargetInfo *STI) const override; }; diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp index 88df64d18cc5..9507836c64c2 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp @@ -134,7 +134,7 @@ static MCInstrAnalysis *createAMDGPUMCInstrAnalysis(const MCInstrInfo *Info) { return new AMDGPUMCInstrAnalysis(Info); } -extern "C" void LLVMInitializeAMDGPUTargetMC() { +extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTargetMC() { TargetRegistry::RegisterMCInstrInfo(getTheGCNTarget(), createAMDGPUMCInstrInfo); TargetRegistry::RegisterMCInstrInfo(getTheAMDGPUTarget(), createR600MCInstrInfo); diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp index c15da8075a34..fef665c2900e 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp @@ -60,6 +60,7 @@ StringRef AMDGPUTargetStreamer::getArchNameFromElfMach(unsigned ElfMach) { AMDGPU::GPUKind AK; switch (ElfMach) { + default: llvm_unreachable("Unhandled ELF::EF_AMDGPU type"); case ELF::EF_AMDGPU_MACH_R600_R600: AK = GK_R600; break; case ELF::EF_AMDGPU_MACH_R600_R630: AK = GK_R630; break; case ELF::EF_AMDGPU_MACH_R600_RS880: AK = GK_RS880; break; diff --git a/llvm/lib/Target/AMDGPU/MIMGInstructions.td b/llvm/lib/Target/AMDGPU/MIMGInstructions.td index f33ad950d5d9..4006a6205fb8 100644 --- a/llvm/lib/Target/AMDGPU/MIMGInstructions.td +++ b/llvm/lib/Target/AMDGPU/MIMGInstructions.td @@ -183,7 +183,7 @@ class MIMGNSAHelper<int num_addrs> { class MIMG_gfx6789<bits<8> op, dag outs, string dns = ""> : MIMG<outs, dns>, MIMGe_gfx6789<op> { let SubtargetPredicate = isGFX6GFX7GFX8GFX9; - let AssemblerPredicates = [isGFX6GFX7GFX8GFX9]; + let AssemblerPredicate = isGFX6GFX7GFX8GFX9; let MIMGEncoding = MIMGEncGfx6; @@ -194,7 +194,7 @@ class MIMG_gfx6789<bits<8> op, dag outs, string dns = ""> class MIMG_gfx10<int op, dag outs, string dns = ""> : MIMG<outs, dns>, MIMGe_gfx10<op> { let SubtargetPredicate = isGFX10Plus; - let AssemblerPredicates = [isGFX10Plus]; + let AssemblerPredicate = isGFX10Plus; let MIMGEncoding = MIMGEncGfx10Default; @@ -207,7 +207,7 @@ class MIMG_gfx10<int op, dag outs, string dns = ""> class MIMG_nsa_gfx10<int op, dag outs, int num_addrs, string dns=""> : MIMG<outs, dns>, MIMGe_gfx10<op> { let SubtargetPredicate = isGFX10Plus; - let AssemblerPredicates = [isGFX10Plus]; + let AssemblerPredicate = isGFX10Plus; let MIMGEncoding = MIMGEncGfx10NSA; @@ -416,13 +416,13 @@ class MIMG_Atomic_si<mimg op, string asm, RegisterClass data_rc, RegisterClass addr_rc, bit enableDasm = 0> : MIMG_Atomic_gfx6789_base<op.SI_GFX10, asm, data_rc, addr_rc, !if(enableDasm, "GFX6GFX7", "")> { - let AssemblerPredicates = [isGFX6GFX7]; + let AssemblerPredicate = isGFX6GFX7; } class MIMG_Atomic_vi<mimg op, string asm, RegisterClass data_rc, RegisterClass addr_rc, bit enableDasm = 0> : MIMG_Atomic_gfx6789_base<op.VI, asm, data_rc, addr_rc, !if(enableDasm, "GFX8", "")> { - let AssemblerPredicates = [isGFX8GFX9]; + let AssemblerPredicate = isGFX8GFX9; let MIMGEncoding = MIMGEncGfx8; } diff --git a/llvm/lib/Target/AMDGPU/R600AsmPrinter.cpp b/llvm/lib/Target/AMDGPU/R600AsmPrinter.cpp index b29cd75f75cf..ed23c8ea814b 100644 --- a/llvm/lib/Target/AMDGPU/R600AsmPrinter.cpp +++ b/llvm/lib/Target/AMDGPU/R600AsmPrinter.cpp @@ -1,4 +1,4 @@ -//===-- R600AsmPrinter.cpp - R600 Assebly printer ------------------------===// +//===-- R600AsmPrinter.cpp - R600 Assembly printer ------------------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp index 659458b0b752..1b1f5f9a404a 100644 --- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp @@ -37,6 +37,7 @@ #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/IntrinsicsR600.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/ErrorHandling.h" @@ -62,6 +63,9 @@ R600TargetLowering::R600TargetLowering(const TargetMachine &TM, addRegisterClass(MVT::v4f32, &R600::R600_Reg128RegClass); addRegisterClass(MVT::v4i32, &R600::R600_Reg128RegClass); + setBooleanContents(ZeroOrNegativeOneBooleanContent); + setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); + computeRegisterProperties(Subtarget->getRegisterInfo()); // Legalize loads and stores to the private address space. @@ -223,10 +227,8 @@ R600TargetLowering::R600TargetLowering(const TargetMachine &TM, setOperationAction(ISD::FMA, MVT::f64, Expand); } - // FIXME: This was moved from AMDGPUTargetLowering, I'm not sure if we - // need it for R600. - if (!Subtarget->hasFP32Denormals()) - setOperationAction(ISD::FMAD, MVT::f32, Legal); + // FIXME: May need no denormals check + setOperationAction(ISD::FMAD, MVT::f32, Legal); if (!Subtarget->hasBFI()) { // fcopysign can be done in a single instruction with BFI. @@ -970,10 +972,9 @@ SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const // // Move hardware True/False values to the correct operand. - ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); - ISD::CondCode InverseCC = - ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); if (isHWTrueValue(False) && isHWFalseValue(True)) { + ISD::CondCode CCOpcode = cast<CondCodeSDNode>(CC)->get(); + ISD::CondCode InverseCC = ISD::getSetCCInverse(CCOpcode, CompareVT); if (isCondCodeLegal(InverseCC, CompareVT.getSimpleVT())) { std::swap(False, True); CC = DAG.getCondCode(InverseCC); @@ -1013,7 +1014,7 @@ SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const CC = DAG.getCondCode(CCSwapped); } else { // Try inverting the conditon and then swapping the operands - ISD::CondCode CCInv = ISD::getSetCCInverse(CCOpcode, CompareVT.isInteger()); + ISD::CondCode CCInv = ISD::getSetCCInverse(CCOpcode, CompareVT); CCSwapped = ISD::getSetCCSwappedOperands(CCInv); if (isCondCodeLegal(CCSwapped, CompareVT.getSimpleVT())) { std::swap(True, False); @@ -1039,7 +1040,7 @@ SDValue R600TargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const case ISD::SETONE: case ISD::SETUNE: case ISD::SETNE: - CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT == MVT::i32); + CCOpcode = ISD::getSetCCInverse(CCOpcode, CompareVT); Temp = True; True = False; False = Temp; @@ -1174,8 +1175,7 @@ SDValue R600TargetLowering::lowerPrivateTruncStore(StoreSDNode *Store, // Load dword // TODO: can we be smarter about machine pointer info? - MachinePointerInfo PtrInfo(UndefValue::get( - Type::getInt32PtrTy(*DAG.getContext(), AMDGPUAS::PRIVATE_ADDRESS))); + MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS); SDValue Dst = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo); Chain = Dst.getValue(1); @@ -1405,8 +1405,7 @@ SDValue R600TargetLowering::lowerPrivateExtLoad(SDValue Op, // Load dword // TODO: can we be smarter about machine pointer info? - MachinePointerInfo PtrInfo(UndefValue::get( - Type::getInt32PtrTy(*DAG.getContext(), AMDGPUAS::PRIVATE_ADDRESS))); + MachinePointerInfo PtrInfo(AMDGPUAS::PRIVATE_ADDRESS); SDValue Read = DAG.getLoad(MVT::i32, DL, Chain, Ptr, PtrInfo); // Get offset within the register. @@ -1457,7 +1456,9 @@ SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { if ((LoadNode->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS || LoadNode->getAddressSpace() == AMDGPUAS::PRIVATE_ADDRESS) && VT.isVector()) { - return scalarizeVectorLoad(LoadNode, DAG); + SDValue Ops[2]; + std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(LoadNode, DAG); + return DAG.getMergeValues(Ops, DL); } // This is still used for explicit load from addrspace(8) @@ -1500,7 +1501,6 @@ SDValue R600TargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { // buffer. However SEXT loads from other address spaces are not supported, so // we need to expand them here. if (LoadNode->getExtensionType() == ISD::SEXTLOAD) { - EVT MemVT = LoadNode->getMemoryVT(); assert(!MemVT.isVector() && (MemVT == MVT::i16 || MemVT == MVT::i8)); SDValue NewLoad = DAG.getExtLoad( ISD::EXTLOAD, DL, VT, Chain, Ptr, LoadNode->getPointerInfo(), MemVT, @@ -1608,9 +1608,6 @@ SDValue R600TargetLowering::LowerFormalArguments( continue; } - PointerType *PtrTy = PointerType::get(VT.getTypeForEVT(*DAG.getContext()), - AMDGPUAS::PARAM_I_ADDRESS); - // i64 isn't a legal type, so the register type used ends up as i32, which // isn't expected here. It attempts to create this sextload, but it ends up // being invalid. Somehow this seems to work with i64 arguments, but breaks @@ -1631,11 +1628,10 @@ SDValue R600TargetLowering::LowerFormalArguments( // XXX - I think PartOffset should give you this, but it seems to give the // size of the register which isn't useful. - unsigned ValBase = ArgLocs[In.getOrigArgIndex()].getLocMemOffset(); unsigned PartOffset = VA.getLocMemOffset(); unsigned Alignment = MinAlign(VT.getStoreSize(), PartOffset); - MachinePointerInfo PtrInfo(UndefValue::get(PtrTy), PartOffset - ValBase); + MachinePointerInfo PtrInfo(AMDGPUAS::PARAM_I_ADDRESS); SDValue Arg = DAG.getLoad( ISD::UNINDEXED, Ext, VT, DL, Chain, DAG.getConstant(PartOffset, DL, MVT::i32), DAG.getUNDEF(MVT::i32), @@ -1715,12 +1711,7 @@ static SDValue CompactSwizzlableVector( if (NewBldVec[i].isUndef()) continue; - // Fix spurious warning with gcc 7.3 -O3 - // warning: array subscript is above array bounds [-Warray-bounds] - // if (NewBldVec[i] == NewBldVec[j]) { - // ~~~~~~~~~~~^ - if (i >= 4) - continue; + for (unsigned j = 0; j < i; j++) { if (NewBldVec[i] == NewBldVec[j]) { NewBldVec[i] = DAG.getUNDEF(NewBldVec[i].getValueType()); @@ -1889,8 +1880,6 @@ SDValue R600TargetLowering::PerformDAGCombine(SDNode *N, DAG.getConstant(-1, DL, MVT::i32), // True DAG.getConstant(0, DL, MVT::i32), // False SelectCC.getOperand(4)); // CC - - break; } // insert_vector_elt (build_vector elt0, ... , eltN), NewEltIdx, idx @@ -1999,8 +1988,7 @@ SDValue R600TargetLowering::PerformDAGCombine(SDNode *N, case ISD::SETNE: return LHS; case ISD::SETEQ: { ISD::CondCode LHSCC = cast<CondCodeSDNode>(LHS.getOperand(4))->get(); - LHSCC = ISD::getSetCCInverse(LHSCC, - LHS.getOperand(0).getValueType().isInteger()); + LHSCC = ISD::getSetCCInverse(LHSCC, LHS.getOperand(0).getValueType()); if (DCI.isBeforeLegalizeOps() || isCondCodeLegal(LHSCC, LHS.getOperand(0).getSimpleValueType())) return DAG.getSelectCC(DL, diff --git a/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp b/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp index 04a5e93f6213..346296c77377 100644 --- a/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/R600InstrInfo.cpp @@ -60,8 +60,8 @@ bool R600InstrInfo::isVector(const MachineInstr &MI) const { void R600InstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, - const DebugLoc &DL, unsigned DestReg, - unsigned SrcReg, bool KillSrc) const { + const DebugLoc &DL, MCRegister DestReg, + MCRegister SrcReg, bool KillSrc) const { unsigned VectorComponents = 0; if ((R600::R600_Reg128RegClass.contains(DestReg) || R600::R600_Reg128VerticalRegClass.contains(DestReg)) && @@ -541,7 +541,7 @@ R600InstrInfo::fitsReadPortLimitations(const std::vector<MachineInstr *> &IG, std::vector<std::vector<std::pair<int, unsigned>>> IGSrcs; ValidSwizzle.clear(); - unsigned ConstCount; + unsigned ConstCount = 0; BankSwizzle TransBS = ALU_VEC_012_SCL_210; for (unsigned i = 0, e = IG.size(); i < e; ++i) { IGSrcs.push_back(ExtractSrcs(*IG[i], PV, ConstCount)); diff --git a/llvm/lib/Target/AMDGPU/R600InstrInfo.h b/llvm/lib/Target/AMDGPU/R600InstrInfo.h index 00d96c9676aa..873ee08470cb 100644 --- a/llvm/lib/Target/AMDGPU/R600InstrInfo.h +++ b/llvm/lib/Target/AMDGPU/R600InstrInfo.h @@ -73,7 +73,7 @@ public: } void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, - const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, + const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override; bool isLegalToSplitMBBAt(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const override; diff --git a/llvm/lib/Target/AMDGPU/R600Instructions.td b/llvm/lib/Target/AMDGPU/R600Instructions.td index f40eece859ee..cbdf0de44f87 100644 --- a/llvm/lib/Target/AMDGPU/R600Instructions.td +++ b/llvm/lib/Target/AMDGPU/R600Instructions.td @@ -295,9 +295,23 @@ class VTX_READ <string name, dag outs, list<dag> pattern> let VTXInst = 1; } -// FIXME: Deprecated. -class LocalLoad <SDPatternOperator op> : LoadFrag <op>, LocalAddress; +// Legacy. +def atomic_cmp_swap_global_noret : PatFrag< + (ops node:$ptr, node:$cmp, node:$value), + (atomic_cmp_swap node:$ptr, node:$cmp, node:$value), + [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (SDValue(N, 0).use_empty());}]>; +def atomic_cmp_swap_global_ret : PatFrag< + (ops node:$ptr, node:$cmp, node:$value), + (atomic_cmp_swap node:$ptr, node:$cmp, node:$value), + [{return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (!SDValue(N, 0).use_empty());}]>; + +def mskor_global : PatFrag<(ops node:$val, node:$ptr), + (AMDGPUstore_mskor node:$val, node:$ptr), [{ + return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; +}]>; + +// FIXME: These are deprecated class AZExtLoadBase <SDPatternOperator ld_node>: PatFrag<(ops node:$ptr), (ld_node node:$ptr), [{ LoadSDNode *L = cast<LoadSDNode>(N); @@ -319,9 +333,10 @@ def az_extloadi32 : PatFrag<(ops node:$ptr), (az_extload node:$ptr), [{ return cast<LoadSDNode>(N)->getMemoryVT() == MVT::i32; }]>; -// FIXME: These are deprecated -def az_extloadi8_local : LocalLoad <az_extloadi8>; -def az_extloadi16_local : LocalLoad <az_extloadi16>; +let AddressSpaces = LoadAddress_local.AddrSpaces in { +def az_extloadi8_local : PatFrag<(ops node:$ptr), (az_extloadi8 node:$ptr)>; +def az_extloadi16_local : PatFrag<(ops node:$ptr), (az_extloadi16 node:$ptr)>; +} class LoadParamFrag <PatFrag load_type> : PatFrag < (ops node:$ptr), (load_type node:$ptr), diff --git a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp index b764ca7d7061..27320472cacb 100644 --- a/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp +++ b/llvm/lib/Target/AMDGPU/SIAnnotateControlFlow.cpp @@ -32,6 +32,7 @@ #include "llvm/IR/Module.h" #include "llvm/IR/Type.h" #include "llvm/IR/ValueHandle.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" diff --git a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp index 65286751c12d..914d2a5ef148 100644 --- a/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp +++ b/llvm/lib/Target/AMDGPU/SIFixSGPRCopies.cpp @@ -66,9 +66,9 @@ #include "AMDGPU.h" #include "AMDGPUSubtarget.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIInstrInfo.h" #include "SIRegisterInfo.h" -#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" @@ -82,6 +82,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/CommandLine.h" diff --git a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp index 4eac03168760..2ff8baf29394 100644 --- a/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp +++ b/llvm/lib/Target/AMDGPU/SIFoldOperands.cpp @@ -14,7 +14,7 @@ #include "SIMachineFunctionInfo.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/DepthFirstIterator.h" -#include "llvm/CodeGen/LiveIntervals.h" +#include "llvm/ADT/SetVector.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -312,6 +312,19 @@ static bool isUseMIInFoldList(ArrayRef<FoldCandidate> FoldList, return false; } +static void appendFoldCandidate(SmallVectorImpl<FoldCandidate> &FoldList, + MachineInstr *MI, unsigned OpNo, + MachineOperand *FoldOp, bool Commuted = false, + int ShrinkOp = -1) { + // Skip additional folding on the same operand. + for (FoldCandidate &Fold : FoldList) + if (Fold.UseMI == MI && Fold.UseOpNo == OpNo) + return; + LLVM_DEBUG(dbgs() << "Append " << (Commuted ? "commuted" : "normal") + << " operand " << OpNo << "\n " << *MI << '\n'); + FoldList.push_back(FoldCandidate(MI, OpNo, FoldOp, Commuted, ShrinkOp)); +} + static bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList, MachineInstr *MI, unsigned OpNo, MachineOperand *OpToFold, @@ -344,7 +357,7 @@ static bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList, // Special case for s_setreg_b32 if (Opc == AMDGPU::S_SETREG_B32 && OpToFold->isImm()) { MI->setDesc(TII->get(AMDGPU::S_SETREG_IMM32_B32)); - FoldList.push_back(FoldCandidate(MI, OpNo, OpToFold)); + appendFoldCandidate(FoldList, MI, OpNo, OpToFold); return true; } @@ -403,8 +416,7 @@ static bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList, unsigned MaybeCommutedOpc = MI->getOpcode(); int Op32 = AMDGPU::getVOPe32(MaybeCommutedOpc); - FoldList.push_back(FoldCandidate(MI, CommuteOpNo, OpToFold, true, - Op32)); + appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true, Op32); return true; } @@ -412,11 +424,34 @@ static bool tryAddToFoldList(SmallVectorImpl<FoldCandidate> &FoldList, return false; } - FoldList.push_back(FoldCandidate(MI, CommuteOpNo, OpToFold, true)); + appendFoldCandidate(FoldList, MI, CommuteOpNo, OpToFold, true); return true; } - FoldList.push_back(FoldCandidate(MI, OpNo, OpToFold)); + // Check the case where we might introduce a second constant operand to a + // scalar instruction + if (TII->isSALU(MI->getOpcode())) { + const MCInstrDesc &InstDesc = MI->getDesc(); + const MCOperandInfo &OpInfo = InstDesc.OpInfo[OpNo]; + const SIRegisterInfo &SRI = TII->getRegisterInfo(); + + // Fine if the operand can be encoded as an inline constant + if (OpToFold->isImm()) { + if (!SRI.opCanUseInlineConstant(OpInfo.OperandType) || + !TII->isInlineConstant(*OpToFold, OpInfo)) { + // Otherwise check for another constant + for (unsigned i = 0, e = InstDesc.getNumOperands(); i != e; ++i) { + auto &Op = MI->getOperand(i); + if (OpNo != i && + TII->isLiteralConstantLike(Op, OpInfo)) { + return false; + } + } + } + } + } + + appendFoldCandidate(FoldList, MI, OpNo, OpToFold); return true; } @@ -429,6 +464,42 @@ static bool isUseSafeToFold(const SIInstrInfo *TII, //return !MI.hasRegisterImplicitUseOperand(UseMO.getReg()); } +// Find a def of the UseReg, check if it is a reg_seqence and find initializers +// for each subreg, tracking it to foldable inline immediate if possible. +// Returns true on success. +static bool getRegSeqInit( + SmallVectorImpl<std::pair<MachineOperand*, unsigned>> &Defs, + Register UseReg, uint8_t OpTy, + const SIInstrInfo *TII, const MachineRegisterInfo &MRI) { + MachineInstr *Def = MRI.getUniqueVRegDef(UseReg); + if (!Def || !Def->isRegSequence()) + return false; + + for (unsigned I = 1, E = Def->getNumExplicitOperands(); I < E; I += 2) { + MachineOperand *Sub = &Def->getOperand(I); + assert (Sub->isReg()); + + for (MachineInstr *SubDef = MRI.getUniqueVRegDef(Sub->getReg()); + SubDef && Sub->isReg() && !Sub->getSubReg() && + TII->isFoldableCopy(*SubDef); + SubDef = MRI.getUniqueVRegDef(Sub->getReg())) { + MachineOperand *Op = &SubDef->getOperand(1); + if (Op->isImm()) { + if (TII->isInlineConstant(*Op, OpTy)) + Sub = Op; + break; + } + if (!Op->isReg()) + break; + Sub = Op; + } + + Defs.push_back(std::make_pair(Sub, Def->getOperand(I + 1).getImm())); + } + + return true; +} + static bool tryToFoldACImm(const SIInstrInfo *TII, const MachineOperand &OpToFold, MachineInstr *UseMI, @@ -462,39 +533,30 @@ static bool tryToFoldACImm(const SIInstrInfo *TII, return false; MachineRegisterInfo &MRI = UseMI->getParent()->getParent()->getRegInfo(); - const MachineInstr *Def = MRI.getUniqueVRegDef(UseReg); - if (!Def || !Def->isRegSequence()) + SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs; + if (!getRegSeqInit(Defs, UseReg, OpTy, TII, MRI)) return false; - int64_t Imm; - MachineOperand *Op; - for (unsigned I = 1, E = Def->getNumExplicitOperands(); I < E; I += 2) { - const MachineOperand &Sub = Def->getOperand(I); - if (!Sub.isReg() || Sub.getSubReg()) + int32_t Imm; + for (unsigned I = 0, E = Defs.size(); I != E; ++I) { + const MachineOperand *Op = Defs[I].first; + if (!Op->isImm()) return false; - MachineInstr *SubDef = MRI.getUniqueVRegDef(Sub.getReg()); - while (SubDef && !SubDef->isMoveImmediate() && - !SubDef->getOperand(1).isImm() && TII->isFoldableCopy(*SubDef)) - SubDef = MRI.getUniqueVRegDef(SubDef->getOperand(1).getReg()); - if (!SubDef || !SubDef->isMoveImmediate() || !SubDef->getOperand(1).isImm()) - return false; - Op = &SubDef->getOperand(1); + auto SubImm = Op->getImm(); - if (I == 1) { - if (!TII->isInlineConstant(SubDef->getOperand(1), OpTy)) + if (!I) { + Imm = SubImm; + if (!TII->isInlineConstant(*Op, OpTy) || + !TII->isOperandLegal(*UseMI, UseOpIdx, Op)) return false; - Imm = SubImm; continue; } if (Imm != SubImm) return false; // Can only fold splat constants } - if (!TII->isOperandLegal(*UseMI, UseOpIdx, Op)) - return false; - - FoldList.push_back(FoldCandidate(UseMI, UseOpIdx, Op)); + appendFoldCandidate(FoldList, UseMI, UseOpIdx, Defs[0].first); return true; } @@ -513,18 +575,6 @@ void SIFoldOperands::foldOperand( if (UseOp.isReg() && OpToFold.isReg()) { if (UseOp.isImplicit() || UseOp.getSubReg() != AMDGPU::NoSubRegister) return; - - // Don't fold subregister extracts into tied operands, only if it is a full - // copy since a subregister use tied to a full register def doesn't really - // make sense. e.g. don't fold: - // - // %1 = COPY %0:sub1 - // %2<tied3> = V_MAC_{F16, F32} %3, %4, %1<tied0> - // - // into - // %2<tied3> = V_MAC_{F16, F32} %3, %4, %0:sub1<tied0> - if (UseOp.isTied() && OpToFold.getSubReg() != AMDGPU::NoSubRegister) - return; } // Special case for REG_SEQUENCE: We can't fold literals into @@ -639,16 +689,97 @@ void SIFoldOperands::foldOperand( CopiesToReplace.push_back(UseMI); } else { if (UseMI->isCopy() && OpToFold.isReg() && - Register::isVirtualRegister(UseMI->getOperand(0).getReg()) && - TRI->isVectorRegister(*MRI, UseMI->getOperand(0).getReg()) && - TRI->isVectorRegister(*MRI, UseMI->getOperand(1).getReg()) && + UseMI->getOperand(0).getReg().isVirtual() && !UseMI->getOperand(1).getSubReg()) { + LLVM_DEBUG(dbgs() << "Folding " << OpToFold + << "\n into " << *UseMI << '\n'); unsigned Size = TII->getOpSize(*UseMI, 1); - UseMI->getOperand(1).setReg(OpToFold.getReg()); + Register UseReg = OpToFold.getReg(); + UseMI->getOperand(1).setReg(UseReg); UseMI->getOperand(1).setSubReg(OpToFold.getSubReg()); UseMI->getOperand(1).setIsKill(false); CopiesToReplace.push_back(UseMI); OpToFold.setIsKill(false); + + // That is very tricky to store a value into an AGPR. v_accvgpr_write_b32 + // can only accept VGPR or inline immediate. Recreate a reg_sequence with + // its initializers right here, so we will rematerialize immediates and + // avoid copies via different reg classes. + SmallVector<std::pair<MachineOperand*, unsigned>, 32> Defs; + if (Size > 4 && TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg()) && + getRegSeqInit(Defs, UseReg, AMDGPU::OPERAND_REG_INLINE_C_INT32, TII, + *MRI)) { + const DebugLoc &DL = UseMI->getDebugLoc(); + MachineBasicBlock &MBB = *UseMI->getParent(); + + UseMI->setDesc(TII->get(AMDGPU::REG_SEQUENCE)); + for (unsigned I = UseMI->getNumOperands() - 1; I > 0; --I) + UseMI->RemoveOperand(I); + + MachineInstrBuilder B(*MBB.getParent(), UseMI); + DenseMap<TargetInstrInfo::RegSubRegPair, Register> VGPRCopies; + SmallSetVector<TargetInstrInfo::RegSubRegPair, 32> SeenAGPRs; + for (unsigned I = 0; I < Size / 4; ++I) { + MachineOperand *Def = Defs[I].first; + TargetInstrInfo::RegSubRegPair CopyToVGPR; + if (Def->isImm() && + TII->isInlineConstant(*Def, AMDGPU::OPERAND_REG_INLINE_C_INT32)) { + int64_t Imm = Def->getImm(); + + auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); + BuildMI(MBB, UseMI, DL, + TII->get(AMDGPU::V_ACCVGPR_WRITE_B32), Tmp).addImm(Imm); + B.addReg(Tmp); + } else if (Def->isReg() && TRI->isAGPR(*MRI, Def->getReg())) { + auto Src = getRegSubRegPair(*Def); + Def->setIsKill(false); + if (!SeenAGPRs.insert(Src)) { + // We cannot build a reg_sequence out of the same registers, they + // must be copied. Better do it here before copyPhysReg() created + // several reads to do the AGPR->VGPR->AGPR copy. + CopyToVGPR = Src; + } else { + B.addReg(Src.Reg, Def->isUndef() ? RegState::Undef : 0, + Src.SubReg); + } + } else { + assert(Def->isReg()); + Def->setIsKill(false); + auto Src = getRegSubRegPair(*Def); + + // Direct copy from SGPR to AGPR is not possible. To avoid creation + // of exploded copies SGPR->VGPR->AGPR in the copyPhysReg() later, + // create a copy here and track if we already have such a copy. + if (TRI->isSGPRReg(*MRI, Src.Reg)) { + CopyToVGPR = Src; + } else { + auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); + BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Tmp).add(*Def); + B.addReg(Tmp); + } + } + + if (CopyToVGPR.Reg) { + Register Vgpr; + if (VGPRCopies.count(CopyToVGPR)) { + Vgpr = VGPRCopies[CopyToVGPR]; + } else { + Vgpr = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); + BuildMI(MBB, UseMI, DL, TII->get(AMDGPU::COPY), Vgpr).add(*Def); + VGPRCopies[CopyToVGPR] = Vgpr; + } + auto Tmp = MRI->createVirtualRegister(&AMDGPU::AGPR_32RegClass); + BuildMI(MBB, UseMI, DL, + TII->get(AMDGPU::V_ACCVGPR_WRITE_B32), Tmp).addReg(Vgpr); + B.addReg(Tmp); + } + + B.addImm(Defs[I].second); + } + LLVM_DEBUG(dbgs() << "Folded " << *UseMI << '\n'); + return; + } + if (Size != 4) return; if (TRI->isAGPR(*MRI, UseMI->getOperand(0).getReg()) && @@ -1099,6 +1230,7 @@ void SIFoldOperands::foldInstOperand(MachineInstr &MI, Copy->addImplicitDefUseOperands(*MF); for (FoldCandidate &Fold : FoldList) { + assert(!Fold.isReg() || Fold.OpToFold); if (Fold.isReg() && Register::isVirtualRegister(Fold.OpToFold->getReg())) { Register Reg = Fold.OpToFold->getReg(); MachineInstr *DefMI = Fold.OpToFold->getParent(); @@ -1249,8 +1381,8 @@ SIFoldOperands::isOMod(const MachineInstr &MI) const { case AMDGPU::V_MUL_F32_e64: case AMDGPU::V_MUL_F16_e64: { // If output denormals are enabled, omod is ignored. - if ((Op == AMDGPU::V_MUL_F32_e64 && ST->hasFP32Denormals()) || - (Op == AMDGPU::V_MUL_F16_e64 && ST->hasFP16Denormals())) + if ((Op == AMDGPU::V_MUL_F32_e64 && MFI->getMode().FP32Denormals) || + (Op == AMDGPU::V_MUL_F16_e64 && MFI->getMode().FP64FP16Denormals)) return std::make_pair(nullptr, SIOutMods::NONE); const MachineOperand *RegOp = nullptr; @@ -1279,8 +1411,8 @@ SIFoldOperands::isOMod(const MachineInstr &MI) const { case AMDGPU::V_ADD_F32_e64: case AMDGPU::V_ADD_F16_e64: { // If output denormals are enabled, omod is ignored. - if ((Op == AMDGPU::V_ADD_F32_e64 && ST->hasFP32Denormals()) || - (Op == AMDGPU::V_ADD_F16_e64 && ST->hasFP16Denormals())) + if ((Op == AMDGPU::V_ADD_F32_e64 && MFI->getMode().FP32Denormals) || + (Op == AMDGPU::V_ADD_F16_e64 && MFI->getMode().FP64FP16Denormals)) return std::make_pair(nullptr, SIOutMods::NONE); // Look through the DAGCombiner canonicalization fmul x, 2 -> fadd x, x @@ -1358,15 +1490,16 @@ bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) { tryFoldInst(TII, &MI); if (!TII->isFoldableCopy(MI)) { + // Saw an unknown clobber of m0, so we no longer know what it is. + if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI)) + CurrentKnownM0Val = nullptr; + // TODO: Omod might be OK if there is NSZ only on the source // instruction, and not the omod multiply. if (IsIEEEMode || (!HasNSZ && !MI.getFlag(MachineInstr::FmNsz)) || !tryFoldOMod(MI)) tryFoldClamp(MI); - // Saw an unknown clobber of m0, so we no longer know what it is. - if (CurrentKnownM0Val && MI.modifiesRegister(AMDGPU::M0, TRI)) - CurrentKnownM0Val = nullptr; continue; } @@ -1409,5 +1542,5 @@ bool SIFoldOperands::runOnMachineFunction(MachineFunction &MF) { foldInstOperand(MI, OpToFold); } } - return false; + return true; } diff --git a/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp b/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp index 26bae5734df7..8ef02e73865d 100644 --- a/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp +++ b/llvm/lib/Target/AMDGPU/SIFormMemoryClauses.cpp @@ -16,13 +16,14 @@ #include "AMDGPU.h" #include "AMDGPUSubtarget.h" #include "GCNRegPressure.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIInstrInfo.h" #include "SIMachineFunctionInfo.h" #include "SIRegisterInfo.h" -#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/DenseMap.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/InitializePasses.h" using namespace llvm; diff --git a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp index ed07ed100a19..8364665dda04 100644 --- a/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIFrameLowering.cpp @@ -578,10 +578,7 @@ void SIFrameLowering::emitEntryFunctionScratchSetup(const GCNSubtarget &ST, // We now have the GIT ptr - now get the scratch descriptor from the entry // at offset 0 (or offset 16 for a compute shader). - PointerType *PtrTy = - PointerType::get(Type::getInt64Ty(MF.getFunction().getContext()), - AMDGPUAS::CONSTANT_ADDRESS); - MachinePointerInfo PtrInfo(UndefValue::get(PtrTy)); + MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); const MCInstrDesc &LoadDwordX4 = TII->get(AMDGPU::S_LOAD_DWORDX4_IMM); auto MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad | @@ -623,10 +620,7 @@ void SIFrameLowering::emitEntryFunctionScratchSetup(const GCNSubtarget &ST, } else { const MCInstrDesc &LoadDwordX2 = TII->get(AMDGPU::S_LOAD_DWORDX2_IMM); - PointerType *PtrTy = - PointerType::get(Type::getInt64Ty(MF.getFunction().getContext()), - AMDGPUAS::CONSTANT_ADDRESS); - MachinePointerInfo PtrInfo(UndefValue::get(PtrTy)); + MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); auto MMO = MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 56ebf9c06741..e73d87cd66af 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -90,16 +90,21 @@ using namespace llvm; STATISTIC(NumTailCalls, "Number of tail calls"); -static cl::opt<bool> EnableVGPRIndexMode( - "amdgpu-vgpr-index-mode", - cl::desc("Use GPR indexing mode instead of movrel for vector indexing"), - cl::init(false)); - static cl::opt<bool> DisableLoopAlignment( "amdgpu-disable-loop-alignment", cl::desc("Do not align and prefetch loops"), cl::init(false)); +static bool hasFP32Denormals(const MachineFunction &MF) { + const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); + return Info->getMode().FP32Denormals; +} + +static bool hasFP64FP16Denormals(const MachineFunction &MF) { + const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); + return Info->getMode().FP64FP16Denormals; +} + static unsigned findFirstFreeSGPR(CCState &CCInfo) { unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs(); for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) { @@ -160,6 +165,13 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, computeRegisterProperties(Subtarget->getRegisterInfo()); + // The boolean content concept here is too inflexible. Compares only ever + // really produce a 1-bit result. Any copy/extend from these will turn into a + // select, and zext/1 or sext/-1 are equally cheap. Arbitrarily choose 0/1, as + // it's what most targets use. + setBooleanContents(ZeroOrOneBooleanContent); + setBooleanVectorContents(ZeroOrOneBooleanContent); + // We need to custom lower vector stores from local memory setOperationAction(ISD::LOAD, MVT::v2i32, Custom); setOperationAction(ISD::LOAD, MVT::v3i32, Custom); @@ -358,14 +370,16 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::DEBUGTRAP, MVT::Other, Custom); if (Subtarget->has16BitInsts()) { + setOperationAction(ISD::FPOW, MVT::f16, Promote); setOperationAction(ISD::FLOG, MVT::f16, Custom); setOperationAction(ISD::FEXP, MVT::f16, Custom); setOperationAction(ISD::FLOG10, MVT::f16, Custom); } - // v_mad_f32 does not support denormals according to some sources. - if (!Subtarget->hasFP32Denormals()) - setOperationAction(ISD::FMAD, MVT::f32, Legal); + // v_mad_f32 does not support denormals. We report it as unconditionally + // legal, and the context where it is formed will disallow it when fp32 + // denormals are enabled. + setOperationAction(ISD::FMAD, MVT::f32, Legal); if (!Subtarget->hasBFI()) { // fcopysign can be done in a single instruction with BFI. @@ -473,8 +487,6 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::FP_TO_SINT, MVT::i16, Promote); setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote); - setOperationAction(ISD::SINT_TO_FP, MVT::i16, Promote); - setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote); // F16 - Constant Actions. setOperationAction(ISD::ConstantFP, MVT::f16, Legal); @@ -489,6 +501,10 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::FP_ROUND, MVT::f16, Custom); setOperationAction(ISD::FCOS, MVT::f16, Promote); setOperationAction(ISD::FSIN, MVT::f16, Promote); + + setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom); + setOperationAction(ISD::UINT_TO_FP, MVT::i16, Custom); + setOperationAction(ISD::FP_TO_SINT, MVT::f16, Promote); setOperationAction(ISD::FP_TO_UINT, MVT::f16, Promote); setOperationAction(ISD::SINT_TO_FP, MVT::f16, Promote); @@ -503,7 +519,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, // F16 - VOP3 Actions. setOperationAction(ISD::FMA, MVT::f16, Legal); - if (!Subtarget->hasFP16Denormals() && STI.hasMadF16()) + if (STI.hasMadF16()) setOperationAction(ISD::FMAD, MVT::f16, Legal); for (MVT VT : {MVT::v2i16, MVT::v2f16, MVT::v4i16, MVT::v4f16}) { @@ -761,12 +777,13 @@ const GCNSubtarget *SITargetLowering::getSubtarget() const { // // There is only one special case when denormals are enabled we don't currently, // where this is OK to use. -bool SITargetLowering::isFPExtFoldable(unsigned Opcode, - EVT DestVT, EVT SrcVT) const { +bool SITargetLowering::isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, + EVT DestVT, EVT SrcVT) const { return ((Opcode == ISD::FMAD && Subtarget->hasMadMixInsts()) || (Opcode == ISD::FMA && Subtarget->hasFmaMixInsts())) && - DestVT.getScalarType() == MVT::f32 && !Subtarget->hasFP32Denormals() && - SrcVT.getScalarType() == MVT::f16; + DestVT.getScalarType() == MVT::f32 && + SrcVT.getScalarType() == MVT::f16 && + !hasFP32Denormals(DAG.getMachineFunction()); } bool SITargetLowering::isShuffleMaskLegal(ArrayRef<int>, EVT) const { @@ -1069,23 +1086,18 @@ bool SITargetLowering::isLegalFlatAddressingMode(const AddrMode &AM) const { return AM.BaseOffs == 0 && AM.Scale == 0; } - // GFX9 added a 13-bit signed offset. When using regular flat instructions, - // the sign bit is ignored and is treated as a 12-bit unsigned offset. - - // GFX10 shrinked signed offset to 12 bits. When using regular flat - // instructions, the sign bit is also ignored and is treated as 11-bit - // unsigned offset. - - if (Subtarget->getGeneration() >= AMDGPUSubtarget::GFX10) - return isUInt<11>(AM.BaseOffs) && AM.Scale == 0; - - // Just r + i - return isUInt<12>(AM.BaseOffs) && AM.Scale == 0; + return AM.Scale == 0 && + (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset( + AM.BaseOffs, AMDGPUAS::FLAT_ADDRESS, + /*Signed=*/false)); } bool SITargetLowering::isLegalGlobalAddressingMode(const AddrMode &AM) const { if (Subtarget->hasFlatGlobalInsts()) - return isInt<13>(AM.BaseOffs) && AM.Scale == 0; + return AM.Scale == 0 && + (AM.BaseOffs == 0 || Subtarget->getInstrInfo()->isLegalFLATOffset( + AM.BaseOffs, AMDGPUAS::GLOBAL_ADDRESS, + /*Signed=*/true)); if (!Subtarget->hasAddr64() || Subtarget->useFlatForGlobal()) { // Assume the we will use FLAT for all global memory accesses @@ -1326,13 +1338,6 @@ EVT SITargetLowering::getOptimalMemOpType( return MVT::Other; } -static bool isFlatGlobalAddrSpace(unsigned AS) { - return AS == AMDGPUAS::GLOBAL_ADDRESS || - AS == AMDGPUAS::FLAT_ADDRESS || - AS == AMDGPUAS::CONSTANT_ADDRESS || - AS > AMDGPUAS::MAX_AMDGPU_ADDRESS; -} - bool SITargetLowering::isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const { return isFlatGlobalAddrSpace(SrcAS) && isFlatGlobalAddrSpace(DestAS); @@ -1466,9 +1471,7 @@ SDValue SITargetLowering::lowerKernargMemParameter( const SDLoc &SL, SDValue Chain, uint64_t Offset, unsigned Align, bool Signed, const ISD::InputArg *Arg) const { - Type *Ty = MemVT.getTypeForEVT(*DAG.getContext()); - PointerType *PtrTy = PointerType::get(Ty, AMDGPUAS::CONSTANT_ADDRESS); - MachinePointerInfo PtrInfo(UndefValue::get(PtrTy)); + MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); // Try to avoid using an extload by loading earlier than the argument address, // and extracting the relevant bits. The load should hopefully be merged with @@ -2666,9 +2669,7 @@ bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { const Function *ParentFn = CI->getParent()->getParent(); if (AMDGPU::isEntryFunctionCC(ParentFn->getCallingConv())) return false; - - auto Attr = ParentFn->getFnAttribute("disable-tail-calls"); - return (Attr.getValueAsString() != "true"); + return true; } // The wave scratch offset register is used as the global base pointer. @@ -2787,10 +2788,9 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, MVT PtrVT = MVT::i32; // Walk the register/memloc assignments, inserting copies/loads. - for (unsigned i = 0, realArgIdx = 0, e = ArgLocs.size(); i != e; - ++i, ++realArgIdx) { + for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { CCValAssign &VA = ArgLocs[i]; - SDValue Arg = OutVals[realArgIdx]; + SDValue Arg = OutVals[i]; // Promote the value if needed. switch (VA.getLocInfo()) { @@ -2830,7 +2830,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, MaybeAlign Alignment; if (IsTailCall) { - ISD::ArgFlagsTy Flags = Outs[realArgIdx].Flags; + ISD::ArgFlagsTy Flags = Outs[i].Flags; unsigned OpSize = Flags.isByVal() ? Flags.getByValSize() : VA.getValVT().getStoreSize(); @@ -2868,8 +2868,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, Chain, DL, DstAddr, Arg, SizeNode, Outs[i].Flags.getByValAlign(), /*isVol = */ false, /*AlwaysInline = */ true, /*isTailCall = */ false, DstInfo, - MachinePointerInfo(UndefValue::get(Type::getInt8PtrTy( - *DAG.getContext(), AMDGPUAS::PRIVATE_ADDRESS)))); + MachinePointerInfo(AMDGPUAS::PRIVATE_ADDRESS)); MemOpChains.push_back(Cpy); } else { @@ -2986,7 +2985,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, IsThisReturn ? OutVals[0] : SDValue()); } -Register SITargetLowering::getRegisterByName(const char* RegName, EVT VT, +Register SITargetLowering::getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const { Register Reg = StringSwitch<Register>(RegName) .Case("m0", AMDGPU::M0) @@ -3411,7 +3410,7 @@ static MachineBasicBlock *emitIndirectSrc(MachineInstr &MI, std::tie(SubReg, Offset) = computeIndirectRegAndOffset(TRI, VecRC, SrcReg, Offset); - bool UseGPRIdxMode = ST.useVGPRIndexMode(EnableVGPRIndexMode); + const bool UseGPRIdxMode = ST.useVGPRIndexMode(); if (setM0ToIndexFromSGPR(TII, MRI, MI, Offset, UseGPRIdxMode, true)) { MachineBasicBlock::iterator I(&MI); @@ -3506,7 +3505,7 @@ static MachineBasicBlock *emitIndirectDst(MachineInstr &MI, std::tie(SubReg, Offset) = computeIndirectRegAndOffset(TRI, VecRC, SrcVec->getReg(), Offset); - bool UseGPRIdxMode = ST.useVGPRIndexMode(EnableVGPRIndexMode); + const bool UseGPRIdxMode = ST.useVGPRIndexMode(); if (Idx->getReg() == AMDGPU::NoRegister) { MachineBasicBlock::iterator I(&MI); @@ -3920,7 +3919,8 @@ MVT SITargetLowering::getScalarShiftAmountTy(const DataLayout &, EVT VT) const { // however does not support denormals, so we do report fma as faster if we have // a fast fma device and require denormals. // -bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { +bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, + EVT VT) const { VT = VT.getScalarType(); switch (VT.getSimpleVT().SimpleTy) { @@ -3929,7 +3929,7 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { // mad available which returns the same result as the separate operations // which we should prefer over fma. We can't use this if we want to support // denormals, so only report this in these cases. - if (Subtarget->hasFP32Denormals()) + if (hasFP32Denormals(MF)) return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts(); // If the subtarget has v_fmac_f32, that's just as good as v_mac_f32. @@ -3938,7 +3938,7 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { case MVT::f64: return true; case MVT::f16: - return Subtarget->has16BitInsts() && Subtarget->hasFP16Denormals(); + return Subtarget->has16BitInsts() && hasFP64FP16Denormals(MF); default: break; } @@ -3946,6 +3946,21 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(EVT VT) const { return false; } +bool SITargetLowering::isFMADLegalForFAddFSub(const SelectionDAG &DAG, + const SDNode *N) const { + // TODO: Check future ftz flag + // v_mad_f32/v_mac_f32 do not support denormals. + EVT VT = N->getValueType(0); + if (VT == MVT::f32) + return !hasFP32Denormals(DAG.getMachineFunction()); + if (VT == MVT::f16) { + return Subtarget->hasMadF16() && + !hasFP64FP16Denormals(DAG.getMachineFunction()); + } + + return false; +} + //===----------------------------------------------------------------------===// // Custom DAG Lowering Operations //===----------------------------------------------------------------------===// @@ -4416,8 +4431,8 @@ unsigned SITargetLowering::isCFIntrinsic(const SDNode *Intr) const { bool SITargetLowering::shouldEmitFixup(const GlobalValue *GV) const { const Triple &TT = getTargetMachine().getTargetTriple(); - return (GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || - GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && + return (GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || + GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && AMDGPU::shouldEmitConstantsToTextSection(TT); } @@ -4425,9 +4440,9 @@ bool SITargetLowering::shouldEmitGOTReloc(const GlobalValue *GV) const { // FIXME: Either avoid relying on address space here or change the default // address space for functions to avoid the explicit check. return (GV->getValueType()->isFunctionTy() || - GV->getType()->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS || - GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || - GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && + GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS || + GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || + GV->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) && !shouldEmitFixup(GV) && !getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV); } @@ -4694,10 +4709,7 @@ SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL, // TODO: Use custom target PseudoSourceValue. // TODO: We should use the value from the IR intrinsic call, but it might not // be available and how do we get it? - Value *V = UndefValue::get(PointerType::get(Type::getInt8Ty(*DAG.getContext()), - AMDGPUAS::CONSTANT_ADDRESS)); - - MachinePointerInfo PtrInfo(V, StructOffset); + MachinePointerInfo PtrInfo(AMDGPUAS::CONSTANT_ADDRESS); return DAG.getLoad(MVT::i32, DL, QueuePtr.getValue(1), Ptr, PtrInfo, MinAlign(64, StructOffset), MachineMemOperand::MODereferenceable | @@ -5646,11 +5658,16 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, SDValue Offset, SDValue GLC, SDValue DLC, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); + + const DataLayout &DataLayout = DAG.getDataLayout(); + unsigned Align = + DataLayout.getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext())); + MachineMemOperand *MMO = MF.getMachineMemOperand( MachinePointerInfo(), MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable | MachineMemOperand::MOInvariant, - VT.getStoreSize(), VT.getStoreSize()); + VT.getStoreSize(), Align); if (!Offset->isDivergent()) { SDValue Ops[] = { @@ -5659,6 +5676,20 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, GLC, DLC, }; + + // Widen vec3 load to vec4. + if (VT.isVector() && VT.getVectorNumElements() == 3) { + EVT WidenedVT = + EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(), 4); + auto WidenedOp = DAG.getMemIntrinsicNode( + AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(WidenedVT), Ops, WidenedVT, + MF.getMachineMemOperand(MMO, 0, WidenedVT.getStoreSize())); + auto Subvector = DAG.getNode( + ISD::EXTRACT_SUBVECTOR, DL, VT, WidenedOp, + DAG.getConstant(0, DL, getVectorIdxTy(DAG.getDataLayout()))); + return Subvector; + } + return DAG.getMemIntrinsicNode(AMDGPUISD::SBUFFER_LOAD, DL, DAG.getVTList(VT), Ops, VT, MMO); } @@ -5670,11 +5701,10 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, MVT LoadVT = VT.getSimpleVT(); unsigned NumElts = LoadVT.isVector() ? LoadVT.getVectorNumElements() : 1; assert((LoadVT.getScalarType() == MVT::i32 || - LoadVT.getScalarType() == MVT::f32) && - isPowerOf2_32(NumElts)); + LoadVT.getScalarType() == MVT::f32)); if (NumElts == 8 || NumElts == 16) { - NumLoads = NumElts == 16 ? 4 : 2; + NumLoads = NumElts / 4; LoadVT = MVT::v4i32; } @@ -5698,8 +5728,8 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, uint64_t InstOffset = cast<ConstantSDNode>(Ops[5])->getZExtValue(); for (unsigned i = 0; i < NumLoads; ++i) { Ops[5] = DAG.getTargetConstant(InstOffset + 16 * i, DL, MVT::i32); - Loads.push_back(DAG.getMemIntrinsicNode(AMDGPUISD::BUFFER_LOAD, DL, VTList, - Ops, LoadVT, MMO)); + Loads.push_back(getMemIntrinsicNode(AMDGPUISD::BUFFER_LOAD, DL, VTList, Ops, + LoadVT, MMO, DAG)); } if (VT == MVT::v8i32 || VT == MVT::v16i32) @@ -5918,22 +5948,6 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, return DAG.getNode(AMDGPUISD::INTERP_P1LL_F16, DL, MVT::f32, Ops); } } - case Intrinsic::amdgcn_interp_p2_f16: { - SDValue ToM0 = DAG.getCopyToReg(DAG.getEntryNode(), DL, AMDGPU::M0, - Op.getOperand(6), SDValue()); - SDValue Ops[] = { - Op.getOperand(2), // Src0 - Op.getOperand(3), // Attrchan - Op.getOperand(4), // Attr - DAG.getTargetConstant(0, DL, MVT::i32), // $src0_modifiers - Op.getOperand(1), // Src2 - DAG.getTargetConstant(0, DL, MVT::i32), // $src2_modifiers - Op.getOperand(5), // high - DAG.getTargetConstant(0, DL, MVT::i1), // $clamp - ToM0.getValue(1) - }; - return DAG.getNode(AMDGPUISD::INTERP_P2_F16, DL, MVT::f16, Ops); - } case Intrinsic::amdgcn_sin: return DAG.getNode(AMDGPUISD::SIN_HW, DL, VT, Op.getOperand(1)); @@ -7088,13 +7102,16 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, EVT VT = Op.getOperand(3).getValueType(); auto *M = cast<MemSDNode>(Op); - unsigned Opcode = VT.isVector() ? AMDGPUISD::ATOMIC_PK_FADD - : AMDGPUISD::ATOMIC_FADD; + if (VT.isVector()) { + return DAG.getMemIntrinsicNode( + AMDGPUISD::ATOMIC_PK_FADD, DL, Op->getVTList(), Ops, VT, + M->getMemOperand()); + } - return DAG.getMemIntrinsicNode(Opcode, DL, Op->getVTList(), Ops, VT, - M->getMemOperand()); + return DAG.getAtomic(ISD::ATOMIC_LOAD_FADD, DL, VT, + DAG.getVTList(VT, MVT::Other), Ops, + M->getMemOperand()).getValue(1); } - case Intrinsic::amdgcn_end_cf: return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other, Op->getOperand(2), Chain), 0); @@ -7451,8 +7468,11 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { // resource descriptor, we can only make private accesses up to a certain // size. switch (Subtarget->getMaxPrivateElementSize()) { - case 4: - return scalarizeVectorLoad(Load, DAG); + case 4: { + SDValue Ops[2]; + std::tie(Ops[0], Ops[1]) = scalarizeVectorLoad(Load, DAG); + return DAG.getMergeValues(Ops, DL); + } case 8: if (NumElements > 2) return SplitVectorLoad(Op, DAG); @@ -7529,7 +7549,7 @@ SDValue SITargetLowering::lowerFastUnsafeFDIV(SDValue Op, const SDNodeFlags Flags = Op->getFlags(); bool Unsafe = DAG.getTarget().Options.UnsafeFPMath || Flags.hasAllowReciprocal(); - if (!Unsafe && VT == MVT::f32 && Subtarget->hasFP32Denormals()) + if (!Unsafe && VT == MVT::f32 && hasFP32Denormals(DAG.getMachineFunction())) return SDValue(); if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(LHS)) { @@ -7672,7 +7692,7 @@ SDValue SITargetLowering::lowerFDIV_FAST(SDValue Op, SelectionDAG &DAG) const { static const SDValue getSPDenormModeValue(int SPDenormMode, SelectionDAG &DAG, const SDLoc &SL, const GCNSubtarget *ST) { assert(ST->hasDenormModeInst() && "Requires S_DENORM_MODE"); - int DPDenormModeDefault = ST->hasFP64Denormals() + int DPDenormModeDefault = hasFP64FP16Denormals(DAG.getMachineFunction()) ? FP_DENORM_FLUSH_NONE : FP_DENORM_FLUSH_IN_FLUSH_OUT; @@ -7708,7 +7728,9 @@ SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const { (1 << AMDGPU::Hwreg::WIDTH_M1_SHIFT_); const SDValue BitField = DAG.getTargetConstant(Denorm32Reg, SL, MVT::i16); - if (!Subtarget->hasFP32Denormals()) { + const bool HasFP32Denormals = hasFP32Denormals(DAG.getMachineFunction()); + + if (!HasFP32Denormals) { SDVTList BindParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); SDValue EnableDenorm; @@ -7752,8 +7774,7 @@ SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const { SDValue Fma4 = getFPTernOp(DAG, ISD::FMA, SL, MVT::f32, NegDivScale0, Fma3, NumeratorScaled, Fma3); - if (!Subtarget->hasFP32Denormals()) { - + if (!HasFP32Denormals) { SDValue DisableDenorm; if (Subtarget->hasDenormModeInst()) { const SDValue DisableDenormValue = @@ -8727,7 +8748,7 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, auto F = CFP->getValueAPF(); if (F.isNaN() && F.isSignaling()) return false; - return !F.isDenormal() || denormalsEnabledForType(Op.getValueType()); + return !F.isDenormal() || denormalsEnabledForType(DAG, Op.getValueType()); } // If source is a result of another standard FP operation it is already in @@ -8796,7 +8817,7 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, // snans will be quieted, so we only need to worry about denormals. if (Subtarget->supportsMinMaxDenormModes() || - denormalsEnabledForType(Op.getValueType())) + denormalsEnabledForType(DAG, Op.getValueType())) return true; // Flushing may be required. @@ -8868,7 +8889,7 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, LLVM_FALLTHROUGH; } default: - return denormalsEnabledForType(Op.getValueType()) && + return denormalsEnabledForType(DAG, Op.getValueType()) && DAG.isKnownNeverSNaN(Op); } @@ -8879,7 +8900,7 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, SDValue SITargetLowering::getCanonicalConstantFP( SelectionDAG &DAG, const SDLoc &SL, EVT VT, const APFloat &C) const { // Flush denormals to 0 if not enabled. - if (C.isDenormal() && !denormalsEnabledForType(VT)) + if (C.isDenormal() && !denormalsEnabledForType(DAG, VT)) return DAG.getConstantFP(0.0, SL, VT); if (C.isNaN()) { @@ -9417,8 +9438,8 @@ unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG, // Only do this if we are not trying to support denormals. v_mad_f32 does not // support denormals ever. - if (((VT == MVT::f32 && !Subtarget->hasFP32Denormals()) || - (VT == MVT::f16 && !Subtarget->hasFP16Denormals() && + if (((VT == MVT::f32 && !hasFP32Denormals(DAG.getMachineFunction())) || + (VT == MVT::f16 && !hasFP64FP16Denormals(DAG.getMachineFunction()) && getSubtarget()->hasMadF16())) && isOperationLegal(ISD::FMAD, VT)) return ISD::FMAD; @@ -9427,7 +9448,7 @@ unsigned SITargetLowering::getFusedOpcode(const SelectionDAG &DAG, if ((Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath || (N0->getFlags().hasAllowContract() && N1->getFlags().hasAllowContract())) && - isFMAFasterThanFMulAndFAdd(VT)) { + isFMAFasterThanFMulAndFAdd(DAG.getMachineFunction(), VT)) { return ISD::FMA; } @@ -9543,6 +9564,8 @@ SDValue SITargetLowering::performAddCombine(SDNode *N, case ISD::SIGN_EXTEND: case ISD::ANY_EXTEND: { auto Cond = RHS.getOperand(0); + // If this won't be a real VOPC output, we would still need to insert an + // extra instruction anyway. if (!isBoolSGPR(Cond)) break; SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); @@ -9573,6 +9596,26 @@ SDValue SITargetLowering::performSubCombine(SDNode *N, SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); + // sub x, zext (setcc) => subcarry x, 0, setcc + // sub x, sext (setcc) => addcarry x, 0, setcc + unsigned Opc = RHS.getOpcode(); + switch (Opc) { + default: break; + case ISD::ZERO_EXTEND: + case ISD::SIGN_EXTEND: + case ISD::ANY_EXTEND: { + auto Cond = RHS.getOperand(0); + // If this won't be a real VOPC output, we would still need to insert an + // extra instruction anyway. + if (!isBoolSGPR(Cond)) + break; + SDVTList VTList = DAG.getVTList(MVT::i32, MVT::i1); + SDValue Args[] = { LHS, DAG.getConstant(0, SL, MVT::i32), Cond }; + Opc = (Opc == ISD::SIGN_EXTEND) ? ISD::ADDCARRY : ISD::SUBCARRY; + return DAG.getNode(Opc, SL, VTList, Args); + } + } + if (LHS.getOpcode() == ISD::SUBCARRY) { // sub (subcarry x, 0, cc), y => subcarry x, y, cc auto C = dyn_cast<ConstantSDNode>(LHS.getOperand(1)); @@ -10884,14 +10927,14 @@ bool SITargetLowering::isSDNodeSourceOfDivergence(const SDNode * N, return false; } -bool SITargetLowering::denormalsEnabledForType(EVT VT) const { +bool SITargetLowering::denormalsEnabledForType(const SelectionDAG &DAG, + EVT VT) const { switch (VT.getScalarType().getSimpleVT().SimpleTy) { case MVT::f32: - return Subtarget->hasFP32Denormals(); + return hasFP32Denormals(DAG.getMachineFunction()); case MVT::f64: - return Subtarget->hasFP64Denormals(); case MVT::f16: - return Subtarget->hasFP16Denormals(); + return hasFP64FP16Denormals(DAG.getMachineFunction()); default: return false; } @@ -10930,6 +10973,12 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { // TODO: Do have these for flat. Older targets also had them for buffers. unsigned AS = RMW->getPointerAddressSpace(); + + if (AS == AMDGPUAS::GLOBAL_ADDRESS && Subtarget->hasAtomicFaddInsts()) { + return RMW->use_empty() ? AtomicExpansionKind::None : + AtomicExpansionKind::CmpXChg; + } + return (AS == AMDGPUAS::LOCAL_ADDRESS && Subtarget->hasLDSFPAtomics()) ? AtomicExpansionKind::None : AtomicExpansionKind::CmpXChg; } @@ -10956,6 +11005,8 @@ SITargetLowering::getRegClassFor(MVT VT, bool isDivergent) const { } static bool hasCFUser(const Value *V, SmallPtrSet<const Value *, 16> &Visited) { + if (!isa<Instruction>(V)) + return false; if (!Visited.insert(V).second) return false; bool Result = false; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index f0102feb65c4..d59495b052a4 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -222,7 +222,8 @@ public: const GCNSubtarget *getSubtarget() const; - bool isFPExtFoldable(unsigned Opcode, EVT DestVT, EVT SrcVT) const override; + bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode, EVT DestVT, + EVT SrcVT) const override; bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const override; @@ -260,6 +261,14 @@ public: bool isMemOpUniform(const SDNode *N) const; bool isMemOpHasNoClobberedMemOperand(const SDNode *N) const; + + static bool isFlatGlobalAddrSpace(unsigned AS) { + return AS == AMDGPUAS::GLOBAL_ADDRESS || + AS == AMDGPUAS::FLAT_ADDRESS || + AS == AMDGPUAS::CONSTANT_ADDRESS || + AS > AMDGPUAS::MAX_AMDGPU_ADDRESS; + } + bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override; bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override; @@ -321,7 +330,7 @@ public: SDValue LowerCall(CallLoweringInfo &CLI, SmallVectorImpl<SDValue> &InVals) const override; - Register getRegisterByName(const char* RegName, EVT VT, + Register getRegisterByName(const char* RegName, LLT VT, const MachineFunction &MF) const override; MachineBasicBlock *splitKillBlock(MachineInstr &MI, @@ -340,7 +349,11 @@ public: EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override; MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override; - bool isFMAFasterThanFMulAndFAdd(EVT VT) const override; + bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, + EVT VT) const override; + bool isFMADLegalForFAddFSub(const SelectionDAG &DAG, + const SDNode *N) const override; + SDValue splitUnaryVectorOp(SDValue Op, SelectionDAG &DAG) const; SDValue splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const; SDValue splitTernaryVectorOp(SDValue Op, SelectionDAG &DAG) const; @@ -380,7 +393,7 @@ public: bool isCanonicalized(SelectionDAG &DAG, SDValue Op, unsigned MaxDepth = 5) const; - bool denormalsEnabledForType(EVT VT) const; + bool denormalsEnabledForType(const SelectionDAG &DAG, EVT VT) const; bool isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, diff --git a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp index 87e63fcc4a04..80c044ec00cb 100644 --- a/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp +++ b/llvm/lib/Target/AMDGPU/SIInsertSkips.cpp @@ -41,7 +41,7 @@ using namespace llvm; #define DEBUG_TYPE "si-insert-skips" static cl::opt<unsigned> SkipThresholdFlag( - "amdgpu-skip-threshold", + "amdgpu-skip-threshold-legacy", cl::desc("Number of instructions before jumping over divergent control flow"), cl::init(12), cl::Hidden); @@ -466,6 +466,9 @@ bool SIInsertSkips::runOnMachineFunction(MachineFunction &MF) { MachineInstr &MI = *I; switch (MI.getOpcode()) { + case AMDGPU::S_CBRANCH_EXECZ: + ExecBranchStack.push_back(MI.getOperand(0).getMBB()); + break; case AMDGPU::SI_MASK_BRANCH: ExecBranchStack.push_back(MI.getOperand(0).getMBB()); MadeChange |= skipMaskBranch(MI, MBB); diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp index dcb04e426584..ef662d55cb0a 100644 --- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp +++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp @@ -42,7 +42,9 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachinePostDominators.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/InitializePasses.h" #include "llvm/IR/DebugLoc.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" @@ -372,7 +374,8 @@ private: AMDGPU::IsaVersion IV; DenseSet<MachineInstr *> TrackedWaitcntSet; - DenseSet<MachineInstr *> VCCZBugHandledSet; + DenseMap<const Value *, MachineBasicBlock *> SLoadAddresses; + MachinePostDominatorTree *PDT; struct BlockInfo { MachineBasicBlock *MBB; @@ -407,6 +410,7 @@ public: void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); + AU.addRequired<MachinePostDominatorTree>(); MachineFunctionPass::getAnalysisUsage(AU); } @@ -752,7 +756,10 @@ void WaitcntBrackets::determineWait(InstCounterType T, uint32_t ScoreToWait, // with a conservative value of 0 for the counter. addWait(Wait, T, 0); } else { - addWait(Wait, T, UB - ScoreToWait); + // If a counter has been maxed out avoid overflow by waiting for + // MAX(CounterType) - 1 instead. + uint32_t NeededWait = std::min(UB - ScoreToWait, getWaitCountMax(T) - 1); + addWait(Wait, T, NeededWait); } } } @@ -790,6 +797,7 @@ bool WaitcntBrackets::counterOutOfOrder(InstCounterType T) const { INITIALIZE_PASS_BEGIN(SIInsertWaitcnts, DEBUG_TYPE, "SI Insert Waitcnts", false, false) +INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree) INITIALIZE_PASS_END(SIInsertWaitcnts, DEBUG_TYPE, "SI Insert Waitcnts", false, false) @@ -939,19 +947,33 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore( } if (MI.isCall() && callWaitsOnFunctionEntry(MI)) { - // Don't bother waiting on anything except the call address. The function - // is going to insert a wait on everything in its prolog. This still needs - // to be careful if the call target is a load (e.g. a GOT load). + // The function is going to insert a wait on everything in its prolog. + // This still needs to be careful if the call target is a load (e.g. a GOT + // load). We also need to check WAW depenancy with saved PC. Wait = AMDGPU::Waitcnt(); int CallAddrOpIdx = AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::src0); - RegInterval Interval = ScoreBrackets.getRegInterval(&MI, TII, MRI, TRI, - CallAddrOpIdx, false); - for (signed RegNo = Interval.first; RegNo < Interval.second; ++RegNo) { + RegInterval CallAddrOpInterval = ScoreBrackets.getRegInterval( + &MI, TII, MRI, TRI, CallAddrOpIdx, false); + + for (signed RegNo = CallAddrOpInterval.first; + RegNo < CallAddrOpInterval.second; ++RegNo) ScoreBrackets.determineWait( LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait); + + int RtnAddrOpIdx = + AMDGPU::getNamedOperandIdx(MI.getOpcode(), AMDGPU::OpName::dst); + if (RtnAddrOpIdx != -1) { + RegInterval RtnAddrOpInterval = ScoreBrackets.getRegInterval( + &MI, TII, MRI, TRI, RtnAddrOpIdx, false); + + for (signed RegNo = RtnAddrOpInterval.first; + RegNo < RtnAddrOpInterval.second; ++RegNo) + ScoreBrackets.determineWait( + LGKM_CNT, ScoreBrackets.getRegScore(RegNo, LGKM_CNT), Wait); } + } else { // FIXME: Should not be relying on memoperands. // Look at the source operands of every instruction to see if @@ -996,6 +1018,13 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore( if (MI.mayStore()) { // FIXME: Should not be relying on memoperands. for (const MachineMemOperand *Memop : MI.memoperands()) { + const Value *Ptr = Memop->getValue(); + if (SLoadAddresses.count(Ptr)) { + addWait(Wait, LGKM_CNT, 0); + if (PDT->dominates(MI.getParent(), + SLoadAddresses.find(Ptr)->second)) + SLoadAddresses.erase(Ptr); + } unsigned AS = Memop->getAddrSpace(); if (AS != AMDGPUAS::LOCAL_ADDRESS) continue; @@ -1065,7 +1094,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore( assert(II->getOpcode() == AMDGPU::S_WAITCNT_VSCNT); assert(II->getOperand(0).getReg() == AMDGPU::SGPR_NULL); ScoreBrackets.applyWaitcnt( - AMDGPU::Waitcnt(0, 0, 0, II->getOperand(1).getImm())); + AMDGPU::Waitcnt(~0u, ~0u, ~0u, II->getOperand(1).getImm())); } } } @@ -1124,7 +1153,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore( Wait.VsCnt = ~0u; } - LLVM_DEBUG(dbgs() << "updateWaitcntInBlock\n" + LLVM_DEBUG(dbgs() << "generateWaitcntInstBefore\n" << "Old Instr: " << MI << '\n' << "New Instr: " << *II << '\n'); @@ -1141,7 +1170,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore( TrackedWaitcntSet.insert(SWaitInst); Modified = true; - LLVM_DEBUG(dbgs() << "insertWaitcntInBlock\n" + LLVM_DEBUG(dbgs() << "generateWaitcntInstBefore\n" << "Old Instr: " << MI << '\n' << "New Instr: " << *SWaitInst << '\n'); } @@ -1157,7 +1186,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore( TrackedWaitcntSet.insert(SWaitInst); Modified = true; - LLVM_DEBUG(dbgs() << "insertWaitcntInBlock\n" + LLVM_DEBUG(dbgs() << "generateWaitcntInstBefore\n" << "Old Instr: " << MI << '\n' << "New Instr: " << *SWaitInst << '\n'); } @@ -1195,7 +1224,7 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst, ScoreBrackets->updateByEvent(TII, TRI, MRI, LDS_ACCESS, Inst); } } else if (TII->isFLAT(Inst)) { - assert(Inst.mayLoad() || Inst.mayStore()); + assert(Inst.mayLoadOrStore()); if (TII->usesVM_CNT(Inst)) { if (!ST->hasVscnt()) @@ -1374,16 +1403,22 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF, } bool VCCZBugWorkAround = false; - if (readsVCCZ(Inst) && - (!VCCZBugHandledSet.count(&Inst))) { + if (readsVCCZ(Inst)) { if (ScoreBrackets.getScoreLB(LGKM_CNT) < ScoreBrackets.getScoreUB(LGKM_CNT) && ScoreBrackets.hasPendingEvent(SMEM_ACCESS)) { - if (ST->getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS) + if (ST->hasReadVCCZBug()) VCCZBugWorkAround = true; } } + if (TII->isSMRD(Inst)) { + for (const MachineMemOperand *Memop : Inst.memoperands()) { + const Value *Ptr = Memop->getValue(); + SLoadAddresses.insert(std::make_pair(Ptr, Inst.getParent())); + } + } + // Generate an s_waitcnt instruction to be placed before // cur_Inst, if needed. Modified |= generateWaitcntInstBefore(Inst, ScoreBrackets, OldWaitcntInstr); @@ -1417,7 +1452,6 @@ bool SIInsertWaitcnts::insertWaitcntInBlock(MachineFunction &MF, TII->get(ST->isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64), TRI->getVCC()) .addReg(TRI->getVCC()); - VCCZBugHandledSet.insert(&Inst); Modified = true; } @@ -1434,6 +1468,7 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) { MRI = &MF.getRegInfo(); IV = AMDGPU::getIsaVersion(ST->getCPU()); const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>(); + PDT = &getAnalysis<MachinePostDominatorTree>(); ForceEmitZeroWaitcnts = ForceEmitZeroFlag; for (auto T : inst_counter_types()) @@ -1457,7 +1492,6 @@ bool SIInsertWaitcnts::runOnMachineFunction(MachineFunction &MF) { RegisterEncoding.SGPR0 + HardwareLimits.NumSGPRsMax - 1; TrackedWaitcntSet.clear(); - VCCZBugHandledSet.clear(); RpotIdxMap.clear(); BlockInfos.clear(); diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index d97e6a62971b..d53950ca4465 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -85,7 +85,9 @@ BranchOffsetBits("amdgpu-s-branch-bits", cl::ReallyHidden, cl::init(16), SIInstrInfo::SIInstrInfo(const GCNSubtarget &ST) : AMDGPUGenInstrInfo(AMDGPU::ADJCALLSTACKUP, AMDGPU::ADJCALLSTACKDOWN), - RI(ST), ST(ST) {} + RI(ST), ST(ST) { + SchedModel.init(&ST); +} //===----------------------------------------------------------------------===// // TargetInstrInfo callbacks @@ -260,6 +262,9 @@ bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt, const MachineOperand *&BaseOp, int64_t &Offset, const TargetRegisterInfo *TRI) const { + if (!LdSt.mayLoadOrStore()) + return false; + unsigned Opc = LdSt.getOpcode(); if (isDS(LdSt)) { @@ -270,12 +275,11 @@ bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt, BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); // TODO: ds_consume/ds_append use M0 for the base address. Is it safe to // report that here? - if (!BaseOp) + if (!BaseOp || !BaseOp->isReg()) return false; Offset = OffsetImm->getImm(); - assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base " - "operands of type register."); + return true; } @@ -307,9 +311,11 @@ bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt, EltSize *= 64; BaseOp = getNamedOperand(LdSt, AMDGPU::OpName::addr); + if (!BaseOp->isReg()) + return false; + Offset = EltSize * Offset0; - assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base " - "operands of type register."); + return true; } @@ -346,12 +352,12 @@ bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt, getNamedOperand(LdSt, AMDGPU::OpName::offset); BaseOp = AddrReg; Offset = OffsetImm->getImm(); - if (SOffset) // soffset can be an inline immediate. Offset += SOffset->getImm(); - assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base " - "operands of type register."); + if (!BaseOp->isReg()) + return false; + return true; } @@ -364,8 +370,9 @@ bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt, const MachineOperand *SBaseReg = getNamedOperand(LdSt, AMDGPU::OpName::sbase); BaseOp = SBaseReg; Offset = OffsetImm->getImm(); - assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base " - "operands of type register."); + if (!BaseOp->isReg()) + return false; + return true; } @@ -383,8 +390,8 @@ bool SIInstrInfo::getMemOperandWithOffset(const MachineInstr &LdSt, } Offset = getNamedOperand(LdSt, AMDGPU::OpName::offset)->getImm(); - assert(BaseOp->isReg() && "getMemOperandWithOffset only supports base " - "operands of type register."); + if (!BaseOp->isReg()) + return false; return true; } @@ -418,7 +425,7 @@ static bool memOpsHaveSameBasePtr(const MachineInstr &MI1, const MachineFunction &MF = *MI1.getParent()->getParent(); const DataLayout &DL = MF.getFunction().getParent()->getDataLayout(); Base1 = GetUnderlyingObject(Base1, DL); - Base2 = GetUnderlyingObject(Base1, DL); + Base2 = GetUnderlyingObject(Base2, DL); if (isa<UndefValue>(Base1) || isa<UndefValue>(Base2)) return false; @@ -508,8 +515,8 @@ bool SIInstrInfo::shouldScheduleLoadsNear(SDNode *Load0, SDNode *Load1, static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, - const DebugLoc &DL, unsigned DestReg, - unsigned SrcReg, bool KillSrc) { + const DebugLoc &DL, MCRegister DestReg, + MCRegister SrcReg, bool KillSrc) { MachineFunction *MF = MBB.getParent(); DiagnosticInfoUnsupported IllegalCopy(MF->getFunction(), "illegal SGPR to VGPR copy", @@ -523,8 +530,8 @@ static void reportIllegalCopy(const SIInstrInfo *TII, MachineBasicBlock &MBB, void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, - const DebugLoc &DL, unsigned DestReg, - unsigned SrcReg, bool KillSrc) const { + const DebugLoc &DL, MCRegister DestReg, + MCRegister SrcReg, bool KillSrc) const { const TargetRegisterClass *RC = RI.getPhysRegClass(DestReg); if (RC == &AMDGPU::VGPR_32RegClass) { @@ -542,7 +549,7 @@ void SIInstrInfo::copyPhysReg(MachineBasicBlock &MBB, RC == &AMDGPU::SReg_32RegClass) { if (SrcReg == AMDGPU::SCC) { BuildMI(MBB, MI, DL, get(AMDGPU::S_CSELECT_B32), DestReg) - .addImm(-1) + .addImm(1) .addImm(0); return; } @@ -840,7 +847,7 @@ void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, Register SReg = MRI.createVirtualRegister(BoolXExecRC); BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64), SReg) - .addImm(-1) + .addImm(1) .addImm(0); BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) .addImm(0) @@ -855,7 +862,7 @@ void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64), SReg) .addImm(0) - .addImm(-1); + .addImm(1); BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) .addImm(0) .addReg(FalseReg) @@ -900,7 +907,7 @@ void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, .addImm(0); BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64), SReg) - .addImm(-1) + .addImm(1) .addImm(0); BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) .addImm(0) @@ -919,7 +926,7 @@ void SIInstrInfo::insertVectorSelect(MachineBasicBlock &MBB, BuildMI(MBB, I, DL, get(ST.isWave32() ? AMDGPU::S_CSELECT_B32 : AMDGPU::S_CSELECT_B64), SReg) .addImm(0) - .addImm(-1); + .addImm(1); BuildMI(MBB, I, DL, get(AMDGPU::V_CNDMASK_B32_e64), DstReg) .addImm(0) .addReg(FalseReg) @@ -1062,6 +1069,7 @@ void SIInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, if (RI.isSGPRClass(RC)) { MFI->setHasSpilledSGPRs(); + assert(SrcReg != AMDGPU::M0 && "m0 should not be spilled"); // We are only allowed to create one new instruction when spilling // registers, so we need to use pseudo instruction for spilling SGPRs. @@ -1190,6 +1198,7 @@ void SIInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, if (RI.isSGPRClass(RC)) { MFI->setHasSpilledSGPRs(); + assert(DestReg != AMDGPU::M0 && "m0 should not be reloaded into"); // FIXME: Maybe this should not include a memoperand because it will be // lowered to non-memory instructions. @@ -2542,9 +2551,9 @@ bool SIInstrInfo::checkInstOffsetsDoNotOverlap(const MachineInstr &MIa, bool SIInstrInfo::areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, const MachineInstr &MIb) const { - assert((MIa.mayLoad() || MIa.mayStore()) && + assert(MIa.mayLoadOrStore() && "MIa must load from or modify a memory location"); - assert((MIb.mayLoad() || MIb.mayStore()) && + assert(MIb.mayLoadOrStore() && "MIb must load from or modify a memory location"); if (MIa.hasUnmodeledSideEffects() || MIb.hasUnmodeledSideEffects()) @@ -3921,20 +3930,18 @@ bool SIInstrInfo::isLegalRegOperand(const MachineRegisterInfo &MRI, ? MRI.getRegClass(Reg) : RI.getPhysRegClass(Reg); - const SIRegisterInfo *TRI = - static_cast<const SIRegisterInfo*>(MRI.getTargetRegisterInfo()); - RC = TRI->getSubRegClass(RC, MO.getSubReg()); - - // In order to be legal, the common sub-class must be equal to the - // class of the current operand. For example: - // - // v_mov_b32 s0 ; Operand defined as vsrc_b32 - // ; RI.getCommonSubClass(s0,vsrc_b32) = sgpr ; LEGAL - // - // s_sendmsg 0, s0 ; Operand defined as m0reg - // ; RI.getCommonSubClass(s0,m0reg) = m0reg ; NOT LEGAL + const TargetRegisterClass *DRC = RI.getRegClass(OpInfo.RegClass); + if (MO.getSubReg()) { + const MachineFunction *MF = MO.getParent()->getParent()->getParent(); + const TargetRegisterClass *SuperRC = RI.getLargestLegalSuperClass(RC, *MF); + if (!SuperRC) + return false; - return RI.getCommonSubClass(RC, RI.getRegClass(OpInfo.RegClass)) == RC; + DRC = RI.getMatchingSuperRegClass(SuperRC, DRC, MO.getSubReg()); + if (!DRC) + return false; + } + return RC->hasSuperClassEq(DRC); } bool SIInstrInfo::isLegalVSrcOperand(const MachineRegisterInfo &MRI, @@ -4451,12 +4458,12 @@ static void loadSRsrcFromVGPR(const SIInstrInfo &TII, MachineInstr &MI, // Update dominators. We know that MBB immediately dominates LoopBB, that // LoopBB immediately dominates RemainderBB, and that RemainderBB immediately // dominates all of the successors transferred to it from MBB that MBB used - // to dominate. + // to properly dominate. if (MDT) { MDT->addNewBlock(LoopBB, &MBB); MDT->addNewBlock(RemainderBB, LoopBB); for (auto &Succ : RemainderBB->successors()) { - if (MDT->dominates(&MBB, Succ)) { + if (MDT->properlyDominates(&MBB, Succ)) { MDT->changeImmediateDominator(Succ, RemainderBB); } } @@ -6211,7 +6218,11 @@ MachineInstrBuilder SIInstrInfo::getAddNoCarry(MachineBasicBlock &MBB, if (ST.hasAddNoCarry()) return BuildMI(MBB, I, DL, get(AMDGPU::V_ADD_U32_e32), DestReg); - Register UnusedCarry = RS.scavengeRegister(RI.getBoolRC(), I, 0, false); + // If available, prefer to use vcc. + Register UnusedCarry = !RS.isRegUsed(AMDGPU::VCC) + ? Register(RI.getVCC()) + : RS.scavengeRegister(RI.getBoolRC(), I, 0, false); + // TODO: Users need to deal with this. if (!UnusedCarry.isValid()) return MachineInstrBuilder(); @@ -6329,6 +6340,26 @@ static SIEncodingFamily subtargetEncodingFamily(const GCNSubtarget &ST) { llvm_unreachable("Unknown subtarget generation!"); } +bool SIInstrInfo::isAsmOnlyOpcode(int MCOp) const { + switch(MCOp) { + // These opcodes use indirect register addressing so + // they need special handling by codegen (currently missing). + // Therefore it is too risky to allow these opcodes + // to be selected by dpp combiner or sdwa peepholer. + case AMDGPU::V_MOVRELS_B32_dpp_gfx10: + case AMDGPU::V_MOVRELS_B32_sdwa_gfx10: + case AMDGPU::V_MOVRELD_B32_dpp_gfx10: + case AMDGPU::V_MOVRELD_B32_sdwa_gfx10: + case AMDGPU::V_MOVRELSD_B32_dpp_gfx10: + case AMDGPU::V_MOVRELSD_B32_sdwa_gfx10: + case AMDGPU::V_MOVRELSD_2_B32_dpp_gfx10: + case AMDGPU::V_MOVRELSD_2_B32_sdwa_gfx10: + return true; + default: + return false; + } +} + int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { SIEncodingFamily Gen = subtargetEncodingFamily(ST); @@ -6367,6 +6398,9 @@ int SIInstrInfo::pseudoToMCOpcode(int Opcode) const { if (MCOp == (uint16_t)-1) return -1; + if (isAsmOnlyOpcode(MCOp)) + return -1; + return MCOp; } @@ -6541,14 +6575,14 @@ MachineInstr *SIInstrInfo::createPHIDestinationCopy( MachineInstr *SIInstrInfo::createPHISourceCopy( MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, - const DebugLoc &DL, Register Src, Register SrcSubReg, Register Dst) const { + const DebugLoc &DL, Register Src, unsigned SrcSubReg, Register Dst) const { if (InsPt != MBB.end() && (InsPt->getOpcode() == AMDGPU::SI_IF || InsPt->getOpcode() == AMDGPU::SI_ELSE || InsPt->getOpcode() == AMDGPU::SI_IF_BREAK) && InsPt->definesRegister(Src)) { InsPt++; - return BuildMI(MBB, InsPt, InsPt->getDebugLoc(), + return BuildMI(MBB, InsPt, DL, get(ST.isWave32() ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term), Dst) @@ -6560,3 +6594,53 @@ MachineInstr *SIInstrInfo::createPHISourceCopy( } bool llvm::SIInstrInfo::isWave32() const { return ST.isWave32(); } + +MachineInstr *SIInstrInfo::foldMemoryOperandImpl( + MachineFunction &MF, MachineInstr &MI, ArrayRef<unsigned> Ops, + MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS, + VirtRegMap *VRM) const { + // This is a bit of a hack (copied from AArch64). Consider this instruction: + // + // %0:sreg_32 = COPY $m0 + // + // We explicitly chose SReg_32 for the virtual register so such a copy might + // be eliminated by RegisterCoalescer. However, that may not be possible, and + // %0 may even spill. We can't spill $m0 normally (it would require copying to + // a numbered SGPR anyway), and since it is in the SReg_32 register class, + // TargetInstrInfo::foldMemoryOperand() is going to try. + // + // To prevent that, constrain the %0 register class here. + if (MI.isFullCopy()) { + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + + if (DstReg == AMDGPU::M0 && SrcReg.isVirtual()) { + MF.getRegInfo().constrainRegClass(SrcReg, &AMDGPU::SReg_32_XM0RegClass); + return nullptr; + } + + if (SrcReg == AMDGPU::M0 && DstReg.isVirtual()) { + MF.getRegInfo().constrainRegClass(DstReg, &AMDGPU::SReg_32_XM0RegClass); + return nullptr; + } + } + + return nullptr; +} + +unsigned SIInstrInfo::getInstrLatency(const InstrItineraryData *ItinData, + const MachineInstr &MI, + unsigned *PredCost) const { + if (MI.isBundle()) { + MachineBasicBlock::const_instr_iterator I(MI.getIterator()); + MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end()); + unsigned Lat = 0, Count = 0; + for (++I; I != E && I->isBundledWithPred(); ++I) { + ++Count; + Lat = std::max(Lat, SchedModel.computeInstrLatency(&*I)); + } + return Lat + Count - 1; + } + + return SchedModel.computeInstrLatency(&MI); +} diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index be463442c888..b151a94b0d11 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -25,6 +25,7 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/TargetSchedule.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/Support/Compiler.h" #include <cassert> @@ -46,6 +47,7 @@ class SIInstrInfo final : public AMDGPUGenInstrInfo { private: const SIRegisterInfo RI; const GCNSubtarget &ST; + TargetSchedModel SchedModel; // The inverse predicate should have the negative value. enum BranchPredicate { @@ -192,7 +194,7 @@ public: int64_t Offset1, unsigned NumLoads) const override; void copyPhysReg(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, - const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, + const DebugLoc &DL, MCRegister DestReg, MCRegister SrcReg, bool KillSrc) const override; unsigned calculateLDSSpillAddress(MachineBasicBlock &MBB, MachineInstr &MI, @@ -692,6 +694,10 @@ public: bool isInlineConstant(const APInt &Imm) const; + bool isInlineConstant(const APFloat &Imm) const { + return isInlineConstant(Imm.bitcastToAPInt()); + } + bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const; bool isInlineConstant(const MachineOperand &MO, @@ -977,7 +983,7 @@ public: MachineInstr *createPHISourceCopy(MachineBasicBlock &MBB, MachineBasicBlock::iterator InsPt, const DebugLoc &DL, Register Src, - Register SrcSubReg, + unsigned SrcSubReg, Register Dst) const override; bool isWave32() const; @@ -1017,6 +1023,10 @@ public: /// not exist. If Opcode is not a pseudo instruction, this is identity. int pseudoToMCOpcode(int Opcode) const; + /// \brief Check if this instruction should only be used by assembler. + /// Return true if this opcode should not be used by codegen. + bool isAsmOnlyOpcode(int MCOp) const; + const TargetRegisterClass *getRegClass(const MCInstrDesc &TID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) @@ -1027,6 +1037,17 @@ public: } void fixImplicitOperands(MachineInstr &MI) const; + + MachineInstr *foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, + ArrayRef<unsigned> Ops, + MachineBasicBlock::iterator InsertPt, + int FrameIndex, + LiveIntervals *LIS = nullptr, + VirtRegMap *VRM = nullptr) const override; + + unsigned getInstrLatency(const InstrItineraryData *ItinData, + const MachineInstr &MI, + unsigned *PredCost = nullptr) const override; }; /// \brief Returns true if a reg:subreg pair P has a TRC class diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index 1eecbf555613..85e8d0582dcd 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -225,7 +225,6 @@ class SDGlobalAtomicNoRtn<string opcode, ValueType ty> : SDNode <opcode, [SDNPMemOperand, SDNPHasChain, SDNPMayLoad, SDNPMayStore] >; -def SIglobal_atomic_fadd : SDGlobalAtomicNoRtn <"AMDGPUISD::ATOMIC_FADD", f32>; def SIglobal_atomic_pk_fadd : SDGlobalAtomicNoRtn <"AMDGPUISD::ATOMIC_PK_FADD", v2f16>; def SIpc_add_rel_offset : SDNode<"AMDGPUISD::PC_ADD_REL_OFFSET", @@ -324,7 +323,7 @@ defm atomic_load_fmax_#as : binary_atomic_op<SIatomic_fmax, 0>; def atomic_fadd_global_noret : PatFrag< (ops node:$ptr, node:$value), - (SIglobal_atomic_fadd node:$ptr, node:$value)> { + (atomic_load_fadd node:$ptr, node:$value)> { // FIXME: Move this let MemoryVT = f32; let IsAtomic = 1; @@ -580,46 +579,37 @@ def si_setcc_uniform : PatFrag < // SDNodes PatFrags for d16 loads //===----------------------------------------------------------------------===// -class LoadD16Frag <SDPatternOperator op> : PatFrag<(ops node:$ptr, node:$tied_in), (op node:$ptr, node:$tied_in)>; -class LocalLoadD16 <SDPatternOperator op> : LoadD16Frag <op>, LocalAddress; -class GlobalLoadD16 <SDPatternOperator op> : LoadD16Frag <op>, GlobalLoadAddress; -class PrivateLoadD16 <SDPatternOperator op> : LoadD16Frag <op>, PrivateAddress; -class FlatLoadD16 <SDPatternOperator op> : LoadD16Frag <op>, FlatLoadAddress; - -def load_d16_hi_local : LocalLoadD16 <SIload_d16_hi>; -def az_extloadi8_d16_hi_local : LocalLoadD16 <SIload_d16_hi_u8>; -def sextloadi8_d16_hi_local : LocalLoadD16 <SIload_d16_hi_i8>; - -def load_d16_hi_global : GlobalLoadD16 <SIload_d16_hi>; -def az_extloadi8_d16_hi_global : GlobalLoadD16 <SIload_d16_hi_u8>; -def sextloadi8_d16_hi_global : GlobalLoadD16 <SIload_d16_hi_i8>; - -def load_d16_hi_private : PrivateLoadD16 <SIload_d16_hi>; -def az_extloadi8_d16_hi_private : PrivateLoadD16 <SIload_d16_hi_u8>; -def sextloadi8_d16_hi_private : PrivateLoadD16 <SIload_d16_hi_i8>; +class LoadD16Frag <SDPatternOperator op> : PatFrag< + (ops node:$ptr, node:$tied_in), + (op node:$ptr, node:$tied_in)> { + let IsLoad = 1; +} -def load_d16_hi_flat : FlatLoadD16 <SIload_d16_hi>; -def az_extloadi8_d16_hi_flat : FlatLoadD16 <SIload_d16_hi_u8>; -def sextloadi8_d16_hi_flat : FlatLoadD16 <SIload_d16_hi_i8>; +foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in { +let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in { +def load_d16_hi_#as : LoadD16Frag <SIload_d16_hi>; -def load_d16_lo_local : LocalLoadD16 <SIload_d16_lo>; -def az_extloadi8_d16_lo_local : LocalLoadD16 <SIload_d16_lo_u8>; -def sextloadi8_d16_lo_local : LocalLoadD16 <SIload_d16_lo_i8>; +def az_extloadi8_d16_hi_#as : LoadD16Frag <SIload_d16_hi_u8> { + let MemoryVT = i8; +} -def load_d16_lo_global : GlobalLoadD16 <SIload_d16_lo>; -def az_extloadi8_d16_lo_global : GlobalLoadD16 <SIload_d16_lo_u8>; -def sextloadi8_d16_lo_global : GlobalLoadD16 <SIload_d16_lo_i8>; +def sextloadi8_d16_hi_#as : LoadD16Frag <SIload_d16_hi_i8> { + let MemoryVT = i8; +} -def load_d16_lo_private : PrivateLoadD16 <SIload_d16_lo>; -def az_extloadi8_d16_lo_private : PrivateLoadD16 <SIload_d16_lo_u8>; -def sextloadi8_d16_lo_private : PrivateLoadD16 <SIload_d16_lo_i8>; +def load_d16_lo_#as : LoadD16Frag <SIload_d16_lo>; -def load_d16_lo_flat : FlatLoadD16 <SIload_d16_lo>; -def az_extloadi8_d16_lo_flat : FlatLoadD16 <SIload_d16_lo_u8>; -def sextloadi8_d16_lo_flat : FlatLoadD16 <SIload_d16_lo_i8>; +def az_extloadi8_d16_lo_#as : LoadD16Frag <SIload_d16_lo_u8> { + let MemoryVT = i8; +} +def sextloadi8_d16_lo_#as : LoadD16Frag <SIload_d16_lo_i8> { + let MemoryVT = i8; +} +} // End let AddressSpaces = ... +} // End foreach AddrSpace def lshr_rev : PatFrag < (ops node:$src1, node:$src0), @@ -687,6 +677,10 @@ def as_i16imm : SDNodeXForm<imm, [{ return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i16); }]>; +def as_i16timm : SDNodeXForm<timm, [{ + return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i16); +}]>; + def as_i32imm: SDNodeXForm<imm, [{ return CurDAG->getTargetConstant(N->getSExtValue(), SDLoc(N), MVT::i32); }]>; @@ -738,14 +732,27 @@ def i64imm_32bit : ImmLeaf<i64, [{ return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm); }]>; -class InlineImm <ValueType vt> : PatLeaf <(vt imm), [{ - return isInlineImmediate(N); +def InlineImm16 : ImmLeaf<i16, [{ + return isInlineImmediate16(Imm); +}]>; + +def InlineImm32 : ImmLeaf<i32, [{ + return isInlineImmediate32(Imm); +}]>; + +def InlineImm64 : ImmLeaf<i64, [{ + return isInlineImmediate64(Imm); }]>; -class InlineFPImm <ValueType vt> : PatLeaf <(vt fpimm), [{ - return isInlineImmediate(N); +def InlineImmFP32 : FPImmLeaf<f32, [{ + return isInlineImmediate(Imm); }]>; +def InlineImmFP64 : FPImmLeaf<f64, [{ + return isInlineImmediate(Imm); +}]>; + + class VGPRImm <dag frag> : PatLeaf<frag, [{ return isVGPRImm(N); }]>; @@ -763,8 +770,8 @@ def NegSubInlineConst16 : ImmLeaf<i16, [{ return Imm < -16 && Imm >= -64; }], NegateImm>; -def ShiftAmt32Imm : PatLeaf <(imm), [{ - return N->getZExtValue() < 32; +def ShiftAmt32Imm : ImmLeaf <i32, [{ + return Imm < 32; }]>; def getNegV2I16Imm : SDNodeXForm<build_vector, [{ @@ -996,6 +1003,12 @@ class NamedOperandBit<string Name, AsmOperandClass MatchClass> : Operand<i1> { let ParserMatchClass = MatchClass; } +class NamedOperandBit_0<string Name, AsmOperandClass MatchClass> : + OperandWithDefaultOps<i1, (ops (i1 0))> { + let PrintMethod = "print"#Name; + let ParserMatchClass = MatchClass; +} + class NamedOperandU8<string Name, AsmOperandClass MatchClass> : Operand<i8> { let PrintMethod = "print"#Name; let ParserMatchClass = MatchClass; @@ -1011,6 +1024,12 @@ class NamedOperandU32<string Name, AsmOperandClass MatchClass> : Operand<i32> { let ParserMatchClass = MatchClass; } +class NamedOperandU32_0<string Name, AsmOperandClass MatchClass> : + OperandWithDefaultOps<i32, (ops (i32 0))> { + let PrintMethod = "print"#Name; + let ParserMatchClass = MatchClass; +} + class NamedOperandU32Default0<string Name, AsmOperandClass MatchClass> : OperandWithDefaultOps<i32, (ops (i32 0))> { let PrintMethod = "print"#Name; @@ -1031,7 +1050,13 @@ def offset1 : NamedOperandU8<"Offset1", NamedMatchClass<"Offset1">>; def gds : NamedOperandBit<"GDS", NamedMatchClass<"GDS">>; def omod : NamedOperandU32<"OModSI", NamedMatchClass<"OModSI">>; +def omod0 : NamedOperandU32_0<"OModSI", NamedMatchClass<"OModSI">>; + +// We need to make the cases with a default of 0 distinct from no +// default to help deal with some cases where the operand appears +// before a mandatory operand. def clampmod : NamedOperandBit<"ClampSI", NamedMatchClass<"ClampSI">>; +def clampmod0 : NamedOperandBit_0<"ClampSI", NamedMatchClass<"ClampSI">>; def highmod : NamedOperandBit<"High", NamedMatchClass<"High">>; def DLC : NamedOperandBit<"DLC", NamedMatchClass<"DLC">>; @@ -1245,7 +1270,6 @@ def MOVRELOffset : ComplexPattern<i32, 2, "SelectMOVRELOffset">; def VOP3Mods0 : ComplexPattern<untyped, 4, "SelectVOP3Mods0">; def VOP3Mods0Clamp : ComplexPattern<untyped, 3, "SelectVOP3Mods0Clamp">; -def VOP3Mods0Clamp0OMod : ComplexPattern<untyped, 4, "SelectVOP3Mods0Clamp0OMod">; def VOP3Mods : ComplexPattern<untyped, 2, "SelectVOP3Mods">; def VOP3NoMods : ComplexPattern<untyped, 1, "SelectVOP3NoMods">; // VOP3Mods, but the input source is known to never be NaN. @@ -1381,7 +1405,7 @@ multiclass EXP_m<bit done, SDPatternOperator node> { def _si : EXP_Helper<done>, SIMCInstr <"exp"#!if(done, "_done", ""), SIEncodingFamily.SI>, EXPe { - let AssemblerPredicates = [isGFX6GFX7]; + let AssemblerPredicate = isGFX6GFX7; let DecoderNamespace = "GFX6GFX7"; let DisableDecoder = DisableSIDecoder; } @@ -1389,7 +1413,7 @@ multiclass EXP_m<bit done, SDPatternOperator node> { def _vi : EXP_Helper<done>, SIMCInstr <"exp"#!if(done, "_done", ""), SIEncodingFamily.VI>, EXPe_vi { - let AssemblerPredicates = [isGFX8GFX9]; + let AssemblerPredicate = isGFX8GFX9; let DecoderNamespace = "GFX8"; let DisableDecoder = DisableVIDecoder; } @@ -1397,7 +1421,7 @@ multiclass EXP_m<bit done, SDPatternOperator node> { def _gfx10 : EXP_Helper<done>, SIMCInstr <"exp"#!if(done, "_done", ""), SIEncodingFamily.GFX10>, EXPe { - let AssemblerPredicates = [isGFX10Plus]; + let AssemblerPredicate = isGFX10Plus; let DecoderNamespace = "GFX10"; let DisableDecoder = DisableSIDecoder; } @@ -1587,11 +1611,11 @@ class getIns64 <RegisterOperand Src0RC, RegisterOperand Src1RC, !if (!eq(HasModifiers, 1), // VOP1 with modifiers (ins Src0Mod:$src0_modifiers, Src0RC:$src0, - clampmod:$clamp, omod:$omod) + clampmod0:$clamp, omod0:$omod) /* else */, // VOP1 without modifiers !if (!eq(HasIntClamp, 1), - (ins Src0RC:$src0, clampmod:$clamp), + (ins Src0RC:$src0, clampmod0:$clamp), (ins Src0RC:$src0)) /* endif */ ), !if (!eq(NumSrcArgs, 2), @@ -1600,14 +1624,14 @@ class getIns64 <RegisterOperand Src0RC, RegisterOperand Src1RC, !if( !eq(HasOMod, 1), (ins Src0Mod:$src0_modifiers, Src0RC:$src0, Src1Mod:$src1_modifiers, Src1RC:$src1, - clampmod:$clamp, omod:$omod), + clampmod0:$clamp, omod0:$omod), (ins Src0Mod:$src0_modifiers, Src0RC:$src0, Src1Mod:$src1_modifiers, Src1RC:$src1, - clampmod:$clamp)) + clampmod0:$clamp)) /* else */, // VOP2 without modifiers !if (!eq(HasIntClamp, 1), - (ins Src0RC:$src0, Src1RC:$src1, clampmod:$clamp), + (ins Src0RC:$src0, Src1RC:$src1, clampmod0:$clamp), (ins Src0RC:$src0, Src1RC:$src1)) /* endif */ ) @@ -1619,12 +1643,12 @@ class getIns64 <RegisterOperand Src0RC, RegisterOperand Src1RC, (ins Src0Mod:$src0_modifiers, Src0RC:$src0, Src1Mod:$src1_modifiers, Src1RC:$src1, Src2Mod:$src2_modifiers, Src2RC:$src2, - clampmod:$clamp, omod:$omod), + clampmod0:$clamp, omod0:$omod), !if (!eq(HasIntClamp, 1), (ins Src0Mod:$src0_modifiers, Src0RC:$src0, Src1Mod:$src1_modifiers, Src1RC:$src1, Src2Mod:$src2_modifiers, Src2RC:$src2, - clampmod:$clamp), + clampmod0:$clamp), (ins Src0Mod:$src0_modifiers, Src0RC:$src0, Src1Mod:$src1_modifiers, Src1RC:$src1, Src2Mod:$src2_modifiers, Src2RC:$src2))), @@ -1632,18 +1656,18 @@ class getIns64 <RegisterOperand Src0RC, RegisterOperand Src1RC, !if (!eq(HasOMod, 1), (ins Src0Mod:$src0_modifiers, Src0RC:$src0, Src1Mod:$src1_modifiers, Src1RC:$src1, - Src2RC:$src2, clampmod:$clamp, omod:$omod), + Src2RC:$src2, clampmod0:$clamp, omod0:$omod), !if (!eq(HasIntClamp, 1), (ins Src0Mod:$src0_modifiers, Src0RC:$src0, Src1Mod:$src1_modifiers, Src1RC:$src1, - Src2RC:$src2, clampmod:$clamp), + Src2RC:$src2, clampmod0:$clamp), (ins Src0Mod:$src0_modifiers, Src0RC:$src0, Src1Mod:$src1_modifiers, Src1RC:$src1, Src2RC:$src2)))) /* else */, // VOP3 without modifiers !if (!eq(HasIntClamp, 1), - (ins Src0RC:$src0, Src1RC:$src1, Src2RC:$src2, clampmod:$clamp), + (ins Src0RC:$src0, Src1RC:$src1, Src2RC:$src2, clampmod0:$clamp), (ins Src0RC:$src0, Src1RC:$src1, Src2RC:$src2)) /* endif */ )))); } diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td index 21984c6ad910..d84720f820ee 100644 --- a/llvm/lib/Target/AMDGPU/SIInstructions.td +++ b/llvm/lib/Target/AMDGPU/SIInstructions.td @@ -86,7 +86,7 @@ defm V_INTERP_MOV_F32 : VINTRP_m < //===----------------------------------------------------------------------===// def ATOMIC_FENCE : SPseudoInstSI< (outs), (ins i32imm:$ordering, i32imm:$scope), - [(atomic_fence (i32 imm:$ordering), (i32 imm:$scope))], + [(atomic_fence (i32 timm:$ordering), (i32 timm:$scope))], "ATOMIC_FENCE $ordering, $scope"> { let hasSideEffects = 1; let maybeAtomic = 1; @@ -646,22 +646,22 @@ def : Pat < def : Pat < (int_amdgcn_kill i1:$src), - (SI_KILL_I1_PSEUDO $src, 0) + (SI_KILL_I1_PSEUDO SCSrc_i1:$src, 0) >; def : Pat < (int_amdgcn_kill (i1 (not i1:$src))), - (SI_KILL_I1_PSEUDO $src, -1) + (SI_KILL_I1_PSEUDO SCSrc_i1:$src, -1) >; def : Pat < (AMDGPUkill i32:$src), - (SI_KILL_F32_COND_IMM_PSEUDO $src, 0, 3) // 3 means SETOGE + (SI_KILL_F32_COND_IMM_PSEUDO VSrc_b32:$src, 0, 3) // 3 means SETOGE >; def : Pat < - (int_amdgcn_kill (i1 (setcc f32:$src, InlineFPImm<f32>:$imm, cond:$cond))), - (SI_KILL_F32_COND_IMM_PSEUDO $src, (bitcast_fpimm_to_i32 $imm), (cond_as_i32imm $cond)) + (int_amdgcn_kill (i1 (setcc f32:$src, InlineImmFP32:$imm, cond:$cond))), + (SI_KILL_F32_COND_IMM_PSEUDO VSrc_b32:$src, (bitcast_fpimm_to_i32 $imm), (cond_as_i32imm $cond)) >; // TODO: we could add more variants for other types of conditionals @@ -782,13 +782,14 @@ defm : FMADPat <f32, V_MAC_F32_e64>; class FMADModsPat<Instruction inst, SDPatternOperator mad_opr, ValueType Ty> : GCNPat< - (Ty (mad_opr (VOP3Mods Ty:$src0, i32:$src0_mod), - (VOP3Mods Ty:$src1, i32:$src1_mod), - (VOP3Mods Ty:$src2, i32:$src2_mod))), + (Ty (mad_opr (Ty (VOP3Mods Ty:$src0, i32:$src0_mod)), + (Ty (VOP3Mods Ty:$src1, i32:$src1_mod)), + (Ty (VOP3Mods Ty:$src2, i32:$src2_mod)))), (inst $src0_mod, $src0, $src1_mod, $src1, $src2_mod, $src2, DSTCLAMP.NONE, DSTOMOD.NONE) >; +// FIXME: This should select to V_MAC_F32 def : FMADModsPat<V_MAD_F32, AMDGPUfmad_ftz, f32>; def : FMADModsPat<V_MAD_F16, AMDGPUfmad_ftz, f16> { let SubtargetPredicate = Has16BitInsts; @@ -1323,8 +1324,8 @@ def : GCNPat < >; def : GCNPat < - (i64 InlineImm<i64>:$imm), - (S_MOV_B64 InlineImm<i64>:$imm) + (i64 InlineImm64:$imm), + (S_MOV_B64 InlineImm64:$imm) >; // XXX - Should this use a s_cmp to set SCC? @@ -1345,14 +1346,15 @@ def : GCNPat < } def : GCNPat < - (f64 InlineFPImm<f64>:$imm), - (S_MOV_B64 (f64 (bitcast_fpimm_to_i64 InlineFPImm<f64>:$imm))) + (f64 InlineImmFP64:$imm), + (S_MOV_B64 (f64 (bitcast_fpimm_to_i64 InlineImmFP64:$imm))) >; /********** ================== **********/ /********** Intrinsic Patterns **********/ /********** ================== **********/ +// FIXME: Should use _e64 and select source modifiers. def : POW_Common <V_LOG_F32_e32, V_EXP_F32_e32, V_MUL_LEGACY_F32_e32>; def : GCNPat < @@ -1792,54 +1794,59 @@ def : ExpPattern<AMDGPUexport_done, i32, EXP_DONE>; // COPY is workaround tablegen bug from multiple outputs // from S_LSHL_B32's multiple outputs from implicit scc def. def : GCNPat < - (v2i16 (build_vector (i16 0), i16:$src1)), - (v2i16 (COPY (S_LSHL_B32 i16:$src1, (i16 16)))) + (v2i16 (build_vector (i16 0), (i16 SReg_32:$src1))), + (S_LSHL_B32 SReg_32:$src1, (i16 16)) >; def : GCNPat < - (v2i16 (build_vector i16:$src0, (i16 undef))), - (v2i16 (COPY $src0)) + (v2i16 (build_vector (i16 SReg_32:$src0), (i16 undef))), + (COPY_TO_REGCLASS SReg_32:$src0, SReg_32) +>; + +def : GCNPat < + (v2i16 (build_vector (i16 VGPR_32:$src0), (i16 undef))), + (COPY_TO_REGCLASS VGPR_32:$src0, VGPR_32) >; def : GCNPat < (v2f16 (build_vector f16:$src0, (f16 undef))), - (v2f16 (COPY $src0)) + (COPY $src0) >; def : GCNPat < - (v2i16 (build_vector (i16 undef), i16:$src1)), - (v2i16 (COPY (S_LSHL_B32 $src1, (i32 16)))) + (v2i16 (build_vector (i16 undef), (i16 SReg_32:$src1))), + (S_LSHL_B32 SReg_32:$src1, (i32 16)) >; def : GCNPat < - (v2f16 (build_vector (f16 undef), f16:$src1)), - (v2f16 (COPY (S_LSHL_B32 $src1, (i32 16)))) + (v2f16 (build_vector (f16 undef), (f16 SReg_32:$src1))), + (S_LSHL_B32 SReg_32:$src1, (i32 16)) >; let SubtargetPredicate = HasVOP3PInsts in { def : GCNPat < - (v2i16 (build_vector i16:$src0, i16:$src1)), - (v2i16 (S_PACK_LL_B32_B16 $src0, $src1)) + (v2i16 (build_vector (i16 SReg_32:$src0), (i16 SReg_32:$src1))), + (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1) >; // With multiple uses of the shift, this will duplicate the shift and // increase register pressure. def : GCNPat < - (v2i16 (build_vector i16:$src0, (i16 (trunc (srl_oneuse i32:$src1, (i32 16)))))), - (v2i16 (S_PACK_LH_B32_B16 i16:$src0, i32:$src1)) + (v2i16 (build_vector (i16 SReg_32:$src0), (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), + (v2i16 (S_PACK_LH_B32_B16 SReg_32:$src0, SReg_32:$src1)) >; def : GCNPat < - (v2i16 (build_vector (i16 (trunc (srl_oneuse i32:$src0, (i32 16)))), - (i16 (trunc (srl_oneuse i32:$src1, (i32 16)))))), - (v2i16 (S_PACK_HH_B32_B16 $src0, $src1)) + (v2i16 (build_vector (i16 (trunc (srl_oneuse SReg_32:$src0, (i32 16)))), + (i16 (trunc (srl_oneuse SReg_32:$src1, (i32 16)))))), + (S_PACK_HH_B32_B16 SReg_32:$src0, SReg_32:$src1) >; // TODO: Should source modifiers be matched to v_pack_b32_f16? def : GCNPat < - (v2f16 (build_vector f16:$src0, f16:$src1)), - (v2f16 (S_PACK_LL_B32_B16 $src0, $src1)) + (v2f16 (build_vector (f16 SReg_32:$src0), (f16 SReg_32:$src1))), + (S_PACK_LL_B32_B16 SReg_32:$src0, SReg_32:$src1) >; } // End SubtargetPredicate = HasVOP3PInsts @@ -1923,15 +1930,28 @@ def : GCNPat < // TODO: Also do for 64-bit. def : GCNPat< (add i32:$src0, (i32 NegSubInlineConst32:$src1)), - (S_SUB_I32 $src0, NegSubInlineConst32:$src1) + (S_SUB_I32 SReg_32:$src0, NegSubInlineConst32:$src1) >; +def : GCNPat< + (add i32:$src0, (i32 NegSubInlineConst32:$src1)), + (V_SUB_U32_e64 VS_32:$src0, NegSubInlineConst32:$src1)> { + let SubtargetPredicate = HasAddNoCarryInsts; +} + +def : GCNPat< + (add i32:$src0, (i32 NegSubInlineConst32:$src1)), + (V_SUB_I32_e64 VS_32:$src0, NegSubInlineConst32:$src1)> { + let SubtargetPredicate = NotHasAddNoCarryInsts; +} + + // Avoid pointlessly materializing a constant in VGPR. // FIXME: Should also do this for readlane, but tablegen crashes on // the ignored src1. def : GCNPat< (int_amdgcn_readfirstlane (i32 imm:$src)), - (S_MOV_B32 $src) + (S_MOV_B32 SReg_32:$src) >; multiclass BFMPatterns <ValueType vt, InstSI BFM, InstSI MOV> { @@ -1952,6 +1972,29 @@ defm : BFMPatterns <i32, S_BFM_B32, S_MOV_B32>; defm : BFEPattern <V_BFE_U32, V_BFE_I32, S_MOV_B32>; defm : SHA256MaPattern <V_BFI_B32, V_XOR_B32_e64, SReg_64>; +multiclass IntMed3Pat<Instruction med3Inst, + SDPatternOperator min, + SDPatternOperator max, + SDPatternOperator min_oneuse, + SDPatternOperator max_oneuse> { + + // This matches 16 permutations of + // min(max(a, b), max(min(a, b), c)) + def : AMDGPUPat < + (min (max_oneuse i32:$src0, i32:$src1), + (max_oneuse (min_oneuse i32:$src0, i32:$src1), i32:$src2)), + (med3Inst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2) +>; + + // This matches 16 permutations of + // max(min(x, y), min(max(x, y), z)) + def : AMDGPUPat < + (max (min_oneuse i32:$src0, i32:$src1), + (min_oneuse (max_oneuse i32:$src0, i32:$src1), i32:$src2)), + (med3Inst VSrc_b32:$src0, VSrc_b32:$src1, VSrc_b32:$src2) +>; +} + defm : IntMed3Pat<V_MED3_I32, smin, smax, smin_oneuse, smax_oneuse>; defm : IntMed3Pat<V_MED3_U32, umin, umax, umin_oneuse, umax_oneuse>; @@ -1982,22 +2025,21 @@ multiclass Int16Med3Pat<Instruction med3Inst, SDPatternOperator min, SDPatternOperator max, SDPatternOperator max_oneuse, - SDPatternOperator min_oneuse, - ValueType vt = i16> { + SDPatternOperator min_oneuse> { // This matches 16 permutations of // max(min(x, y), min(max(x, y), z)) def : GCNPat < - (max (min_oneuse vt:$src0, vt:$src1), - (min_oneuse (max_oneuse vt:$src0, vt:$src1), vt:$src2)), - (med3Inst SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, SRCMODS.NONE, $src2, DSTCLAMP.NONE) + (max (min_oneuse i16:$src0, i16:$src1), + (min_oneuse (max_oneuse i16:$src0, i16:$src1), i16:$src2)), + (med3Inst SRCMODS.NONE, VSrc_b16:$src0, SRCMODS.NONE, VSrc_b16:$src1, SRCMODS.NONE, VSrc_b16:$src2, DSTCLAMP.NONE) >; // This matches 16 permutations of // min(max(a, b), max(min(a, b), c)) def : GCNPat < - (min (max_oneuse vt:$src0, vt:$src1), - (max_oneuse (min_oneuse vt:$src0, vt:$src1), vt:$src2)), - (med3Inst SRCMODS.NONE, $src0, SRCMODS.NONE, $src1, SRCMODS.NONE, $src2, DSTCLAMP.NONE) + (min (max_oneuse i16:$src0, i16:$src1), + (max_oneuse (min_oneuse i16:$src0, i16:$src1), i16:$src2)), + (med3Inst SRCMODS.NONE, VSrc_b16:$src0, SRCMODS.NONE, VSrc_b16:$src1, SRCMODS.NONE, VSrc_b16:$src2, DSTCLAMP.NONE) >; } @@ -2018,3 +2060,14 @@ def G_AMDGPU_FFBH_U32 : AMDGPUGenericInstruction { let InOperandList = (ins type1:$src); let hasSideEffects = 0; } + +// Atomic cmpxchg. $cmpval ad $newval are packed in a single vector +// operand Expects a MachineMemOperand in addition to explicit +// operands. +def G_AMDGPU_ATOMIC_CMPXCHG : AMDGPUGenericInstruction { + let OutOperandList = (outs type0:$oldval); + let InOperandList = (ins ptype1:$addr, type0:$cmpval_nnenwval); + let hasSideEffects = 0; + let mayLoad = 1; + let mayStore = 1; +} diff --git a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp index 20db1c37f354..d2b1abc8a9fb 100644 --- a/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp +++ b/llvm/lib/Target/AMDGPU/SILoadStoreOptimizer.cpp @@ -75,6 +75,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/IR/DebugLoc.h" +#include "llvm/InitializePasses.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" @@ -98,6 +99,8 @@ enum InstClassEnum { BUFFER_LOAD, BUFFER_STORE, MIMG, + TBUFFER_LOAD, + TBUFFER_STORE, }; enum RegisterEnum { @@ -112,22 +115,16 @@ enum RegisterEnum { class SILoadStoreOptimizer : public MachineFunctionPass { struct CombineInfo { MachineBasicBlock::iterator I; - MachineBasicBlock::iterator Paired; unsigned EltSize; - unsigned Offset0; - unsigned Offset1; - unsigned Width0; - unsigned Width1; + unsigned Offset; + unsigned Width; + unsigned Format; unsigned BaseOff; - unsigned DMask0; - unsigned DMask1; + unsigned DMask; InstClassEnum InstClass; - bool GLC0; - bool GLC1; - bool SLC0; - bool SLC1; - bool DLC0; - bool DLC1; + bool GLC; + bool SLC; + bool DLC; bool UseST64; SmallVector<MachineInstr *, 8> InstsToMove; int AddrIdx[5]; @@ -183,7 +180,6 @@ class SILoadStoreOptimizer : public MachineFunctionPass { void setMI(MachineBasicBlock::iterator MI, const SIInstrInfo &TII, const GCNSubtarget &STM); - void setPaired(MachineBasicBlock::iterator MI, const SIInstrInfo &TII); }; struct BaseRegisters { @@ -205,30 +201,39 @@ private: const GCNSubtarget *STM = nullptr; const SIInstrInfo *TII = nullptr; const SIRegisterInfo *TRI = nullptr; + const MCSubtargetInfo *STI = nullptr; MachineRegisterInfo *MRI = nullptr; AliasAnalysis *AA = nullptr; bool OptimizeAgain; - static bool dmasksCanBeCombined(const CombineInfo &CI, const SIInstrInfo &TII); - static bool offsetsCanBeCombined(CombineInfo &CI); - static bool widthsFit(const GCNSubtarget &STM, const CombineInfo &CI); - static unsigned getNewOpcode(const CombineInfo &CI); - static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI); - const TargetRegisterClass *getTargetRegisterClass(const CombineInfo &CI); + static bool dmasksCanBeCombined(const CombineInfo &CI, + const SIInstrInfo &TII, + const CombineInfo &Paired); + static bool offsetsCanBeCombined(CombineInfo &CI, const MCSubtargetInfo &STI, + CombineInfo &Paired); + static bool widthsFit(const GCNSubtarget &STM, const CombineInfo &CI, + const CombineInfo &Paired); + static unsigned getNewOpcode(const CombineInfo &CI, const CombineInfo &Paired); + static std::pair<unsigned, unsigned> getSubRegIdxs(const CombineInfo &CI, + const CombineInfo &Paired); + const TargetRegisterClass *getTargetRegisterClass(const CombineInfo &CI, + const CombineInfo &Paired); - bool findMatchingInst(CombineInfo &CI); + bool findMatchingInst(CombineInfo &CI, CombineInfo &Paired); unsigned read2Opcode(unsigned EltSize) const; unsigned read2ST64Opcode(unsigned EltSize) const; - MachineBasicBlock::iterator mergeRead2Pair(CombineInfo &CI); + MachineBasicBlock::iterator mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired); unsigned write2Opcode(unsigned EltSize) const; unsigned write2ST64Opcode(unsigned EltSize) const; - MachineBasicBlock::iterator mergeWrite2Pair(CombineInfo &CI); - MachineBasicBlock::iterator mergeImagePair(CombineInfo &CI); - MachineBasicBlock::iterator mergeSBufferLoadImmPair(CombineInfo &CI); - MachineBasicBlock::iterator mergeBufferLoadPair(CombineInfo &CI); - MachineBasicBlock::iterator mergeBufferStorePair(CombineInfo &CI); + MachineBasicBlock::iterator mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired); + MachineBasicBlock::iterator mergeImagePair(CombineInfo &CI, CombineInfo &Paired); + MachineBasicBlock::iterator mergeSBufferLoadImmPair(CombineInfo &CI, CombineInfo &Paired); + MachineBasicBlock::iterator mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired); + MachineBasicBlock::iterator mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired); + MachineBasicBlock::iterator mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired); + MachineBasicBlock::iterator mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired); void updateBaseAndOffset(MachineInstr &I, unsigned NewBase, int32_t NewOffset) const; @@ -284,6 +289,9 @@ static unsigned getOpcodeWidth(const MachineInstr &MI, const SIInstrInfo &TII) { TII.getNamedOperand(MI, AMDGPU::OpName::dmask)->getImm(); return countPopulation(DMaskImm); } + if (TII.isMTBUF(Opc)) { + return AMDGPU::getMTBUFElements(Opc); + } switch (Opc) { case AMDGPU::S_BUFFER_LOAD_DWORD_IMM: @@ -322,10 +330,27 @@ static InstClassEnum getInstClass(unsigned Opc, const SIInstrInfo &TII) { if (AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::vaddr) == -1) return UNKNOWN; // TODO: Support IMAGE_GET_RESINFO and IMAGE_GET_LOD. - if (TII.get(Opc).mayStore() || !TII.get(Opc).mayLoad() || TII.isGather4(Opc)) + if (TII.get(Opc).mayStore() || !TII.get(Opc).mayLoad() || + TII.isGather4(Opc)) return UNKNOWN; return MIMG; } + if (TII.isMTBUF(Opc)) { + switch (AMDGPU::getMTBUFBaseOpcode(Opc)) { + default: + return UNKNOWN; + case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN: + case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFEN_exact: + case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET: + case AMDGPU::TBUFFER_LOAD_FORMAT_X_OFFSET_exact: + return TBUFFER_LOAD; + case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN: + case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFEN_exact: + case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET: + case AMDGPU::TBUFFER_STORE_FORMAT_X_OFFSET_exact: + return TBUFFER_STORE; + } + } return UNKNOWN; case AMDGPU::S_BUFFER_LOAD_DWORD_IMM: case AMDGPU::S_BUFFER_LOAD_DWORDX2_IMM: @@ -356,6 +381,8 @@ static unsigned getInstSubclass(unsigned Opc, const SIInstrInfo &TII) { assert(Info); return Info->BaseOpcode; } + if (TII.isMTBUF(Opc)) + return AMDGPU::getMTBUFBaseOpcode(Opc); return -1; case AMDGPU::DS_READ_B32: case AMDGPU::DS_READ_B32_gfx9: @@ -397,6 +424,24 @@ static unsigned getRegs(unsigned Opc, const SIInstrInfo &TII) { const AMDGPU::MIMGInfo *Info = AMDGPU::getMIMGInfo(Opc); if (Info && AMDGPU::getMIMGBaseOpcodeInfo(Info->BaseOpcode)->Sampler) result |= SSAMP; + + return result; + } + if (TII.isMTBUF(Opc)) { + unsigned result = 0; + + if (AMDGPU::getMTBUFHasVAddr(Opc)) { + result |= VADDR; + } + + if (AMDGPU::getMTBUFHasSrsrc(Opc)) { + result |= SRSRC; + } + + if (AMDGPU::getMTBUFHasSoffset(Opc)) { + result |= SOFFSET; + } + return result; } @@ -419,7 +464,6 @@ static unsigned getRegs(unsigned Opc, const SIInstrInfo &TII) { } } - void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI, const SIInstrInfo &TII, const GCNSubtarget &STM) { @@ -450,22 +494,25 @@ void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI, } if (InstClass == MIMG) { - DMask0 = TII.getNamedOperand(*I, AMDGPU::OpName::dmask)->getImm(); + DMask = TII.getNamedOperand(*I, AMDGPU::OpName::dmask)->getImm(); } else { int OffsetIdx = AMDGPU::getNamedOperandIdx(Opc, AMDGPU::OpName::offset); - Offset0 = I->getOperand(OffsetIdx).getImm(); + Offset = I->getOperand(OffsetIdx).getImm(); } - Width0 = getOpcodeWidth(*I, TII); + if (InstClass == TBUFFER_LOAD || InstClass == TBUFFER_STORE) + Format = TII.getNamedOperand(*I, AMDGPU::OpName::format)->getImm(); + + Width = getOpcodeWidth(*I, TII); if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) { - Offset0 &= 0xffff; + Offset &= 0xffff; } else if (InstClass != MIMG) { - GLC0 = TII.getNamedOperand(*I, AMDGPU::OpName::glc)->getImm(); + GLC = TII.getNamedOperand(*I, AMDGPU::OpName::glc)->getImm(); if (InstClass != S_BUFFER_LOAD_IMM) { - SLC0 = TII.getNamedOperand(*I, AMDGPU::OpName::slc)->getImm(); + SLC = TII.getNamedOperand(*I, AMDGPU::OpName::slc)->getImm(); } - DLC0 = TII.getNamedOperand(*I, AMDGPU::OpName::dlc)->getImm(); + DLC = TII.getNamedOperand(*I, AMDGPU::OpName::dlc)->getImm(); } unsigned AddrOpName[5] = {0}; @@ -504,32 +551,6 @@ void SILoadStoreOptimizer::CombineInfo::setMI(MachineBasicBlock::iterator MI, InstsToMove.clear(); } -void SILoadStoreOptimizer::CombineInfo::setPaired(MachineBasicBlock::iterator MI, - const SIInstrInfo &TII) { - Paired = MI; - assert(InstClass == getInstClass(Paired->getOpcode(), TII)); - - if (InstClass == MIMG) { - DMask1 = TII.getNamedOperand(*Paired, AMDGPU::OpName::dmask)->getImm(); - } else { - int OffsetIdx = - AMDGPU::getNamedOperandIdx(I->getOpcode(), AMDGPU::OpName::offset); - Offset1 = Paired->getOperand(OffsetIdx).getImm(); - } - - Width1 = getOpcodeWidth(*Paired, TII); - if ((InstClass == DS_READ) || (InstClass == DS_WRITE)) { - Offset1 &= 0xffff; - } else if (InstClass != MIMG) { - GLC1 = TII.getNamedOperand(*Paired, AMDGPU::OpName::glc)->getImm(); - if (InstClass != S_BUFFER_LOAD_IMM) { - SLC1 = TII.getNamedOperand(*Paired, AMDGPU::OpName::slc)->getImm(); - } - DLC1 = TII.getNamedOperand(*Paired, AMDGPU::OpName::dlc)->getImm(); - } -} - - } // end anonymous namespace. INITIALIZE_PASS_BEGIN(SILoadStoreOptimizer, DEBUG_TYPE, @@ -635,7 +656,9 @@ static MachineMemOperand *combineKnownAdjacentMMOs(MachineFunction &MF, return MMO; } -bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI, const SIInstrInfo &TII) { +bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI, + const SIInstrInfo &TII, + const CombineInfo &Paired) { assert(CI.InstClass == MIMG); // Ignore instructions with tfe/lwe set. @@ -652,16 +675,16 @@ bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI, const SIIn for (auto op : OperandsToMatch) { int Idx = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), op); - if (AMDGPU::getNamedOperandIdx(CI.Paired->getOpcode(), op) != Idx) + if (AMDGPU::getNamedOperandIdx(Paired.I->getOpcode(), op) != Idx) return false; if (Idx != -1 && - CI.I->getOperand(Idx).getImm() != CI.Paired->getOperand(Idx).getImm()) + CI.I->getOperand(Idx).getImm() != Paired.I->getOperand(Idx).getImm()) return false; } // Check DMask for overlaps. - unsigned MaxMask = std::max(CI.DMask0, CI.DMask1); - unsigned MinMask = std::min(CI.DMask0, CI.DMask1); + unsigned MaxMask = std::max(CI.DMask, Paired.DMask); + unsigned MinMask = std::min(CI.DMask, Paired.DMask); unsigned AllowedBitsForMin = llvm::countTrailingZeros(MaxMask); if ((1u << AllowedBitsForMin) <= MinMask) @@ -670,62 +693,113 @@ bool SILoadStoreOptimizer::dmasksCanBeCombined(const CombineInfo &CI, const SIIn return true; } -bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI) { +static unsigned getBufferFormatWithCompCount(unsigned OldFormat, + unsigned ComponentCount, + const MCSubtargetInfo &STI) { + if (ComponentCount > 4) + return 0; + + const llvm::AMDGPU::GcnBufferFormatInfo *OldFormatInfo = + llvm::AMDGPU::getGcnBufferFormatInfo(OldFormat, STI); + if (!OldFormatInfo) + return 0; + + const llvm::AMDGPU::GcnBufferFormatInfo *NewFormatInfo = + llvm::AMDGPU::getGcnBufferFormatInfo(OldFormatInfo->BitsPerComp, + ComponentCount, + OldFormatInfo->NumFormat, STI); + + if (!NewFormatInfo) + return 0; + + assert(NewFormatInfo->NumFormat == OldFormatInfo->NumFormat && + NewFormatInfo->BitsPerComp == OldFormatInfo->BitsPerComp); + + return NewFormatInfo->Format; +} + +bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI, + const MCSubtargetInfo &STI, + CombineInfo &Paired) { assert(CI.InstClass != MIMG); // XXX - Would the same offset be OK? Is there any reason this would happen or // be useful? - if (CI.Offset0 == CI.Offset1) + if (CI.Offset == Paired.Offset) return false; // This won't be valid if the offset isn't aligned. - if ((CI.Offset0 % CI.EltSize != 0) || (CI.Offset1 % CI.EltSize != 0)) + if ((CI.Offset % CI.EltSize != 0) || (Paired.Offset % CI.EltSize != 0)) return false; - unsigned EltOffset0 = CI.Offset0 / CI.EltSize; - unsigned EltOffset1 = CI.Offset1 / CI.EltSize; + if (CI.InstClass == TBUFFER_LOAD || CI.InstClass == TBUFFER_STORE) { + + const llvm::AMDGPU::GcnBufferFormatInfo *Info0 = + llvm::AMDGPU::getGcnBufferFormatInfo(CI.Format, STI); + if (!Info0) + return false; + const llvm::AMDGPU::GcnBufferFormatInfo *Info1 = + llvm::AMDGPU::getGcnBufferFormatInfo(Paired.Format, STI); + if (!Info1) + return false; + + if (Info0->BitsPerComp != Info1->BitsPerComp || + Info0->NumFormat != Info1->NumFormat) + return false; + + // TODO: Should be possible to support more formats, but if format loads + // are not dword-aligned, the merged load might not be valid. + if (Info0->BitsPerComp != 32) + return false; + + if (getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, STI) == 0) + return false; + } + + unsigned EltOffset0 = CI.Offset / CI.EltSize; + unsigned EltOffset1 = Paired.Offset / CI.EltSize; CI.UseST64 = false; CI.BaseOff = 0; // Handle SMEM and VMEM instructions. if ((CI.InstClass != DS_READ) && (CI.InstClass != DS_WRITE)) { - return (EltOffset0 + CI.Width0 == EltOffset1 || - EltOffset1 + CI.Width1 == EltOffset0) && - CI.GLC0 == CI.GLC1 && CI.DLC0 == CI.DLC1 && - (CI.InstClass == S_BUFFER_LOAD_IMM || CI.SLC0 == CI.SLC1); + return (EltOffset0 + CI.Width == EltOffset1 || + EltOffset1 + Paired.Width == EltOffset0) && + CI.GLC == Paired.GLC && CI.DLC == Paired.DLC && + (CI.InstClass == S_BUFFER_LOAD_IMM || CI.SLC == Paired.SLC); } // If the offset in elements doesn't fit in 8-bits, we might be able to use // the stride 64 versions. if ((EltOffset0 % 64 == 0) && (EltOffset1 % 64) == 0 && isUInt<8>(EltOffset0 / 64) && isUInt<8>(EltOffset1 / 64)) { - CI.Offset0 = EltOffset0 / 64; - CI.Offset1 = EltOffset1 / 64; + CI.Offset = EltOffset0 / 64; + Paired.Offset = EltOffset1 / 64; CI.UseST64 = true; return true; } // Check if the new offsets fit in the reduced 8-bit range. if (isUInt<8>(EltOffset0) && isUInt<8>(EltOffset1)) { - CI.Offset0 = EltOffset0; - CI.Offset1 = EltOffset1; + CI.Offset = EltOffset0; + Paired.Offset = EltOffset1; return true; } // Try to shift base address to decrease offsets. unsigned OffsetDiff = std::abs((int)EltOffset1 - (int)EltOffset0); - CI.BaseOff = std::min(CI.Offset0, CI.Offset1); + CI.BaseOff = std::min(CI.Offset, Paired.Offset); if ((OffsetDiff % 64 == 0) && isUInt<8>(OffsetDiff / 64)) { - CI.Offset0 = (EltOffset0 - CI.BaseOff / CI.EltSize) / 64; - CI.Offset1 = (EltOffset1 - CI.BaseOff / CI.EltSize) / 64; + CI.Offset = (EltOffset0 - CI.BaseOff / CI.EltSize) / 64; + Paired.Offset = (EltOffset1 - CI.BaseOff / CI.EltSize) / 64; CI.UseST64 = true; return true; } if (isUInt<8>(OffsetDiff)) { - CI.Offset0 = EltOffset0 - CI.BaseOff / CI.EltSize; - CI.Offset1 = EltOffset1 - CI.BaseOff / CI.EltSize; + CI.Offset = EltOffset0 - CI.BaseOff / CI.EltSize; + Paired.Offset = EltOffset1 - CI.BaseOff / CI.EltSize; return true; } @@ -733,8 +807,9 @@ bool SILoadStoreOptimizer::offsetsCanBeCombined(CombineInfo &CI) { } bool SILoadStoreOptimizer::widthsFit(const GCNSubtarget &STM, - const CombineInfo &CI) { - const unsigned Width = (CI.Width0 + CI.Width1); + const CombineInfo &CI, + const CombineInfo &Paired) { + const unsigned Width = (CI.Width + Paired.Width); switch (CI.InstClass) { default: return (Width <= 4) && (STM.hasDwordx3LoadStores() || (Width != 3)); @@ -749,7 +824,8 @@ bool SILoadStoreOptimizer::widthsFit(const GCNSubtarget &STM, } } -bool SILoadStoreOptimizer::findMatchingInst(CombineInfo &CI) { +bool SILoadStoreOptimizer::findMatchingInst(CombineInfo &CI, + CombineInfo &Paired) { MachineBasicBlock *MBB = CI.I->getParent(); MachineBasicBlock::iterator E = MBB->end(); MachineBasicBlock::iterator MBBI = CI.I; @@ -813,6 +889,11 @@ bool SILoadStoreOptimizer::findMatchingInst(CombineInfo &CI) { if (MBBI->hasOrderedMemoryRef()) return false; + int Swizzled = + AMDGPU::getNamedOperandIdx(MBBI->getOpcode(), AMDGPU::OpName::swz); + if (Swizzled != -1 && MBBI->getOperand(Swizzled).getImm()) + return false; + // Handle a case like // DS_WRITE_B32 addr, v, idx0 // w = DS_READ_B32 addr, idx0 @@ -826,14 +907,14 @@ bool SILoadStoreOptimizer::findMatchingInst(CombineInfo &CI) { bool Match = CI.hasSameBaseAddress(*MBBI); if (Match) { - CI.setPaired(MBBI, *TII); + Paired.setMI(MBBI, *TII, *STM); // Check both offsets (or masks for MIMG) can be combined and fit in the // reduced range. bool canBeCombined = CI.InstClass == MIMG - ? dmasksCanBeCombined(CI, *TII) - : widthsFit(*STM, CI) && offsetsCanBeCombined(CI); + ? dmasksCanBeCombined(CI, *TII, Paired) + : widthsFit(*STM, CI, Paired) && offsetsCanBeCombined(CI, *STI, Paired); // We also need to go through the list of instructions that we plan to // move and make sure they are all safe to move down past the merged @@ -869,7 +950,7 @@ unsigned SILoadStoreOptimizer::read2ST64Opcode(unsigned EltSize) const { } MachineBasicBlock::iterator -SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI) { +SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI, CombineInfo &Paired) { MachineBasicBlock *MBB = CI.I->getParent(); // Be careful, since the addresses could be subregisters themselves in weird @@ -877,10 +958,10 @@ SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI) { const auto *AddrReg = TII->getNamedOperand(*CI.I, AMDGPU::OpName::addr); const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdst); - const auto *Dest1 = TII->getNamedOperand(*CI.Paired, AMDGPU::OpName::vdst); + const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdst); - unsigned NewOffset0 = CI.Offset0; - unsigned NewOffset1 = CI.Offset1; + unsigned NewOffset0 = CI.Offset; + unsigned NewOffset1 = Paired.Offset; unsigned Opc = CI.UseST64 ? read2ST64Opcode(CI.EltSize) : read2Opcode(CI.EltSize); @@ -909,13 +990,13 @@ SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI) { unsigned BaseRegFlags = 0; if (CI.BaseOff) { Register ImmReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); - BuildMI(*MBB, CI.Paired, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg) + BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg) .addImm(CI.BaseOff); BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); BaseRegFlags = RegState::Kill; - TII->getAddNoCarry(*MBB, CI.Paired, DL, BaseReg) + TII->getAddNoCarry(*MBB, Paired.I, DL, BaseReg) .addReg(ImmReg) .addReg(AddrReg->getReg(), 0, BaseSubReg) .addImm(0); // clamp bit @@ -923,29 +1004,29 @@ SILoadStoreOptimizer::mergeRead2Pair(CombineInfo &CI) { } MachineInstrBuilder Read2 = - BuildMI(*MBB, CI.Paired, DL, Read2Desc, DestReg) + BuildMI(*MBB, Paired.I, DL, Read2Desc, DestReg) .addReg(BaseReg, BaseRegFlags, BaseSubReg) // addr .addImm(NewOffset0) // offset0 .addImm(NewOffset1) // offset1 .addImm(0) // gds - .cloneMergedMemRefs({&*CI.I, &*CI.Paired}); + .cloneMergedMemRefs({&*CI.I, &*Paired.I}); (void)Read2; const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); // Copy to the old destination registers. - BuildMI(*MBB, CI.Paired, DL, CopyDesc) + BuildMI(*MBB, Paired.I, DL, CopyDesc) .add(*Dest0) // Copy to same destination including flags and sub reg. .addReg(DestReg, 0, SubRegIdx0); - MachineInstr *Copy1 = BuildMI(*MBB, CI.Paired, DL, CopyDesc) + MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) .add(*Dest1) .addReg(DestReg, RegState::Kill, SubRegIdx1); moveInstsAfter(Copy1, CI.InstsToMove); CI.I->eraseFromParent(); - CI.Paired->eraseFromParent(); + Paired.I->eraseFromParent(); LLVM_DEBUG(dbgs() << "Inserted read2: " << *Read2 << '\n'); return Read2; @@ -968,7 +1049,7 @@ unsigned SILoadStoreOptimizer::write2ST64Opcode(unsigned EltSize) const { } MachineBasicBlock::iterator -SILoadStoreOptimizer::mergeWrite2Pair(CombineInfo &CI) { +SILoadStoreOptimizer::mergeWrite2Pair(CombineInfo &CI, CombineInfo &Paired) { MachineBasicBlock *MBB = CI.I->getParent(); // Be sure to use .addOperand(), and not .addReg() with these. We want to be @@ -978,10 +1059,10 @@ SILoadStoreOptimizer::mergeWrite2Pair(CombineInfo &CI) { const MachineOperand *Data0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::data0); const MachineOperand *Data1 = - TII->getNamedOperand(*CI.Paired, AMDGPU::OpName::data0); + TII->getNamedOperand(*Paired.I, AMDGPU::OpName::data0); - unsigned NewOffset0 = CI.Offset0; - unsigned NewOffset1 = CI.Offset1; + unsigned NewOffset0 = CI.Offset; + unsigned NewOffset1 = Paired.Offset; unsigned Opc = CI.UseST64 ? write2ST64Opcode(CI.EltSize) : write2Opcode(CI.EltSize); @@ -1002,13 +1083,13 @@ SILoadStoreOptimizer::mergeWrite2Pair(CombineInfo &CI) { unsigned BaseRegFlags = 0; if (CI.BaseOff) { Register ImmReg = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass); - BuildMI(*MBB, CI.Paired, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg) + BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::S_MOV_B32), ImmReg) .addImm(CI.BaseOff); BaseReg = MRI->createVirtualRegister(&AMDGPU::VGPR_32RegClass); BaseRegFlags = RegState::Kill; - TII->getAddNoCarry(*MBB, CI.Paired, DL, BaseReg) + TII->getAddNoCarry(*MBB, Paired.I, DL, BaseReg) .addReg(ImmReg) .addReg(AddrReg->getReg(), 0, BaseSubReg) .addImm(0); // clamp bit @@ -1016,38 +1097,38 @@ SILoadStoreOptimizer::mergeWrite2Pair(CombineInfo &CI) { } MachineInstrBuilder Write2 = - BuildMI(*MBB, CI.Paired, DL, Write2Desc) + BuildMI(*MBB, Paired.I, DL, Write2Desc) .addReg(BaseReg, BaseRegFlags, BaseSubReg) // addr .add(*Data0) // data0 .add(*Data1) // data1 .addImm(NewOffset0) // offset0 .addImm(NewOffset1) // offset1 .addImm(0) // gds - .cloneMergedMemRefs({&*CI.I, &*CI.Paired}); + .cloneMergedMemRefs({&*CI.I, &*Paired.I}); moveInstsAfter(Write2, CI.InstsToMove); CI.I->eraseFromParent(); - CI.Paired->eraseFromParent(); + Paired.I->eraseFromParent(); LLVM_DEBUG(dbgs() << "Inserted write2 inst: " << *Write2 << '\n'); return Write2; } MachineBasicBlock::iterator -SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI) { +SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI, CombineInfo &Paired) { MachineBasicBlock *MBB = CI.I->getParent(); DebugLoc DL = CI.I->getDebugLoc(); - const unsigned Opcode = getNewOpcode(CI); + const unsigned Opcode = getNewOpcode(CI, Paired); - const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI); + const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); Register DestReg = MRI->createVirtualRegister(SuperRC); - unsigned MergedDMask = CI.DMask0 | CI.DMask1; + unsigned MergedDMask = CI.DMask | Paired.DMask; unsigned DMaskIdx = AMDGPU::getNamedOperandIdx(CI.I->getOpcode(), AMDGPU::OpName::dmask); - auto MIB = BuildMI(*MBB, CI.Paired, DL, TII->get(Opcode), DestReg); + auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg); for (unsigned I = 1, E = (*CI.I).getNumOperands(); I != E; ++I) { if (I == DMaskIdx) MIB.addImm(MergedDMask); @@ -1058,100 +1139,99 @@ SILoadStoreOptimizer::mergeImagePair(CombineInfo &CI) { // It shouldn't be possible to get this far if the two instructions // don't have a single memoperand, because MachineInstr::mayAlias() // will return true if this is the case. - assert(CI.I->hasOneMemOperand() && CI.Paired->hasOneMemOperand()); + assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); - const MachineMemOperand *MMOb = *CI.Paired->memoperands_begin(); + const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); MachineInstr *New = MIB.addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); - std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI); - const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); - const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); + unsigned SubRegIdx0, SubRegIdx1; + std::tie(SubRegIdx0, SubRegIdx1) = getSubRegIdxs(CI, Paired); // Copy to the old destination registers. const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); - const auto *Dest1 = TII->getNamedOperand(*CI.Paired, AMDGPU::OpName::vdata); + const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); - BuildMI(*MBB, CI.Paired, DL, CopyDesc) + BuildMI(*MBB, Paired.I, DL, CopyDesc) .add(*Dest0) // Copy to same destination including flags and sub reg. .addReg(DestReg, 0, SubRegIdx0); - MachineInstr *Copy1 = BuildMI(*MBB, CI.Paired, DL, CopyDesc) + MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) .add(*Dest1) .addReg(DestReg, RegState::Kill, SubRegIdx1); moveInstsAfter(Copy1, CI.InstsToMove); CI.I->eraseFromParent(); - CI.Paired->eraseFromParent(); + Paired.I->eraseFromParent(); return New; } MachineBasicBlock::iterator -SILoadStoreOptimizer::mergeSBufferLoadImmPair(CombineInfo &CI) { +SILoadStoreOptimizer::mergeSBufferLoadImmPair(CombineInfo &CI, CombineInfo &Paired) { MachineBasicBlock *MBB = CI.I->getParent(); DebugLoc DL = CI.I->getDebugLoc(); - const unsigned Opcode = getNewOpcode(CI); + const unsigned Opcode = getNewOpcode(CI, Paired); - const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI); + const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); Register DestReg = MRI->createVirtualRegister(SuperRC); - unsigned MergedOffset = std::min(CI.Offset0, CI.Offset1); + unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); // It shouldn't be possible to get this far if the two instructions // don't have a single memoperand, because MachineInstr::mayAlias() // will return true if this is the case. - assert(CI.I->hasOneMemOperand() && CI.Paired->hasOneMemOperand()); + assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); - const MachineMemOperand *MMOb = *CI.Paired->memoperands_begin(); + const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); MachineInstr *New = - BuildMI(*MBB, CI.Paired, DL, TII->get(Opcode), DestReg) + BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg) .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::sbase)) .addImm(MergedOffset) // offset - .addImm(CI.GLC0) // glc - .addImm(CI.DLC0) // dlc + .addImm(CI.GLC) // glc + .addImm(CI.DLC) // dlc .addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); - std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI); + std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); // Copy to the old destination registers. const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::sdst); - const auto *Dest1 = TII->getNamedOperand(*CI.Paired, AMDGPU::OpName::sdst); + const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::sdst); - BuildMI(*MBB, CI.Paired, DL, CopyDesc) + BuildMI(*MBB, Paired.I, DL, CopyDesc) .add(*Dest0) // Copy to same destination including flags and sub reg. .addReg(DestReg, 0, SubRegIdx0); - MachineInstr *Copy1 = BuildMI(*MBB, CI.Paired, DL, CopyDesc) + MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) .add(*Dest1) .addReg(DestReg, RegState::Kill, SubRegIdx1); moveInstsAfter(Copy1, CI.InstsToMove); CI.I->eraseFromParent(); - CI.Paired->eraseFromParent(); + Paired.I->eraseFromParent(); return New; } MachineBasicBlock::iterator -SILoadStoreOptimizer::mergeBufferLoadPair(CombineInfo &CI) { +SILoadStoreOptimizer::mergeBufferLoadPair(CombineInfo &CI, CombineInfo &Paired) { MachineBasicBlock *MBB = CI.I->getParent(); DebugLoc DL = CI.I->getDebugLoc(); - const unsigned Opcode = getNewOpcode(CI); + const unsigned Opcode = getNewOpcode(CI, Paired); - const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI); + const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); // Copy to the new source register. Register DestReg = MRI->createVirtualRegister(SuperRC); - unsigned MergedOffset = std::min(CI.Offset0, CI.Offset1); + unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); - auto MIB = BuildMI(*MBB, CI.Paired, DL, TII->get(Opcode), DestReg); + auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg); const unsigned Regs = getRegs(Opcode, *TII); @@ -1161,47 +1241,178 @@ SILoadStoreOptimizer::mergeBufferLoadPair(CombineInfo &CI) { // It shouldn't be possible to get this far if the two instructions // don't have a single memoperand, because MachineInstr::mayAlias() // will return true if this is the case. - assert(CI.I->hasOneMemOperand() && CI.Paired->hasOneMemOperand()); + assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); - const MachineMemOperand *MMOb = *CI.Paired->memoperands_begin(); + const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); MachineInstr *New = MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) .addImm(MergedOffset) // offset - .addImm(CI.GLC0) // glc - .addImm(CI.SLC0) // slc + .addImm(CI.GLC) // glc + .addImm(CI.SLC) // slc .addImm(0) // tfe - .addImm(CI.DLC0) // dlc + .addImm(CI.DLC) // dlc .addImm(0) // swz .addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); - std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI); + std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); // Copy to the old destination registers. const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); - const auto *Dest1 = TII->getNamedOperand(*CI.Paired, AMDGPU::OpName::vdata); + const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); - BuildMI(*MBB, CI.Paired, DL, CopyDesc) + BuildMI(*MBB, Paired.I, DL, CopyDesc) .add(*Dest0) // Copy to same destination including flags and sub reg. .addReg(DestReg, 0, SubRegIdx0); - MachineInstr *Copy1 = BuildMI(*MBB, CI.Paired, DL, CopyDesc) + MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) .add(*Dest1) .addReg(DestReg, RegState::Kill, SubRegIdx1); moveInstsAfter(Copy1, CI.InstsToMove); CI.I->eraseFromParent(); - CI.Paired->eraseFromParent(); + Paired.I->eraseFromParent(); return New; } -unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI) { - const unsigned Width = CI.Width0 + CI.Width1; +MachineBasicBlock::iterator +SILoadStoreOptimizer::mergeTBufferLoadPair(CombineInfo &CI, CombineInfo &Paired) { + MachineBasicBlock *MBB = CI.I->getParent(); + DebugLoc DL = CI.I->getDebugLoc(); + + const unsigned Opcode = getNewOpcode(CI, Paired); + + const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); + + // Copy to the new source register. + Register DestReg = MRI->createVirtualRegister(SuperRC); + unsigned MergedOffset = std::min(CI.Offset, Paired.Offset); + + auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode), DestReg); + + const unsigned Regs = getRegs(Opcode, *TII); + + if (Regs & VADDR) + MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); + + unsigned JoinedFormat = + getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STI); + + // It shouldn't be possible to get this far if the two instructions + // don't have a single memoperand, because MachineInstr::mayAlias() + // will return true if this is the case. + assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); + + const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); + const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); + + MachineInstr *New = + MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) + .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) + .addImm(MergedOffset) // offset + .addImm(JoinedFormat) // format + .addImm(CI.GLC) // glc + .addImm(CI.SLC) // slc + .addImm(0) // tfe + .addImm(CI.DLC) // dlc + .addImm(0) // swz + .addMemOperand( + combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); + + std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); + const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); + const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); + + // Copy to the old destination registers. + const MCInstrDesc &CopyDesc = TII->get(TargetOpcode::COPY); + const auto *Dest0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); + const auto *Dest1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); + + BuildMI(*MBB, Paired.I, DL, CopyDesc) + .add(*Dest0) // Copy to same destination including flags and sub reg. + .addReg(DestReg, 0, SubRegIdx0); + MachineInstr *Copy1 = BuildMI(*MBB, Paired.I, DL, CopyDesc) + .add(*Dest1) + .addReg(DestReg, RegState::Kill, SubRegIdx1); + + moveInstsAfter(Copy1, CI.InstsToMove); + + CI.I->eraseFromParent(); + Paired.I->eraseFromParent(); + return New; +} + +MachineBasicBlock::iterator +SILoadStoreOptimizer::mergeTBufferStorePair(CombineInfo &CI, CombineInfo &Paired) { + MachineBasicBlock *MBB = CI.I->getParent(); + DebugLoc DL = CI.I->getDebugLoc(); + + const unsigned Opcode = getNewOpcode(CI, Paired); + + std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); + const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); + const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); + + // Copy to the new source register. + const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); + Register SrcReg = MRI->createVirtualRegister(SuperRC); + + const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); + const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); + + BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::REG_SEQUENCE), SrcReg) + .add(*Src0) + .addImm(SubRegIdx0) + .add(*Src1) + .addImm(SubRegIdx1); + + auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode)) + .addReg(SrcReg, RegState::Kill); + + const unsigned Regs = getRegs(Opcode, *TII); + + if (Regs & VADDR) + MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::vaddr)); + + unsigned JoinedFormat = + getBufferFormatWithCompCount(CI.Format, CI.Width + Paired.Width, *STI); + + // It shouldn't be possible to get this far if the two instructions + // don't have a single memoperand, because MachineInstr::mayAlias() + // will return true if this is the case. + assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); + + const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); + const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); + + MachineInstr *New = + MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) + .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) + .addImm(std::min(CI.Offset, Paired.Offset)) // offset + .addImm(JoinedFormat) // format + .addImm(CI.GLC) // glc + .addImm(CI.SLC) // slc + .addImm(0) // tfe + .addImm(CI.DLC) // dlc + .addImm(0) // swz + .addMemOperand( + combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); + + moveInstsAfter(MIB, CI.InstsToMove); + + CI.I->eraseFromParent(); + Paired.I->eraseFromParent(); + return New; +} + +unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI, + const CombineInfo &Paired) { + const unsigned Width = CI.Width + Paired.Width; switch (CI.InstClass) { default: @@ -1209,6 +1420,11 @@ unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI) { // FIXME: Handle d16 correctly return AMDGPU::getMUBUFOpcode(AMDGPU::getMUBUFBaseOpcode(CI.I->getOpcode()), Width); + case TBUFFER_LOAD: + case TBUFFER_STORE: + return AMDGPU::getMTBUFOpcode(AMDGPU::getMTBUFBaseOpcode(CI.I->getOpcode()), + Width); + case UNKNOWN: llvm_unreachable("Unknown instruction class"); case S_BUFFER_LOAD_IMM: @@ -1221,24 +1437,24 @@ unsigned SILoadStoreOptimizer::getNewOpcode(const CombineInfo &CI) { return AMDGPU::S_BUFFER_LOAD_DWORDX4_IMM; } case MIMG: - assert("No overlaps" && (countPopulation(CI.DMask0 | CI.DMask1) == Width)); + assert("No overlaps" && (countPopulation(CI.DMask | Paired.DMask) == Width)); return AMDGPU::getMaskedMIMGOp(CI.I->getOpcode(), Width); } } std::pair<unsigned, unsigned> -SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI) { +SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI, const CombineInfo &Paired) { - if (CI.Width0 == 0 || CI.Width0 == 0 || CI.Width0 + CI.Width1 > 4) + if (CI.Width == 0 || Paired.Width == 0 || CI.Width + Paired.Width > 4) return std::make_pair(0, 0); bool ReverseOrder; if (CI.InstClass == MIMG) { - assert((countPopulation(CI.DMask0 | CI.DMask1) == CI.Width0 + CI.Width1) && + assert((countPopulation(CI.DMask | Paired.DMask) == CI.Width + Paired.Width) && "No overlaps"); - ReverseOrder = CI.DMask0 > CI.DMask1; + ReverseOrder = CI.DMask > Paired.DMask; } else - ReverseOrder = CI.Offset0 > CI.Offset1; + ReverseOrder = CI.Offset > Paired.Offset; static const unsigned Idxs[4][4] = { {AMDGPU::sub0, AMDGPU::sub0_sub1, AMDGPU::sub0_sub1_sub2, AMDGPU::sub0_sub1_sub2_sub3}, @@ -1249,24 +1465,25 @@ SILoadStoreOptimizer::getSubRegIdxs(const CombineInfo &CI) { unsigned Idx0; unsigned Idx1; - assert(CI.Width0 >= 1 && CI.Width0 <= 3); - assert(CI.Width1 >= 1 && CI.Width1 <= 3); + assert(CI.Width >= 1 && CI.Width <= 3); + assert(Paired.Width >= 1 && Paired.Width <= 3); if (ReverseOrder) { - Idx1 = Idxs[0][CI.Width1 - 1]; - Idx0 = Idxs[CI.Width1][CI.Width0 - 1]; + Idx1 = Idxs[0][Paired.Width - 1]; + Idx0 = Idxs[Paired.Width][CI.Width - 1]; } else { - Idx0 = Idxs[0][CI.Width0 - 1]; - Idx1 = Idxs[CI.Width0][CI.Width1 - 1]; + Idx0 = Idxs[0][CI.Width - 1]; + Idx1 = Idxs[CI.Width][Paired.Width - 1]; } return std::make_pair(Idx0, Idx1); } const TargetRegisterClass * -SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI) { +SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI, + const CombineInfo &Paired) { if (CI.InstClass == S_BUFFER_LOAD_IMM) { - switch (CI.Width0 + CI.Width1) { + switch (CI.Width + Paired.Width) { default: return nullptr; case 2: @@ -1279,7 +1496,7 @@ SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI) { return &AMDGPU::SReg_512RegClass; } } else { - switch (CI.Width0 + CI.Width1) { + switch (CI.Width + Paired.Width) { default: return nullptr; case 2: @@ -1293,30 +1510,30 @@ SILoadStoreOptimizer::getTargetRegisterClass(const CombineInfo &CI) { } MachineBasicBlock::iterator -SILoadStoreOptimizer::mergeBufferStorePair(CombineInfo &CI) { +SILoadStoreOptimizer::mergeBufferStorePair(CombineInfo &CI, CombineInfo &Paired) { MachineBasicBlock *MBB = CI.I->getParent(); DebugLoc DL = CI.I->getDebugLoc(); - const unsigned Opcode = getNewOpcode(CI); + const unsigned Opcode = getNewOpcode(CI, Paired); - std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI); + std::pair<unsigned, unsigned> SubRegIdx = getSubRegIdxs(CI, Paired); const unsigned SubRegIdx0 = std::get<0>(SubRegIdx); const unsigned SubRegIdx1 = std::get<1>(SubRegIdx); // Copy to the new source register. - const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI); + const TargetRegisterClass *SuperRC = getTargetRegisterClass(CI, Paired); Register SrcReg = MRI->createVirtualRegister(SuperRC); const auto *Src0 = TII->getNamedOperand(*CI.I, AMDGPU::OpName::vdata); - const auto *Src1 = TII->getNamedOperand(*CI.Paired, AMDGPU::OpName::vdata); + const auto *Src1 = TII->getNamedOperand(*Paired.I, AMDGPU::OpName::vdata); - BuildMI(*MBB, CI.Paired, DL, TII->get(AMDGPU::REG_SEQUENCE), SrcReg) + BuildMI(*MBB, Paired.I, DL, TII->get(AMDGPU::REG_SEQUENCE), SrcReg) .add(*Src0) .addImm(SubRegIdx0) .add(*Src1) .addImm(SubRegIdx1); - auto MIB = BuildMI(*MBB, CI.Paired, DL, TII->get(Opcode)) + auto MIB = BuildMI(*MBB, Paired.I, DL, TII->get(Opcode)) .addReg(SrcReg, RegState::Kill); const unsigned Regs = getRegs(Opcode, *TII); @@ -1328,26 +1545,26 @@ SILoadStoreOptimizer::mergeBufferStorePair(CombineInfo &CI) { // It shouldn't be possible to get this far if the two instructions // don't have a single memoperand, because MachineInstr::mayAlias() // will return true if this is the case. - assert(CI.I->hasOneMemOperand() && CI.Paired->hasOneMemOperand()); + assert(CI.I->hasOneMemOperand() && Paired.I->hasOneMemOperand()); const MachineMemOperand *MMOa = *CI.I->memoperands_begin(); - const MachineMemOperand *MMOb = *CI.Paired->memoperands_begin(); + const MachineMemOperand *MMOb = *Paired.I->memoperands_begin(); MachineInstr *New = MIB.add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::srsrc)) .add(*TII->getNamedOperand(*CI.I, AMDGPU::OpName::soffset)) - .addImm(std::min(CI.Offset0, CI.Offset1)) // offset - .addImm(CI.GLC0) // glc - .addImm(CI.SLC0) // slc + .addImm(std::min(CI.Offset, Paired.Offset)) // offset + .addImm(CI.GLC) // glc + .addImm(CI.SLC) // slc .addImm(0) // tfe - .addImm(CI.DLC0) // dlc + .addImm(CI.DLC) // dlc .addImm(0) // swz .addMemOperand(combineKnownAdjacentMMOs(*MBB->getParent(), MMOa, MMOb)); moveInstsAfter(MIB, CI.InstsToMove); CI.I->eraseFromParent(); - CI.Paired->eraseFromParent(); + Paired.I->eraseFromParent(); return New; } @@ -1429,7 +1646,9 @@ unsigned SILoadStoreOptimizer::computeBase(MachineInstr &MI, void SILoadStoreOptimizer::updateBaseAndOffset(MachineInstr &MI, unsigned NewBase, int32_t NewOffset) const { - TII->getNamedOperand(MI, AMDGPU::OpName::vaddr)->setReg(NewBase); + auto Base = TII->getNamedOperand(MI, AMDGPU::OpName::vaddr); + Base->setReg(NewBase); + Base->setIsKill(false); TII->getNamedOperand(MI, AMDGPU::OpName::offset)->setImm(NewOffset); } @@ -1664,8 +1883,8 @@ bool SILoadStoreOptimizer::promoteConstantOffsetToImm( void SILoadStoreOptimizer::addInstToMergeableList(const CombineInfo &CI, std::list<std::list<CombineInfo> > &MergeableInsts) const { for (std::list<CombineInfo> &AddrList : MergeableInsts) { - if (AddrList.front().hasSameBaseAddress(*CI.I) && - AddrList.front().InstClass == CI.InstClass) { + if (AddrList.front().InstClass == CI.InstClass && + AddrList.front().hasSameBaseAddress(*CI.I)) { AddrList.emplace_back(CI); return; } @@ -1760,63 +1979,70 @@ SILoadStoreOptimizer::optimizeInstsWithSameBaseAddr( bool Modified = false; for (auto I = MergeList.begin(); I != MergeList.end(); ++I) { CombineInfo &CI = *I; + CombineInfo Paired; + + if (CI.InstClass == UNKNOWN) + continue; + + if (!findMatchingInst(CI, Paired)) + goto done; + + Modified = true; + removeCombinedInst(MergeList, *Paired.I); switch (CI.InstClass) { default: + llvm_unreachable("unknown InstClass"); break; - case DS_READ: - if (findMatchingInst(CI)) { - Modified = true; - removeCombinedInst(MergeList, *CI.Paired); - MachineBasicBlock::iterator NewMI = mergeRead2Pair(CI); - CI.setMI(NewMI, *TII, *STM); - } + case DS_READ: { + MachineBasicBlock::iterator NewMI = mergeRead2Pair(CI, Paired); + CI.setMI(NewMI, *TII, *STM); break; - case DS_WRITE: - if (findMatchingInst(CI)) { - Modified = true; - removeCombinedInst(MergeList, *CI.Paired); - MachineBasicBlock::iterator NewMI = mergeWrite2Pair(CI); - CI.setMI(NewMI, *TII, *STM); - } + } + case DS_WRITE: { + MachineBasicBlock::iterator NewMI = mergeWrite2Pair(CI, Paired); + CI.setMI(NewMI, *TII, *STM); break; - case S_BUFFER_LOAD_IMM: - if (findMatchingInst(CI)) { - Modified = true; - removeCombinedInst(MergeList, *CI.Paired); - MachineBasicBlock::iterator NewMI = mergeSBufferLoadImmPair(CI); - CI.setMI(NewMI, *TII, *STM); - OptimizeListAgain |= (CI.Width0 + CI.Width1) < 16; - } + } + case S_BUFFER_LOAD_IMM: { + MachineBasicBlock::iterator NewMI = mergeSBufferLoadImmPair(CI, Paired); + CI.setMI(NewMI, *TII, *STM); + OptimizeListAgain |= (CI.Width + Paired.Width) < 16; break; - case BUFFER_LOAD: - if (findMatchingInst(CI)) { - Modified = true; - removeCombinedInst(MergeList, *CI.Paired); - MachineBasicBlock::iterator NewMI = mergeBufferLoadPair(CI); - CI.setMI(NewMI, *TII, *STM); - OptimizeListAgain |= (CI.Width0 + CI.Width1) < 4; - } + } + case BUFFER_LOAD: { + MachineBasicBlock::iterator NewMI = mergeBufferLoadPair(CI, Paired); + CI.setMI(NewMI, *TII, *STM); + OptimizeListAgain |= (CI.Width + Paired.Width) < 4; break; - case BUFFER_STORE: - if (findMatchingInst(CI)) { - Modified = true; - removeCombinedInst(MergeList, *CI.Paired); - MachineBasicBlock::iterator NewMI = mergeBufferStorePair(CI); - CI.setMI(NewMI, *TII, *STM); - OptimizeListAgain |= (CI.Width0 + CI.Width1) < 4; - } + } + case BUFFER_STORE: { + MachineBasicBlock::iterator NewMI = mergeBufferStorePair(CI, Paired); + CI.setMI(NewMI, *TII, *STM); + OptimizeListAgain |= (CI.Width + Paired.Width) < 4; break; - case MIMG: - if (findMatchingInst(CI)) { - Modified = true; - removeCombinedInst(MergeList, *CI.Paired); - MachineBasicBlock::iterator NewMI = mergeImagePair(CI); - CI.setMI(NewMI, *TII, *STM); - OptimizeListAgain |= (CI.Width0 + CI.Width1) < 4; - } + } + case MIMG: { + MachineBasicBlock::iterator NewMI = mergeImagePair(CI, Paired); + CI.setMI(NewMI, *TII, *STM); + OptimizeListAgain |= (CI.Width + Paired.Width) < 4; + break; + } + case TBUFFER_LOAD: { + MachineBasicBlock::iterator NewMI = mergeTBufferLoadPair(CI, Paired); + CI.setMI(NewMI, *TII, *STM); + OptimizeListAgain |= (CI.Width + Paired.Width) < 4; + break; + } + case TBUFFER_STORE: { + MachineBasicBlock::iterator NewMI = mergeTBufferStorePair(CI, Paired); + CI.setMI(NewMI, *TII, *STM); + OptimizeListAgain |= (CI.Width + Paired.Width) < 4; break; } + } + +done: // Clear the InstsToMove after we have finished searching so we don't have // stale values left over if we search for this CI again in another pass // over the block. @@ -1836,6 +2062,7 @@ bool SILoadStoreOptimizer::runOnMachineFunction(MachineFunction &MF) { TII = STM->getInstrInfo(); TRI = &TII->getRegisterInfo(); + STI = &MF.getSubtarget<MCSubtargetInfo>(); MRI = &MF.getRegInfo(); AA = &getAnalysis<AAResultsWrapperPass>().getAAResults(); diff --git a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp index 6f9abd3a8d9b..61d2719a3aad 100644 --- a/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp +++ b/llvm/lib/Target/AMDGPU/SILowerControlFlow.cpp @@ -244,9 +244,9 @@ void SILowerControlFlow::emitIf(MachineInstr &MI) { BuildMI(MBB, I, DL, TII->get(MovTermOpc), Exec) .addReg(Tmp, RegState::Kill); - // Insert a pseudo terminator to help keep the verifier happy. This will also - // be used later when inserting skips. - MachineInstr *NewBr = BuildMI(MBB, I, DL, TII->get(AMDGPU::SI_MASK_BRANCH)) + // Insert the S_CBRANCH_EXECZ instruction which will be optimized later + // during SIRemoveShortExecBranches. + MachineInstr *NewBr = BuildMI(MBB, I, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ)) .add(MI.getOperand(2)); if (!LIS) { @@ -323,8 +323,8 @@ void SILowerControlFlow::emitElse(MachineInstr &MI) { .addReg(DstReg); MachineInstr *Branch = - BuildMI(MBB, ElsePt, DL, TII->get(AMDGPU::SI_MASK_BRANCH)) - .addMBB(DestBB); + BuildMI(MBB, ElsePt, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ)) + .addMBB(DestBB); if (!LIS) { MI.eraseFromParent(); @@ -372,12 +372,15 @@ void SILowerControlFlow::emitIfBreak(MachineInstr &MI) { // exit" mask. MachineInstr *And = nullptr, *Or = nullptr; if (!SkipAnding) { - And = BuildMI(MBB, &MI, DL, TII->get(AndOpc), Dst) + Register AndReg = MRI->createVirtualRegister(BoolRC); + And = BuildMI(MBB, &MI, DL, TII->get(AndOpc), AndReg) .addReg(Exec) .add(MI.getOperand(1)); Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst) - .addReg(Dst) + .addReg(AndReg) .add(MI.getOperand(2)); + if (LIS) + LIS->createAndComputeVirtRegInterval(AndReg); } else Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst) .add(MI.getOperand(1)) diff --git a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp index b45412536356..1d45e6241d22 100644 --- a/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp +++ b/llvm/lib/Target/AMDGPU/SILowerI1Copies.cpp @@ -33,6 +33,7 @@ #include "llvm/CodeGen/MachineSSAUpdater.h" #include "llvm/IR/Function.h" #include "llvm/IR/LLVMContext.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/Debug.h" #include "llvm/Target/TargetMachine.h" @@ -541,7 +542,7 @@ void SILowerI1Copies::lowerPhis() { MachineSSAUpdater SSAUpdater(*MF); LoopFinder LF(*DT, *PDT); PhiIncomingAnalysis PIA(*PDT); - SmallVector<MachineInstr *, 4> DeadPhis; + SmallVector<MachineInstr *, 4> Vreg1Phis; SmallVector<MachineBasicBlock *, 4> IncomingBlocks; SmallVector<unsigned, 4> IncomingRegs; SmallVector<unsigned, 4> IncomingUpdated; @@ -550,118 +551,117 @@ void SILowerI1Copies::lowerPhis() { #endif for (MachineBasicBlock &MBB : *MF) { - LF.initialize(MBB); - for (MachineInstr &MI : MBB.phis()) { - Register DstReg = MI.getOperand(0).getReg(); - if (!isVreg1(DstReg)) - continue; + if (isVreg1(MI.getOperand(0).getReg())) + Vreg1Phis.push_back(&MI); + } + } - LLVM_DEBUG(dbgs() << "Lower PHI: " << MI); + MachineBasicBlock *PrevMBB = nullptr; + for (MachineInstr *MI : Vreg1Phis) { + MachineBasicBlock &MBB = *MI->getParent(); + if (&MBB != PrevMBB) { + LF.initialize(MBB); + PrevMBB = &MBB; + } - MRI->setRegClass(DstReg, IsWave32 ? &AMDGPU::SReg_32RegClass - : &AMDGPU::SReg_64RegClass); + LLVM_DEBUG(dbgs() << "Lower PHI: " << *MI); - // Collect incoming values. - for (unsigned i = 1; i < MI.getNumOperands(); i += 2) { - assert(i + 1 < MI.getNumOperands()); - Register IncomingReg = MI.getOperand(i).getReg(); - MachineBasicBlock *IncomingMBB = MI.getOperand(i + 1).getMBB(); - MachineInstr *IncomingDef = MRI->getUniqueVRegDef(IncomingReg); + Register DstReg = MI->getOperand(0).getReg(); + MRI->setRegClass(DstReg, IsWave32 ? &AMDGPU::SReg_32RegClass + : &AMDGPU::SReg_64RegClass); - if (IncomingDef->getOpcode() == AMDGPU::COPY) { - IncomingReg = IncomingDef->getOperand(1).getReg(); - assert(isLaneMaskReg(IncomingReg) || isVreg1(IncomingReg)); - assert(!IncomingDef->getOperand(1).getSubReg()); - } else if (IncomingDef->getOpcode() == AMDGPU::IMPLICIT_DEF) { - continue; - } else { - assert(IncomingDef->isPHI() || PhiRegisters.count(IncomingReg)); - } + // Collect incoming values. + for (unsigned i = 1; i < MI->getNumOperands(); i += 2) { + assert(i + 1 < MI->getNumOperands()); + Register IncomingReg = MI->getOperand(i).getReg(); + MachineBasicBlock *IncomingMBB = MI->getOperand(i + 1).getMBB(); + MachineInstr *IncomingDef = MRI->getUniqueVRegDef(IncomingReg); - IncomingBlocks.push_back(IncomingMBB); - IncomingRegs.push_back(IncomingReg); + if (IncomingDef->getOpcode() == AMDGPU::COPY) { + IncomingReg = IncomingDef->getOperand(1).getReg(); + assert(isLaneMaskReg(IncomingReg) || isVreg1(IncomingReg)); + assert(!IncomingDef->getOperand(1).getSubReg()); + } else if (IncomingDef->getOpcode() == AMDGPU::IMPLICIT_DEF) { + continue; + } else { + assert(IncomingDef->isPHI() || PhiRegisters.count(IncomingReg)); } + IncomingBlocks.push_back(IncomingMBB); + IncomingRegs.push_back(IncomingReg); + } + #ifndef NDEBUG - PhiRegisters.insert(DstReg); + PhiRegisters.insert(DstReg); #endif - // Phis in a loop that are observed outside the loop receive a simple but - // conservatively correct treatment. - std::vector<MachineBasicBlock *> DomBlocks = {&MBB}; - for (MachineInstr &Use : MRI->use_instructions(DstReg)) - DomBlocks.push_back(Use.getParent()); - - MachineBasicBlock *PostDomBound = - PDT->findNearestCommonDominator(DomBlocks); - unsigned FoundLoopLevel = LF.findLoop(PostDomBound); - - SSAUpdater.Initialize(DstReg); + // Phis in a loop that are observed outside the loop receive a simple but + // conservatively correct treatment. + std::vector<MachineBasicBlock *> DomBlocks = {&MBB}; + for (MachineInstr &Use : MRI->use_instructions(DstReg)) + DomBlocks.push_back(Use.getParent()); - if (FoundLoopLevel) { - LF.addLoopEntries(FoundLoopLevel, SSAUpdater, IncomingBlocks); + MachineBasicBlock *PostDomBound = + PDT->findNearestCommonDominator(DomBlocks); + unsigned FoundLoopLevel = LF.findLoop(PostDomBound); - for (unsigned i = 0; i < IncomingRegs.size(); ++i) { - IncomingUpdated.push_back(createLaneMaskReg(*MF)); - SSAUpdater.AddAvailableValue(IncomingBlocks[i], - IncomingUpdated.back()); - } + SSAUpdater.Initialize(DstReg); - for (unsigned i = 0; i < IncomingRegs.size(); ++i) { - MachineBasicBlock &IMBB = *IncomingBlocks[i]; - buildMergeLaneMasks( - IMBB, getSaluInsertionAtEnd(IMBB), {}, IncomingUpdated[i], - SSAUpdater.GetValueInMiddleOfBlock(&IMBB), IncomingRegs[i]); - } - } else { - // The phi is not observed from outside a loop. Use a more accurate - // lowering. - PIA.analyze(MBB, IncomingBlocks); + if (FoundLoopLevel) { + LF.addLoopEntries(FoundLoopLevel, SSAUpdater, IncomingBlocks); - for (MachineBasicBlock *MBB : PIA.predecessors()) - SSAUpdater.AddAvailableValue(MBB, insertUndefLaneMask(*MBB)); + for (unsigned i = 0; i < IncomingRegs.size(); ++i) { + IncomingUpdated.push_back(createLaneMaskReg(*MF)); + SSAUpdater.AddAvailableValue(IncomingBlocks[i], + IncomingUpdated.back()); + } - for (unsigned i = 0; i < IncomingRegs.size(); ++i) { - MachineBasicBlock &IMBB = *IncomingBlocks[i]; - if (PIA.isSource(IMBB)) { - IncomingUpdated.push_back(0); - SSAUpdater.AddAvailableValue(&IMBB, IncomingRegs[i]); - } else { - IncomingUpdated.push_back(createLaneMaskReg(*MF)); - SSAUpdater.AddAvailableValue(&IMBB, IncomingUpdated.back()); - } - } + for (unsigned i = 0; i < IncomingRegs.size(); ++i) { + MachineBasicBlock &IMBB = *IncomingBlocks[i]; + buildMergeLaneMasks( + IMBB, getSaluInsertionAtEnd(IMBB), {}, IncomingUpdated[i], + SSAUpdater.GetValueInMiddleOfBlock(&IMBB), IncomingRegs[i]); + } + } else { + // The phi is not observed from outside a loop. Use a more accurate + // lowering. + PIA.analyze(MBB, IncomingBlocks); - for (unsigned i = 0; i < IncomingRegs.size(); ++i) { - if (!IncomingUpdated[i]) - continue; + for (MachineBasicBlock *MBB : PIA.predecessors()) + SSAUpdater.AddAvailableValue(MBB, insertUndefLaneMask(*MBB)); - MachineBasicBlock &IMBB = *IncomingBlocks[i]; - buildMergeLaneMasks( - IMBB, getSaluInsertionAtEnd(IMBB), {}, IncomingUpdated[i], - SSAUpdater.GetValueInMiddleOfBlock(&IMBB), IncomingRegs[i]); + for (unsigned i = 0; i < IncomingRegs.size(); ++i) { + MachineBasicBlock &IMBB = *IncomingBlocks[i]; + if (PIA.isSource(IMBB)) { + IncomingUpdated.push_back(0); + SSAUpdater.AddAvailableValue(&IMBB, IncomingRegs[i]); + } else { + IncomingUpdated.push_back(createLaneMaskReg(*MF)); + SSAUpdater.AddAvailableValue(&IMBB, IncomingUpdated.back()); } } - unsigned NewReg = SSAUpdater.GetValueInMiddleOfBlock(&MBB); - if (NewReg != DstReg) { - MRI->replaceRegWith(NewReg, DstReg); + for (unsigned i = 0; i < IncomingRegs.size(); ++i) { + if (!IncomingUpdated[i]) + continue; - // Ensure that DstReg has a single def and mark the old PHI node for - // deletion. - MI.getOperand(0).setReg(NewReg); - DeadPhis.push_back(&MI); + MachineBasicBlock &IMBB = *IncomingBlocks[i]; + buildMergeLaneMasks( + IMBB, getSaluInsertionAtEnd(IMBB), {}, IncomingUpdated[i], + SSAUpdater.GetValueInMiddleOfBlock(&IMBB), IncomingRegs[i]); } - - IncomingBlocks.clear(); - IncomingRegs.clear(); - IncomingUpdated.clear(); } - for (MachineInstr *MI : DeadPhis) + unsigned NewReg = SSAUpdater.GetValueInMiddleOfBlock(&MBB); + if (NewReg != DstReg) { + MRI->replaceRegWith(NewReg, DstReg); MI->eraseFromParent(); - DeadPhis.clear(); + } + + IncomingBlocks.clear(); + IncomingRegs.clear(); + IncomingUpdated.clear(); } } diff --git a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp index 714d403a3e8f..57ccf7641666 100644 --- a/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp +++ b/llvm/lib/Target/AMDGPU/SILowerSGPRSpills.cpp @@ -27,6 +27,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/InitializePasses.h" #include "llvm/Target/TargetMachine.h" using namespace llvm; diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp index 7dd0f11c95de..0c67b1467a5d 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp @@ -28,7 +28,6 @@ using namespace llvm; SIMachineFunctionInfo::SIMachineFunctionInfo(const MachineFunction &MF) : AMDGPUMachineFunction(MF), - Mode(MF.getFunction()), PrivateSegmentBuffer(false), DispatchPtr(false), QueuePtr(false), diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h index 7d70c786b594..ef0186f7d57f 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h +++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.h @@ -236,17 +236,23 @@ template <> struct MappingTraits<SIArgumentInfo> { struct SIMode { bool IEEE = true; bool DX10Clamp = true; + bool FP32Denormals = true; + bool FP64FP16Denormals = true; SIMode() = default; - SIMode(const AMDGPU::SIModeRegisterDefaults &Mode) { IEEE = Mode.IEEE; DX10Clamp = Mode.DX10Clamp; + FP32Denormals = Mode.FP32Denormals; + FP64FP16Denormals = Mode.FP64FP16Denormals; } bool operator ==(const SIMode Other) const { - return IEEE == Other.IEEE && DX10Clamp == Other.DX10Clamp; + return IEEE == Other.IEEE && + DX10Clamp == Other.DX10Clamp && + FP32Denormals == Other.FP32Denormals && + FP64FP16Denormals == Other.FP64FP16Denormals; } }; @@ -254,6 +260,8 @@ template <> struct MappingTraits<SIMode> { static void mapping(IO &YamlIO, SIMode &Mode) { YamlIO.mapOptional("ieee", Mode.IEEE, true); YamlIO.mapOptional("dx10-clamp", Mode.DX10Clamp, true); + YamlIO.mapOptional("fp32-denormals", Mode.FP32Denormals, true); + YamlIO.mapOptional("fp64-fp16-denormals", Mode.FP64FP16Denormals, true); } }; @@ -332,9 +340,6 @@ class SIMachineFunctionInfo final : public AMDGPUMachineFunction { AMDGPUFunctionArgInfo ArgInfo; - // State of MODE register, assumed FP mode. - AMDGPU::SIModeRegisterDefaults Mode; - // Graphics info. unsigned PSInputAddr = 0; unsigned PSInputEnable = 0; @@ -507,10 +512,6 @@ public: : I->second.Lanes[Lane]; } - AMDGPU::SIModeRegisterDefaults getMode() const { - return Mode; - } - bool haveFreeLanesForSGPRSpill(const MachineFunction &MF, unsigned NumLane) const; bool allocateSGPRSpillToVGPR(MachineFunction &MF, int FI); diff --git a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp index c072ba6b2d1c..004a3cb185d6 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp +++ b/llvm/lib/Target/AMDGPU/SIMachineScheduler.cpp @@ -609,13 +609,8 @@ void SIScheduleBlock::printDebug(bool full) { } dbgs() << "\nInstructions:\n"; - if (!Scheduled) { - for (const SUnit* SU : SUnits) + for (const SUnit* SU : SUnits) DAG->dumpNode(*SU); - } else { - for (const SUnit* SU : SUnits) - DAG->dumpNode(*SU); - } dbgs() << "///////////////////////\n"; } @@ -623,11 +618,8 @@ void SIScheduleBlock::printDebug(bool full) { // SIScheduleBlockCreator // -SIScheduleBlockCreator::SIScheduleBlockCreator(SIScheduleDAGMI *DAG) : -DAG(DAG) { -} - -SIScheduleBlockCreator::~SIScheduleBlockCreator() = default; +SIScheduleBlockCreator::SIScheduleBlockCreator(SIScheduleDAGMI *DAG) + : DAG(DAG) {} SIScheduleBlocks SIScheduleBlockCreator::getBlocks(SISchedulerBlockCreatorVariant BlockVariant) { diff --git a/llvm/lib/Target/AMDGPU/SIMachineScheduler.h b/llvm/lib/Target/AMDGPU/SIMachineScheduler.h index c28a7be4d03a..ec450a316467 100644 --- a/llvm/lib/Target/AMDGPU/SIMachineScheduler.h +++ b/llvm/lib/Target/AMDGPU/SIMachineScheduler.h @@ -248,7 +248,6 @@ class SIScheduleBlockCreator { public: SIScheduleBlockCreator(SIScheduleDAGMI *DAG); - ~SIScheduleBlockCreator(); SIScheduleBlocks getBlocks(SISchedulerBlockCreatorVariant BlockVariant); diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp index cc9b46a75582..a9717c6ffb70 100644 --- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp +++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMasking.cpp @@ -8,12 +8,13 @@ #include "AMDGPU.h" #include "AMDGPUSubtarget.h" -#include "SIInstrInfo.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "SIInstrInfo.h" #include "llvm/ADT/SmallSet.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/InitializePasses.h" #include "llvm/Support/Debug.h" using namespace llvm; @@ -56,7 +57,7 @@ char SIOptimizeExecMasking::ID = 0; char &llvm::SIOptimizeExecMaskingID = SIOptimizeExecMasking::ID; /// If \p MI is a copy from exec, return the register copied to. -static unsigned isCopyFromExec(const MachineInstr &MI, const GCNSubtarget &ST) { +static Register isCopyFromExec(const MachineInstr &MI, const GCNSubtarget &ST) { switch (MI.getOpcode()) { case AMDGPU::COPY: case AMDGPU::S_MOV_B64: @@ -74,7 +75,7 @@ static unsigned isCopyFromExec(const MachineInstr &MI, const GCNSubtarget &ST) { } /// If \p MI is a copy to exec, return the register copied from. -static unsigned isCopyToExec(const MachineInstr &MI, const GCNSubtarget &ST) { +static Register isCopyToExec(const MachineInstr &MI, const GCNSubtarget &ST) { switch (MI.getOpcode()) { case AMDGPU::COPY: case AMDGPU::S_MOV_B64: @@ -91,12 +92,12 @@ static unsigned isCopyToExec(const MachineInstr &MI, const GCNSubtarget &ST) { llvm_unreachable("should have been replaced"); } - return AMDGPU::NoRegister; + return Register(); } /// If \p MI is a logical operation on an exec value, /// return the register copied to. -static unsigned isLogicalOpOnExec(const MachineInstr &MI) { +static Register isLogicalOpOnExec(const MachineInstr &MI) { switch (MI.getOpcode()) { case AMDGPU::S_AND_B64: case AMDGPU::S_OR_B64: @@ -244,8 +245,8 @@ static MachineBasicBlock::reverse_iterator findExecCopy( auto E = MBB.rend(); for (unsigned N = 0; N <= InstLimit && I != E; ++I, ++N) { - unsigned CopyFromExec = isCopyFromExec(*I, ST); - if (CopyFromExec != AMDGPU::NoRegister) + Register CopyFromExec = isCopyFromExec(*I, ST); + if (CopyFromExec.isValid()) return I; } @@ -271,7 +272,7 @@ bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) { const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); const SIRegisterInfo *TRI = ST.getRegisterInfo(); const SIInstrInfo *TII = ST.getInstrInfo(); - unsigned Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; + MCRegister Exec = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; // Optimize sequences emitted for control flow lowering. They are originally // emitted as the separate operations because spill code may need to be @@ -290,8 +291,8 @@ bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) { if (I == E) continue; - unsigned CopyToExec = isCopyToExec(*I, ST); - if (CopyToExec == AMDGPU::NoRegister) + Register CopyToExec = isCopyToExec(*I, ST); + if (!CopyToExec.isValid()) continue; // Scan backwards to find the def. diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp index fdd30db6a7cb..34199d3e425c 100644 --- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp +++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp @@ -21,10 +21,11 @@ #include "AMDGPU.h" #include "AMDGPUSubtarget.h" -#include "SIInstrInfo.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "SIInstrInfo.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/InitializePasses.h" using namespace llvm; diff --git a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp index 9b3b2436475c..05c81feb23ec 100644 --- a/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp +++ b/llvm/lib/Target/AMDGPU/SIPeepholeSDWA.cpp @@ -26,6 +26,7 @@ #include "SIRegisterInfo.h" #include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "Utils/AMDGPUBaseInfo.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" @@ -73,8 +74,8 @@ private: const SIRegisterInfo *TRI; const SIInstrInfo *TII; - std::unordered_map<MachineInstr *, std::unique_ptr<SDWAOperand>> SDWAOperands; - std::unordered_map<MachineInstr *, SDWAOperandsVector> PotentialMatches; + MapVector<MachineInstr *, std::unique_ptr<SDWAOperand>> SDWAOperands; + MapVector<MachineInstr *, SDWAOperandsVector> PotentialMatches; SmallVector<MachineInstr *, 8> ConvertedInstructions; Optional<int64_t> foldToImm(const MachineOperand &Op) const; diff --git a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp index 6cdd12d0e7bd..09dfe8753792 100644 --- a/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp +++ b/llvm/lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp @@ -13,18 +13,19 @@ #include "AMDGPU.h" #include "AMDGPUSubtarget.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIInstrInfo.h" -#include "SIRegisterInfo.h" #include "SIMachineFunctionInfo.h" -#include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "SIRegisterInfo.h" #include "llvm/ADT/PostOrderIterator.h" -#include "llvm/CodeGen/VirtRegMap.h" #include "llvm/CodeGen/LiveInterval.h" #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveRegMatrix.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/RegisterClassInfo.h" +#include "llvm/CodeGen/VirtRegMap.h" +#include "llvm/InitializePasses.h" using namespace llvm; diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp index f58bc3060c42..fbadad3c84ad 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.cpp @@ -771,8 +771,6 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); - unsigned M0CopyReg = AMDGPU::NoRegister; - unsigned EltSize = 4; const TargetRegisterClass *RC = getPhysRegClass(SuperReg); @@ -850,11 +848,6 @@ bool SIRegisterInfo::spillSGPR(MachineBasicBlock::iterator MI, } } - if (M0CopyReg != AMDGPU::NoRegister) { - BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), AMDGPU::M0) - .addReg(M0CopyReg, RegState::Kill); - } - MI->eraseFromParent(); MFI->addToSpilledSGPRs(NumSubRegs); return true; @@ -882,8 +875,6 @@ bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, assert(SuperReg != AMDGPU::M0 && "m0 should never spill"); - unsigned M0CopyReg = AMDGPU::NoRegister; - unsigned EltSize = 4; const TargetRegisterClass *RC = getPhysRegClass(SuperReg); @@ -940,11 +931,6 @@ bool SIRegisterInfo::restoreSGPR(MachineBasicBlock::iterator MI, } } - if (M0CopyReg != AMDGPU::NoRegister) { - BuildMI(*MBB, MI, DL, TII->get(AMDGPU::COPY), AMDGPU::M0) - .addReg(M0CopyReg, RegState::Kill); - } - MI->eraseFromParent(); return true; } @@ -1137,11 +1123,15 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, if (!IsVOP2) MIB.addImm(0); // clamp bit } else { - Register ConstOffsetReg = - RS->scavengeRegister(&AMDGPU::SReg_32_XM0RegClass, MIB, 0, false); + assert(MIB->getOpcode() == AMDGPU::V_ADD_I32_e64 && + "Need to reuse carry out register"); - // This should always be able to use the unused carry out. - assert(ConstOffsetReg && "this scavenge should not be able to fail"); + // Use scavenged unused carry out as offset register. + Register ConstOffsetReg; + if (!isWave32) + ConstOffsetReg = getSubReg(MIB.getReg(1), AMDGPU::sub0); + else + ConstOffsetReg = MIB.getReg(1); BuildMI(*MBB, *MIB, DL, TII->get(AMDGPU::S_MOV_B32), ConstOffsetReg) .addImm(Offset); @@ -1150,10 +1140,9 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, MIB.addImm(0); // clamp bit } } else { - // We have to produce a carry out, and we there isn't a free SGPR - // pair for it. We can keep the whole computation on the SALU to - // avoid clobbering an additional register at the cost of an extra - // mov. + // We have to produce a carry out, and there isn't a free SGPR pair + // for it. We can keep the whole computation on the SALU to avoid + // clobbering an additional register at the cost of an extra mov. // We may have 1 free scratch SGPR even though a carry out is // unavailable. Only one additional mov is needed. @@ -1175,9 +1164,9 @@ void SIRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI, BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_SUB_U32), ScaledReg) .addReg(ScaledReg, RegState::Kill) .addImm(Offset); - BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) - .addReg(DiffReg, RegState::Kill) - .addImm(ST.getWavefrontSizeLog2()); + BuildMI(*MBB, MI, DL, TII->get(AMDGPU::S_LSHL_B32), ScaledReg) + .addReg(DiffReg, RegState::Kill) + .addImm(ST.getWavefrontSizeLog2()); } } } @@ -1786,14 +1775,6 @@ SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size, &AMDGPU::SReg_32_XM0_XEXECRegClass : &AMDGPU::SReg_64_XEXECRegClass; case AMDGPU::SGPRRegBankID: return &AMDGPU::SReg_32RegClass; - case AMDGPU::SCCRegBankID: - // This needs to return an allocatable class, so don't bother returning - // the dummy SCC class. - // - // FIXME: This is a grotesque hack. We use SGPR_32 as an indication this - // was not an VCC bank value since we use the larger class SReg_32 for - // other values. These should all use SReg_32. - return &AMDGPU::SGPR_32RegClass; default: llvm_unreachable("unknown register bank"); } @@ -1803,7 +1784,7 @@ SIRegisterInfo::getRegClassForSizeOnBank(unsigned Size, &AMDGPU::SReg_32RegClass; case 64: return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_64RegClass : - &AMDGPU::SReg_64_XEXECRegClass; + &AMDGPU::SReg_64RegClass; case 96: return RB.getID() == AMDGPU::VGPRRegBankID ? &AMDGPU::VReg_96RegClass : &AMDGPU::SReg_96RegClass; diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h index ac3dea1a1a28..ac8c56fa3a03 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.h +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.h @@ -144,6 +144,11 @@ public: return isSGPRClass(RC); } + /// \returns true if this class contains only AGPR registers + bool isAGPRClass(const TargetRegisterClass *RC) const { + return hasAGPRs(RC) && !hasVGPRs(RC); + } + /// \returns true if this class contains VGPR registers. bool hasVGPRs(const TargetRegisterClass *RC) const; diff --git a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td index 82219cbdf3b2..6ea6ec00e742 100644 --- a/llvm/lib/Target/AMDGPU/SIRegisterInfo.td +++ b/llvm/lib/Target/AMDGPU/SIRegisterInfo.td @@ -682,7 +682,21 @@ def AReg_1024 : RegisterClass<"AMDGPU", [v32i32, v32f32], 32, let AllocationPriority = 8; } -def VReg_1 : RegisterClass<"AMDGPU", [i1], 32, (add VGPR_32)> { + +// This is not a real register. This is just to have a register to add +// to VReg_1 that does not alias any real register that would +// introduce inferred register classess. +def ARTIFICIAL_VGPR : SIReg <"invalid vgpr", 0> { + let isArtificial = 1; +} + +// FIXME: Should specify an empty set for this. No register should +// ever be allocated using VReg_1. This is a hack for SelectionDAG +// that should always be lowered by SILowerI1Copies. TableGen crashes +// on an empty register set, but also sorts register classes based on +// the number of registerss in them. Add only one register so this is +// sorted to the end and not preferred over VGPR_32. +def VReg_1 : RegisterClass<"AMDGPU", [i1], 32, (add ARTIFICIAL_VGPR)> { let Size = 1; } diff --git a/llvm/lib/Target/AMDGPU/SIRemoveShortExecBranches.cpp b/llvm/lib/Target/AMDGPU/SIRemoveShortExecBranches.cpp new file mode 100644 index 000000000000..51779e97ac62 --- /dev/null +++ b/llvm/lib/Target/AMDGPU/SIRemoveShortExecBranches.cpp @@ -0,0 +1,158 @@ +//===-- SIRemoveShortExecBranches.cpp ------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// This pass optmizes the s_cbranch_execz instructions. +/// The pass removes this skip instruction for short branches, +/// if there is no unwanted sideeffect in the fallthrough code sequence. +/// +//===----------------------------------------------------------------------===// + +#include "AMDGPU.h" +#include "AMDGPUSubtarget.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" +#include "SIInstrInfo.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/Support/CommandLine.h" + +using namespace llvm; + +#define DEBUG_TYPE "si-remove-short-exec-branches" + +static unsigned SkipThreshold; + +static cl::opt<unsigned, true> SkipThresholdFlag( + "amdgpu-skip-threshold", cl::Hidden, + cl::desc( + "Number of instructions before jumping over divergent control flow"), + cl::location(SkipThreshold), cl::init(12)); + +namespace { + +class SIRemoveShortExecBranches : public MachineFunctionPass { +private: + const SIInstrInfo *TII = nullptr; + bool getBlockDestinations(MachineBasicBlock &SrcMBB, + MachineBasicBlock *&TrueMBB, + MachineBasicBlock *&FalseMBB, + SmallVectorImpl<MachineOperand> &Cond); + bool mustRetainExeczBranch(const MachineBasicBlock &From, + const MachineBasicBlock &To) const; + bool removeExeczBranch(MachineInstr &MI, MachineBasicBlock &SrcMBB); + +public: + static char ID; + + SIRemoveShortExecBranches() : MachineFunctionPass(ID) { + initializeSIRemoveShortExecBranchesPass(*PassRegistry::getPassRegistry()); + } + + bool runOnMachineFunction(MachineFunction &MF) override; +}; + +} // End anonymous namespace. + +INITIALIZE_PASS(SIRemoveShortExecBranches, DEBUG_TYPE, + "SI remove short exec branches", false, false) + +char SIRemoveShortExecBranches::ID = 0; + +char &llvm::SIRemoveShortExecBranchesID = SIRemoveShortExecBranches::ID; + +bool SIRemoveShortExecBranches::getBlockDestinations( + MachineBasicBlock &SrcMBB, MachineBasicBlock *&TrueMBB, + MachineBasicBlock *&FalseMBB, SmallVectorImpl<MachineOperand> &Cond) { + if (TII->analyzeBranch(SrcMBB, TrueMBB, FalseMBB, Cond)) + return false; + + if (!FalseMBB) + FalseMBB = SrcMBB.getNextNode(); + + return true; +} + +bool SIRemoveShortExecBranches::mustRetainExeczBranch( + const MachineBasicBlock &From, const MachineBasicBlock &To) const { + unsigned NumInstr = 0; + const MachineFunction *MF = From.getParent(); + + for (MachineFunction::const_iterator MBBI(&From), ToI(&To), End = MF->end(); + MBBI != End && MBBI != ToI; ++MBBI) { + const MachineBasicBlock &MBB = *MBBI; + + for (MachineBasicBlock::const_iterator I = MBB.begin(), E = MBB.end(); + I != E; ++I) { + // When a uniform loop is inside non-uniform control flow, the branch + // leaving the loop might be an S_CBRANCH_VCCNZ, which is never taken + // when EXEC = 0. We should skip the loop lest it becomes infinite. + if (I->getOpcode() == AMDGPU::S_CBRANCH_VCCNZ || + I->getOpcode() == AMDGPU::S_CBRANCH_VCCZ) + return true; + + if (TII->hasUnwantedEffectsWhenEXECEmpty(*I)) + return true; + + // These instructions are potentially expensive even if EXEC = 0. + if (TII->isSMRD(*I) || TII->isVMEM(*I) || TII->isFLAT(*I) || + I->getOpcode() == AMDGPU::S_WAITCNT) + return true; + + ++NumInstr; + if (NumInstr >= SkipThreshold) + return true; + } + } + + return false; +} + +// Returns true if the skip branch instruction is removed. +bool SIRemoveShortExecBranches::removeExeczBranch(MachineInstr &MI, + MachineBasicBlock &SrcMBB) { + MachineBasicBlock *TrueMBB = nullptr; + MachineBasicBlock *FalseMBB = nullptr; + SmallVector<MachineOperand, 1> Cond; + + if (!getBlockDestinations(SrcMBB, TrueMBB, FalseMBB, Cond)) + return false; + + // Consider only the forward branches. + if ((SrcMBB.getNumber() >= TrueMBB->getNumber()) || + mustRetainExeczBranch(*FalseMBB, *TrueMBB)) + return false; + + LLVM_DEBUG(dbgs() << "Removing the execz branch: " << MI); + MI.eraseFromParent(); + SrcMBB.removeSuccessor(TrueMBB); + + return true; +} + +bool SIRemoveShortExecBranches::runOnMachineFunction(MachineFunction &MF) { + const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); + TII = ST.getInstrInfo(); + MF.RenumberBlocks(); + bool Changed = false; + + for (MachineBasicBlock &MBB : MF) { + MachineBasicBlock::iterator MBBI = MBB.getFirstTerminator(); + if (MBBI == MBB.end()) + continue; + + MachineInstr &MI = *MBBI; + switch (MI.getOpcode()) { + case AMDGPU::S_CBRANCH_EXECZ: + Changed = removeExeczBranch(MI, MBB); + break; + default: + break; + } + } + + return Changed; +} diff --git a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp index 8afca2cdc325..3986ca6dfa81 100644 --- a/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp +++ b/llvm/lib/Target/AMDGPU/SIShrinkInstructions.cpp @@ -603,8 +603,10 @@ bool SIShrinkInstructions::runOnMachineFunction(MachineFunction &MF) { // => // s_nop (N + M) if (MI.getOpcode() == AMDGPU::S_NOP && + MI.getNumOperands() == 1 && // Don't merge with implicit operands Next != MBB.end() && - (*Next).getOpcode() == AMDGPU::S_NOP) { + (*Next).getOpcode() == AMDGPU::S_NOP && + (*Next).getNumOperands() == 1) { MachineInstr &NextMI = *Next; // The instruction encodes the amount to wait with an offset of 1, diff --git a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp index cb4cf68d709a..39f5df767977 100644 --- a/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp +++ b/llvm/lib/Target/AMDGPU/SIWholeQuadMode.cpp @@ -57,9 +57,9 @@ #include "AMDGPU.h" #include "AMDGPUSubtarget.h" +#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "SIInstrInfo.h" #include "SIMachineFunctionInfo.h" -#include "MCTargetDesc/AMDGPUMCTargetDesc.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SmallVector.h" @@ -77,6 +77,7 @@ #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/DebugLoc.h" +#include "llvm/InitializePasses.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" @@ -155,6 +156,7 @@ private: DenseMap<const MachineInstr *, InstrInfo> Instructions; DenseMap<MachineBasicBlock *, BlockInfo> Blocks; SmallVector<MachineInstr *, 1> LiveMaskQueries; + SmallVector<MachineInstr *, 4> LowerToMovInstrs; SmallVector<MachineInstr *, 4> LowerToCopyInstrs; void printInfo(); @@ -351,7 +353,7 @@ char SIWholeQuadMode::scanInstructions(MachineFunction &MF, // inactive lanes. markInstructionUses(MI, StateWWM, Worklist); GlobalFlags |= StateWWM; - LowerToCopyInstrs.push_back(&MI); + LowerToMovInstrs.push_back(&MI); continue; } else if (Opcode == AMDGPU::V_SET_INACTIVE_B32 || Opcode == AMDGPU::V_SET_INACTIVE_B64) { @@ -851,9 +853,8 @@ void SIWholeQuadMode::lowerLiveMaskQueries(unsigned LiveMaskReg) { } void SIWholeQuadMode::lowerCopyInstrs() { - for (MachineInstr *MI : LowerToCopyInstrs) { - for (unsigned i = MI->getNumExplicitOperands() - 1; i > 1; i--) - MI->RemoveOperand(i); + for (MachineInstr *MI : LowerToMovInstrs) { + assert(MI->getNumExplicitOperands() == 2); const Register Reg = MI->getOperand(0).getReg(); @@ -871,6 +872,22 @@ void SIWholeQuadMode::lowerCopyInstrs() { MI->setDesc(TII->get(AMDGPU::COPY)); } } + for (MachineInstr *MI : LowerToCopyInstrs) { + if (MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B32 || + MI->getOpcode() == AMDGPU::V_SET_INACTIVE_B64) { + assert(MI->getNumExplicitOperands() == 3); + // the only reason we should be here is V_SET_INACTIVE has + // an undef input so it is being replaced by a simple copy. + // There should be a second undef source that we should remove. + assert(MI->getOperand(2).isUndef()); + MI->RemoveOperand(2); + MI->untieRegOperand(1); + } else { + assert(MI->getNumExplicitOperands() == 2); + } + + MI->setDesc(TII->get(AMDGPU::COPY)); + } } bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { @@ -878,6 +895,7 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { Blocks.clear(); LiveMaskQueries.clear(); LowerToCopyInstrs.clear(); + LowerToMovInstrs.clear(); CallingConv = MF.getFunction().getCallingConv(); ST = &MF.getSubtarget<GCNSubtarget>(); @@ -892,7 +910,7 @@ bool SIWholeQuadMode::runOnMachineFunction(MachineFunction &MF) { unsigned Exec = ST->isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC; if (!(GlobalFlags & StateWQM)) { lowerLiveMaskQueries(Exec); - if (!(GlobalFlags & StateWWM) && LowerToCopyInstrs.empty()) + if (!(GlobalFlags & StateWWM) && LowerToCopyInstrs.empty() && LowerToMovInstrs.empty()) return !LiveMaskQueries.empty(); } else { // Store a copy of the original live mask when required diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td index 1a74ebbf8165..79982d96c2c8 100644 --- a/llvm/lib/Target/AMDGPU/SMInstructions.td +++ b/llvm/lib/Target/AMDGPU/SMInstructions.td @@ -417,7 +417,7 @@ class SMRD_Real_si <bits<5> op, SM_Pseudo ps> , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> , Enc32 { - let AssemblerPredicates = [isGFX6GFX7]; + let AssemblerPredicate = isGFX6GFX7; let DecoderNamespace = "GFX6GFX7"; let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); @@ -471,7 +471,7 @@ class SMEM_Real_vi <bits<8> op, SM_Pseudo ps> , Enc64 { bit glc; - let AssemblerPredicates = [isGFX8GFX9]; + let AssemblerPredicate = isGFX8GFX9; let DecoderNamespace = "GFX8"; let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); @@ -660,7 +660,7 @@ class SMRD_Real_Load_IMM_ci <bits<5> op, SM_Load_Pseudo ps> : SM_Real<ps>, Enc64 { - let AssemblerPredicates = [isGFX7Only]; + let AssemblerPredicate = isGFX7Only; let DecoderNamespace = "GFX7"; let InOperandList = (ins ps.BaseClass:$sbase, smrd_literal_offset:$offset, GLC:$glc, DLC:$dlc); @@ -697,7 +697,7 @@ class SMRD_Real_ci <bits<5> op, SM_Pseudo ps> , SIMCInstr<ps.PseudoInstr, SIEncodingFamily.SI> , Enc32 { - let AssemblerPredicates = [isGFX7Only]; + let AssemblerPredicate = isGFX7Only; let DecoderNamespace = "GFX7"; let Inst{7-0} = !if(ps.has_offset, offset{7-0}, ?); @@ -835,7 +835,7 @@ class SMEM_Real_gfx10<bits<8> op, SM_Pseudo ps> : bit glc; bit dlc; - let AssemblerPredicates = [isGFX10Plus]; + let AssemblerPredicate = isGFX10Plus; let DecoderNamespace = "GFX10"; let Inst{5-0} = !if(ps.has_sbase, sbase{6-1}, ?); diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td index d31a49f428ee..73ba2ae367f7 100644 --- a/llvm/lib/Target/AMDGPU/SOPInstructions.td +++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td @@ -85,6 +85,11 @@ class SOP1_32 <string opName, list<dag> pattern=[], bit tied_in = 0> : SOP1_Pseu let Constraints = !if(tied_in, "$sdst = $sdst_in", ""); } +// Only register input allowed. +class SOP1_32R <string opName, list<dag> pattern=[]> : SOP1_Pseudo < + opName, (outs SReg_32:$sdst), (ins SReg_32:$src0), + "$sdst, $src0", pattern>; + // 32-bit input, no output. class SOP1_0_32 <string opName, list<dag> pattern = []> : SOP1_Pseudo < opName, (outs), (ins SSrc_b32:$src0), @@ -103,6 +108,12 @@ class SOP1_64 <string opName, list<dag> pattern=[]> : SOP1_Pseudo < "$sdst, $src0", pattern >; +// Only register input allowed. +class SOP1_64R <string opName, list<dag> pattern=[]> : SOP1_Pseudo < + opName, (outs SReg_64:$sdst), (ins SReg_64:$src0), + "$sdst, $src0", pattern +>; + // 64-bit input, 32-bit output. class SOP1_32_64 <string opName, list<dag> pattern=[]> : SOP1_Pseudo < opName, (outs SReg_32:$sdst), (ins SSrc_b64:$src0), @@ -159,14 +170,14 @@ let Defs = [SCC] in { let WaveSizePredicate = isWave32 in { def : GCNPat < (int_amdgcn_wqm_vote i1:$src0), - (S_WQM_B32 $src0) + (S_WQM_B32 SSrc_b32:$src0) >; } let WaveSizePredicate = isWave64 in { def : GCNPat < (int_amdgcn_wqm_vote i1:$src0), - (S_WQM_B64 $src0) + (S_WQM_B64 SSrc_b64:$src0) >; } @@ -254,8 +265,8 @@ def S_QUADMASK_B32 : SOP1_32 <"s_quadmask_b32">; def S_QUADMASK_B64 : SOP1_64 <"s_quadmask_b64">; let Uses = [M0] in { -def S_MOVRELS_B32 : SOP1_32 <"s_movrels_b32">; -def S_MOVRELS_B64 : SOP1_64 <"s_movrels_b64">; +def S_MOVRELS_B32 : SOP1_32R <"s_movrels_b32">; +def S_MOVRELS_B64 : SOP1_64R <"s_movrels_b64">; def S_MOVRELD_B32 : SOP1_32 <"s_movreld_b32">; def S_MOVRELD_B64 : SOP1_64 <"s_movreld_b64">; } // End Uses = [M0] @@ -1202,7 +1213,7 @@ def : GCNPat < def : GCNPat < (i32 (smax i32:$x, (i32 (ineg i32:$x)))), - (S_ABS_I32 $x) + (S_ABS_I32 SReg_32:$x) >; def : GCNPat < @@ -1503,7 +1514,7 @@ defm S_SETREG_IMM32_B32 : SOPK_Real64_gfx6_gfx7_gfx10<0x015>; class Select_vi<string opName> : SIMCInstr<opName, SIEncodingFamily.VI> { - list<Predicate> AssemblerPredicates = [isGFX8GFX9]; + Predicate AssemblerPredicate = isGFX8GFX9; string DecoderNamespace = "GFX8"; } diff --git a/llvm/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp b/llvm/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp index 30cf12337c6e..9ec437760c0a 100644 --- a/llvm/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp +++ b/llvm/lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp @@ -28,7 +28,7 @@ Target &llvm::getTheGCNTarget() { } /// Extern function to initialize the targets for the AMDGPU backend -extern "C" void LLVMInitializeAMDGPUTargetInfo() { +extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTargetInfo() { RegisterTarget<Triple::r600, false> R600(getTheAMDGPUTarget(), "r600", "AMD GPUs HD2XXX-HD6XXX", "AMDGPU"); RegisterTarget<Triple::amdgcn, false> GCN(getTheGCNTarget(), "amdgcn", diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp index afb2fd987afd..5271bc3aacc6 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp @@ -7,10 +7,10 @@ //===----------------------------------------------------------------------===// #include "AMDGPUBaseInfo.h" -#include "AMDGPUTargetTransformInfo.h" #include "AMDGPU.h" -#include "SIDefines.h" #include "AMDGPUAsmUtils.h" +#include "AMDGPUTargetTransformInfo.h" +#include "SIDefines.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/BinaryFormat/ELF.h" @@ -20,6 +20,8 @@ #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/Instruction.h" +#include "llvm/IR/IntrinsicsAMDGPU.h" +#include "llvm/IR/IntrinsicsR600.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/MC/MCContext.h" @@ -312,7 +314,8 @@ unsigned getMinFlatWorkGroupSize(const MCSubtargetInfo *STI) { } unsigned getMaxFlatWorkGroupSize(const MCSubtargetInfo *STI) { - return 2048; + // Some subtargets allow encoding 2048, but this isn't tested or supported. + return 1024; } unsigned getWavesPerWorkGroup(const MCSubtargetInfo *STI, @@ -542,16 +545,17 @@ amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor( } bool isGroupSegment(const GlobalValue *GV) { - return GV->getType()->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; + return GV->getAddressSpace() == AMDGPUAS::LOCAL_ADDRESS; } bool isGlobalSegment(const GlobalValue *GV) { - return GV->getType()->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; + return GV->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; } bool isReadOnlySegment(const GlobalValue *GV) { - return GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS || - GV->getType()->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT; + unsigned AS = GV->getAddressSpace(); + return AS == AMDGPUAS::CONSTANT_ADDRESS || + AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT; } bool shouldEmitConstantsToTextSection(const Triple &TT) { @@ -1301,7 +1305,8 @@ bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, return true; } -SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { +SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F, + const GCNSubtarget &ST) { *this = getDefaultForCallingConv(F.getCallingConv()); StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString(); @@ -1312,6 +1317,9 @@ SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) { = F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString(); if (!DX10ClampAttr.empty()) DX10Clamp = DX10ClampAttr == "true"; + + FP32Denormals = ST.hasFP32Denormals(F); + FP64FP16Denormals = ST.hasFP64FP16Denormals(F); } namespace { @@ -1322,6 +1330,8 @@ struct SourceOfDivergence { const SourceOfDivergence *lookupSourceOfDivergence(unsigned Intr); #define GET_SourcesOfDivergence_IMPL +#define GET_Gfx9BufferFormat_IMPL +#define GET_Gfx10PlusBufferFormat_IMPL #include "AMDGPUGenSearchableTables.inc" } // end anonymous namespace @@ -1330,5 +1340,21 @@ bool isIntrinsicSourceOfDivergence(unsigned IntrID) { return lookupSourceOfDivergence(IntrID); } +const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, + uint8_t NumComponents, + uint8_t NumFormat, + const MCSubtargetInfo &STI) { + return isGFX10(STI) + ? getGfx10PlusBufferFormatInfo(BitsPerComp, NumComponents, + NumFormat) + : getGfx9BufferFormatInfo(BitsPerComp, NumComponents, NumFormat); +} + +const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, + const MCSubtargetInfo &STI) { + return isGFX10(STI) ? getGfx10PlusBufferFormatInfo(Format) + : getGfx9BufferFormatInfo(Format); +} + } // namespace AMDGPU } // namespace llvm diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h index f78dadd447ff..a5bada2890d2 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h @@ -41,6 +41,14 @@ class Triple; namespace AMDGPU { +struct GcnBufferFormatInfo { + unsigned Format; + unsigned BitsPerComp; + unsigned NumComponents; + unsigned NumFormat; + unsigned DataFormat; +}; + #define GET_MIMGBaseOpcode_DECL #define GET_MIMGDim_DECL #define GET_MIMGEncoding_DECL @@ -300,6 +308,15 @@ LLVM_READONLY bool getMUBUFHasSoffset(unsigned Opc); LLVM_READONLY +const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t BitsPerComp, + uint8_t NumComponents, + uint8_t NumFormat, + const MCSubtargetInfo &STI); +LLVM_READONLY +const GcnBufferFormatInfo *getGcnBufferFormatInfo(uint8_t Format, + const MCSubtargetInfo &STI); + +LLVM_READONLY int getMCOpcode(uint16_t Opcode, unsigned Gen); void initDefaultAMDKernelCodeT(amd_kernel_code_t &Header, @@ -646,7 +663,6 @@ bool splitMUBUFOffset(uint32_t Imm, uint32_t &SOffset, uint32_t &ImmOffset, /// \returns true if the intrinsic is divergent bool isIntrinsicSourceOfDivergence(unsigned IntrID); - // Track defaults for fields in the MODE registser. struct SIModeRegisterDefaults { /// Floating point opcodes that support exception flag gathering quiet and @@ -659,29 +675,60 @@ struct SIModeRegisterDefaults { /// clamp NaN to zero; otherwise, pass NaN through. bool DX10Clamp : 1; - // TODO: FP mode fields + /// If this is set, neither input or output denormals are flushed for most f32 + /// instructions. + /// + /// TODO: Split into separate input and output fields if necessary like the + /// control bits really provide? + bool FP32Denormals : 1; + + /// If this is set, neither input or output denormals are flushed for both f64 + /// and f16/v2f16 instructions. + bool FP64FP16Denormals : 1; SIModeRegisterDefaults() : IEEE(true), - DX10Clamp(true) {} + DX10Clamp(true), + FP32Denormals(true), + FP64FP16Denormals(true) {} - SIModeRegisterDefaults(const Function &F); + // FIXME: Should not depend on the subtarget + SIModeRegisterDefaults(const Function &F, const GCNSubtarget &ST); static SIModeRegisterDefaults getDefaultForCallingConv(CallingConv::ID CC) { + const bool IsCompute = AMDGPU::isCompute(CC); + SIModeRegisterDefaults Mode; Mode.DX10Clamp = true; - Mode.IEEE = AMDGPU::isCompute(CC); + Mode.IEEE = IsCompute; + Mode.FP32Denormals = false; // FIXME: Should be on by default. + Mode.FP64FP16Denormals = true; return Mode; } bool operator ==(const SIModeRegisterDefaults Other) const { - return IEEE == Other.IEEE && DX10Clamp == Other.DX10Clamp; + return IEEE == Other.IEEE && DX10Clamp == Other.DX10Clamp && + FP32Denormals == Other.FP32Denormals && + FP64FP16Denormals == Other.FP64FP16Denormals; + } + + /// Returns true if a flag is compatible if it's enabled in the callee, but + /// disabled in the caller. + static bool oneWayCompatible(bool CallerMode, bool CalleeMode) { + return CallerMode == CalleeMode || (CallerMode && !CalleeMode); } // FIXME: Inlining should be OK for dx10-clamp, since the caller's mode should // be able to override. bool isInlineCompatible(SIModeRegisterDefaults CalleeMode) const { - return *this == CalleeMode; + if (DX10Clamp != CalleeMode.DX10Clamp) + return false; + if (IEEE != CalleeMode.IEEE) + return false; + + // Allow inlining denormals enabled into denormals flushed functions. + return oneWayCompatible(FP64FP16Denormals, CalleeMode.FP64FP16Denormals) && + oneWayCompatible(FP32Denormals, CalleeMode.FP32Denormals); } }; diff --git a/llvm/lib/Target/AMDGPU/VOP1Instructions.td b/llvm/lib/Target/AMDGPU/VOP1Instructions.td index f1cdc3097dc0..c7aed0985540 100644 --- a/llvm/lib/Target/AMDGPU/VOP1Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP1Instructions.td @@ -156,7 +156,7 @@ def V_READFIRSTLANE_B32 : InstSI <(outs SReg_32:$vdst), (ins VRegOrLds_32:$src0), "v_readfirstlane_b32 $vdst, $src0", - [(set i32:$vdst, (int_amdgcn_readfirstlane i32:$src0))]>, + [(set i32:$vdst, (int_amdgcn_readfirstlane (i32 VRegOrLds_32:$src0)))]>, Enc32 { let isCodeGenOnly = 0; @@ -260,62 +260,58 @@ defm V_CLREXCP : VOP1Inst <"v_clrexcp", VOP_NO_EXT<VOP_NONE>>; } // Restrict src0 to be VGPR -def VOP_I32_VI32_NO_EXT : VOPProfile<[i32, i32, untyped, untyped]> { +def VOP_MOVRELS : VOPProfile<[i32, i32, untyped, untyped]> { let Src0RC32 = VRegSrc_32; let Src0RC64 = VRegSrc_32; - - let HasExt = 0; - let HasExtDPP = 0; - let HasExtSDWA = 0; - let HasExtSDWA9 = 0; } // Special case because there are no true output operands. Hack vdst // to be a src operand. The custom inserter must add a tied implicit // def and use of the super register since there seems to be no way to // add an implicit def of a virtual register in tablegen. -def VOP_MOVRELD : VOPProfile<[untyped, i32, untyped, untyped]> { +class VOP_MOVREL<RegisterOperand Src1RC> : VOPProfile<[untyped, i32, untyped, untyped]> { let Src0RC32 = VOPDstOperand<VGPR_32>; let Src0RC64 = VOPDstOperand<VGPR_32>; let Outs = (outs); - let Ins32 = (ins Src0RC32:$vdst, VSrc_b32:$src0); - let Ins64 = (ins Src0RC64:$vdst, VSrc_b32:$src0); - let InsDPP = (ins DstRC:$vdst, DstRC:$old, Src0RC32:$src0, - dpp_ctrl:$dpp_ctrl, row_mask:$row_mask, - bank_mask:$bank_mask, bound_ctrl:$bound_ctrl); - let InsDPP16 = !con(InsDPP, (ins FI:$fi)); + let Ins32 = (ins Src0RC32:$vdst, Src1RC:$src0); + let Ins64 = (ins Src0RC64:$vdst, Src1RC:$src0); + let Asm32 = getAsm32<1, 1>.ret; + let Asm64 = getAsm64<1, 1, 0, 0, 1>.ret; - let InsSDWA = (ins Src0RC32:$vdst, Src0ModSDWA:$src0_modifiers, Src0SDWA:$src0, - clampmod:$clamp, omod:$omod, dst_sel:$dst_sel, dst_unused:$dst_unused, + let OutsSDWA = (outs Src0RC32:$vdst); + let InsSDWA = (ins Src0ModSDWA:$src0_modifiers, Src0SDWA:$src0, + clampmod:$clamp, dst_sel:$dst_sel, dst_unused:$dst_unused, src0_sel:$src0_sel); + let AsmSDWA9 = getAsmSDWA9<1, 0, 1>.ret; - let Asm32 = getAsm32<1, 1>.ret; - let Asm64 = getAsm64<1, 1, 0, 0, 1>.ret; - let AsmDPP = getAsmDPP<1, 1, 0>.ret; + let OutsDPP = (outs Src0RC32:$vdst); + let InsDPP16 = (ins Src0RC32:$old, Src0RC32:$src0, + dpp_ctrl:$dpp_ctrl, row_mask:$row_mask, + bank_mask:$bank_mask, bound_ctrl:$bound_ctrl, FI:$fi); let AsmDPP16 = getAsmDPP16<1, 1, 0>.ret; - let AsmSDWA = getAsmSDWA<1, 1>.ret; - let AsmSDWA9 = getAsmSDWA9<1, 0, 1>.ret; - let HasExt = 0; - let HasExtDPP = 0; - let HasExtSDWA = 0; - let HasExtSDWA9 = 0; + let OutsDPP8 = (outs Src0RC32:$vdst); + let InsDPP8 = (ins Src0RC32:$old, Src0RC32:$src0, dpp8:$dpp8, FI:$fi); + let AsmDPP8 = getAsmDPP8<1, 1, 0>.ret; let HasDst = 0; let EmitDst = 1; // force vdst emission } +def VOP_MOVRELD : VOP_MOVREL<VSrc_b32>; +def VOP_MOVRELSD : VOP_MOVREL<VRegSrc_32>; + let SubtargetPredicate = HasMovrel, Uses = [M0, EXEC] in { -// v_movreld_b32 is a special case because the destination output + // v_movreld_b32 is a special case because the destination output // register is really a source. It isn't actually read (but may be // written), and is only to provide the base register to start // indexing from. Tablegen seems to not let you define an implicit // virtual register output for the super register being written into, // so this must have an implicit def of the register added to it. defm V_MOVRELD_B32 : VOP1Inst <"v_movreld_b32", VOP_MOVRELD>; -defm V_MOVRELS_B32 : VOP1Inst <"v_movrels_b32", VOP_I32_VI32_NO_EXT>; -defm V_MOVRELSD_B32 : VOP1Inst <"v_movrelsd_b32", VOP_NO_EXT<VOP_I32_I32>>; +defm V_MOVRELS_B32 : VOP1Inst <"v_movrels_b32", VOP_MOVRELS>; +defm V_MOVRELSD_B32 : VOP1Inst <"v_movrelsd_b32", VOP_MOVRELSD>; } // End Uses = [M0, EXEC] defm V_MOV_FED_B32 : VOP1Inst <"v_mov_fed_b32", VOP_I32_I32>; @@ -430,9 +426,8 @@ let SubtargetPredicate = isGFX10Plus in { defm V_PIPEFLUSH : VOP1Inst<"v_pipeflush", VOP_NONE>; let Uses = [M0] in { - // FIXME-GFX10: Should V_MOVRELSD_2_B32 be VOP_NO_EXT? defm V_MOVRELSD_2_B32 : - VOP1Inst<"v_movrelsd_2_b32", VOP_NO_EXT<VOP_I32_I32>>; + VOP1Inst<"v_movrelsd_2_b32", VOP_MOVRELSD>; def V_SWAPREL_B32 : VOP1_Pseudo<"v_swaprel_b32", VOP_SWAP_I32, [], 1> { let Constraints = "$vdst = $src1, $vdst1 = $src0"; @@ -526,16 +521,10 @@ let AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" in { } } // End AssemblerPredicate = isGFX10Plus, DecoderNamespace = "GFX10" -multiclass VOP1_Real_gfx10_no_dpp<bits<9> op> : - VOP1_Real_e32_gfx10<op>, VOP1_Real_e64_gfx10<op>, - VOP1_Real_sdwa_gfx10<op>; - -multiclass VOP1_Real_gfx10_no_dpp8<bits<9> op> : - VOP1_Real_e32_gfx10<op>, VOP1_Real_e64_gfx10<op>, - VOP1_Real_sdwa_gfx10<op>, VOP1_Real_dpp_gfx10<op>; - multiclass VOP1_Real_gfx10<bits<9> op> : - VOP1_Real_gfx10_no_dpp8<op>, VOP1_Real_dpp8_gfx10<op>; + VOP1_Real_e32_gfx10<op>, VOP1_Real_e64_gfx10<op>, + VOP1_Real_sdwa_gfx10<op>, VOP1_Real_dpp_gfx10<op>, + VOP1_Real_dpp8_gfx10<op>; defm V_PIPEFLUSH : VOP1_Real_gfx10<0x01b>; defm V_MOVRELSD_2_B32 : VOP1_Real_gfx10<0x048>; @@ -618,12 +607,6 @@ multiclass VOP1_Real_gfx6_gfx7<bits<9> op> : multiclass VOP1_Real_gfx6_gfx7_gfx10<bits<9> op> : VOP1_Real_gfx6_gfx7<op>, VOP1_Real_gfx10<op>; -multiclass VOP1_Real_gfx6_gfx7_gfx10_no_dpp8<bits<9> op> : - VOP1_Real_gfx6_gfx7<op>, VOP1_Real_gfx10_no_dpp8<op>; - -multiclass VOP1_Real_gfx6_gfx7_gfx10_no_dpp<bits<9> op> : - VOP1_Real_gfx6_gfx7<op>, VOP1_Real_gfx10_no_dpp<op>; - defm V_LOG_CLAMP_F32 : VOP1_Real_gfx6_gfx7<0x026>; defm V_RCP_CLAMP_F32 : VOP1_Real_gfx6_gfx7<0x028>; defm V_RCP_LEGACY_F32 : VOP1_Real_gfx6_gfx7<0x029>; @@ -681,9 +664,9 @@ defm V_FRACT_F64 : VOP1_Real_gfx6_gfx7_gfx10<0x03e>; defm V_FREXP_EXP_I32_F32 : VOP1_Real_gfx6_gfx7_gfx10<0x03f>; defm V_FREXP_MANT_F32 : VOP1_Real_gfx6_gfx7_gfx10<0x040>; defm V_CLREXCP : VOP1_Real_gfx6_gfx7_gfx10<0x041>; -defm V_MOVRELD_B32 : VOP1_Real_gfx6_gfx7_gfx10_no_dpp<0x042>; -defm V_MOVRELS_B32 : VOP1_Real_gfx6_gfx7_gfx10_no_dpp8<0x043>; -defm V_MOVRELSD_B32 : VOP1_Real_gfx6_gfx7_gfx10_no_dpp8<0x044>; +defm V_MOVRELD_B32 : VOP1_Real_gfx6_gfx7_gfx10<0x042>; +defm V_MOVRELS_B32 : VOP1_Real_gfx6_gfx7_gfx10<0x043>; +defm V_MOVRELSD_B32 : VOP1_Real_gfx6_gfx7_gfx10<0x044>; //===----------------------------------------------------------------------===// // GFX8, GFX9 (VI). @@ -699,7 +682,7 @@ class VOP1_DPPe <bits<8> op, VOP1_DPP_Pseudo ps, VOPProfile P = ps.Pfl> : } multiclass VOP1Only_Real_vi <bits<10> op> { - let AssemblerPredicates = [isGFX8GFX9], DecoderNamespace = "GFX8" in { + let AssemblerPredicate = isGFX8GFX9, DecoderNamespace = "GFX8" in { def _vi : VOP1_Real<!cast<VOP1_Pseudo>(NAME), SIEncodingFamily.VI>, VOP1e<op{7-0}, !cast<VOP1_Pseudo>(NAME).Pfl>; @@ -707,7 +690,7 @@ multiclass VOP1Only_Real_vi <bits<10> op> { } multiclass VOP1_Real_e32e64_vi <bits<10> op> { - let AssemblerPredicates = [isGFX8GFX9], DecoderNamespace = "GFX8" in { + let AssemblerPredicate = isGFX8GFX9, DecoderNamespace = "GFX8" in { def _e32_vi : VOP1_Real<!cast<VOP1_Pseudo>(NAME#"_e32"), SIEncodingFamily.VI>, VOP1e<op{7-0}, !cast<VOP1_Pseudo>(NAME#"_e32").Pfl>; @@ -899,7 +882,7 @@ def : GCNPat < //===----------------------------------------------------------------------===// multiclass VOP1_Real_gfx9 <bits<10> op> { - let AssemblerPredicates = [isGFX9Only], DecoderNamespace = "GFX9" in { + let AssemblerPredicate = isGFX9Only, DecoderNamespace = "GFX9" in { defm NAME : VOP1_Real_e32e64_vi <op>; } diff --git a/llvm/lib/Target/AMDGPU/VOP2Instructions.td b/llvm/lib/Target/AMDGPU/VOP2Instructions.td index 1ab0fc1ab58d..aaadc3dbc721 100644 --- a/llvm/lib/Target/AMDGPU/VOP2Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP2Instructions.td @@ -467,9 +467,9 @@ defm V_SUB_F32 : VOP2Inst <"v_sub_f32", VOP_F32_F32_F32, fsub>; defm V_SUBREV_F32 : VOP2Inst <"v_subrev_f32", VOP_F32_F32_F32, null_frag, "v_sub_f32">; defm V_MUL_LEGACY_F32 : VOP2Inst <"v_mul_legacy_f32", VOP_F32_F32_F32, AMDGPUfmul_legacy>; defm V_MUL_F32 : VOP2Inst <"v_mul_f32", VOP_F32_F32_F32, fmul>; -defm V_MUL_I32_I24 : VOP2Inst <"v_mul_i32_i24", VOP_PAT_GEN<VOP_I32_I32_I32, 2>, AMDGPUmul_i24>; +defm V_MUL_I32_I24 : VOP2Inst <"v_mul_i32_i24", VOP_I32_I32_I32, AMDGPUmul_i24>; defm V_MUL_HI_I32_I24 : VOP2Inst <"v_mul_hi_i32_i24", VOP_PAT_GEN<VOP_I32_I32_I32, 2>, AMDGPUmulhi_i24>; -defm V_MUL_U32_U24 : VOP2Inst <"v_mul_u32_u24", VOP_PAT_GEN<VOP_I32_I32_I32, 2>, AMDGPUmul_u24>; +defm V_MUL_U32_U24 : VOP2Inst <"v_mul_u32_u24", VOP_I32_I32_I32, AMDGPUmul_u24>; defm V_MUL_HI_U32_U24 : VOP2Inst <"v_mul_hi_u32_u24", VOP_PAT_GEN<VOP_I32_I32_I32, 2>, AMDGPUmulhi_u24>; defm V_MIN_F32 : VOP2Inst <"v_min_f32", VOP_F32_F32_F32, fminnum_like>; defm V_MAX_F32 : VOP2Inst <"v_max_f32", VOP_F32_F32_F32, fmaxnum_like>; @@ -729,7 +729,7 @@ multiclass Arithmetic_i16_0Hi_Pats <SDPatternOperator op, Instruction inst> { def : GCNPat< (i32 (zext (op i16:$src0, i16:$src1))), - (inst $src0, $src1) + (inst VSrc_b16:$src0, VSrc_b16:$src1) >; def : GCNPat< @@ -766,7 +766,22 @@ def : GCNPat < let Predicates = [Has16BitInsts] in { +// Undo sub x, c -> add x, -c canonicalization since c is more likely +// an inline immediate than -c. +// TODO: Also do for 64-bit. +def : GCNPat< + (add i16:$src0, (i16 NegSubInlineConst16:$src1)), + (V_SUB_U16_e64 VSrc_b16:$src0, NegSubInlineConst16:$src1) +>; + + let Predicates = [Has16BitInsts, isGFX7GFX8GFX9] in { + +def : GCNPat< + (i32 (zext (add i16:$src0, (i16 NegSubInlineConst16:$src1)))), + (V_SUB_U16_e64 VSrc_b16:$src0, NegSubInlineConst16:$src1) +>; + defm : Arithmetic_i16_0Hi_Pats<add, V_ADD_U16_e64>; defm : Arithmetic_i16_0Hi_Pats<mul, V_MUL_LO_U16_e64>; defm : Arithmetic_i16_0Hi_Pats<sub, V_SUB_U16_e64>; @@ -777,7 +792,7 @@ defm : Arithmetic_i16_0Hi_Pats<umax, V_MAX_U16_e64>; defm : Arithmetic_i16_0Hi_Pats<lshl_rev, V_LSHLREV_B16_e64>; defm : Arithmetic_i16_0Hi_Pats<lshr_rev, V_LSHRREV_B16_e64>; defm : Arithmetic_i16_0Hi_Pats<ashr_rev, V_ASHRREV_I16_e64>; -} +} // End Predicates = [Has16BitInsts, isGFX7GFX8GFX9] def : ZExt_i16_i1_Pat<zext>; def : ZExt_i16_i1_Pat<anyext>; @@ -788,15 +803,7 @@ def : GCNPat < /*src1mod*/(i32 0), /*src1*/(i32 -1), $src) >; -// Undo sub x, c -> add x, -c canonicalization since c is more likely -// an inline immediate than -c. -// TODO: Also do for 64-bit. -def : GCNPat< - (add i16:$src0, (i16 NegSubInlineConst16:$src1)), - (V_SUB_U16_e64 $src0, NegSubInlineConst16:$src1) ->; - -} // End Predicates = [Has16BitInsts, isGFX7GFX8GFX9] +} // End Predicates = [Has16BitInsts] //===----------------------------------------------------------------------===// @@ -1282,7 +1289,7 @@ defm V_MADAK_F32 : VOP2Only_Real_MADK_gfx6_gfx7_gfx10<0x021>; // GFX8, GFX9 (VI). //===----------------------------------------------------------------------===// -let AssemblerPredicates = [isGFX8GFX9], DecoderNamespace = "GFX8" in { +let AssemblerPredicate = isGFX8GFX9, DecoderNamespace = "GFX8" in { multiclass VOP2_Real_MADK_vi <bits<6> op> { def _vi : VOP2_Real<!cast<VOP2_Pseudo>(NAME), SIEncodingFamily.VI>, @@ -1316,7 +1323,7 @@ multiclass Base_VOP2_Real_e32e64_vi <bits<6> op> : VOP2_Real_e32_vi<op>, VOP2_Real_e64_vi<{0, 1, 0, 0, op{5-0}}>; -} // End AssemblerPredicates = [isGFX8GFX9], DecoderNamespace = "GFX8" +} // End AssemblerPredicate = isGFX8GFX9, DecoderNamespace = "GFX8" multiclass VOP2_SDWA_Real <bits<6> op> { foreach _ = BoolToList<!cast<VOP2_Pseudo>(NAME#"_e32").Pfl.HasExtSDWA>.ret in @@ -1332,7 +1339,7 @@ multiclass VOP2_SDWA9_Real <bits<6> op> { VOP2_SDWA9Ae <op{5-0}, !cast<VOP2_SDWA_Pseudo>(NAME#"_sdwa").Pfl>; } -let AssemblerPredicates = [isGFX8Only] in { +let AssemblerPredicate = isGFX8Only in { multiclass VOP2be_Real_e32e64_vi_only <bits<6> op, string OpName, string AsmName> { def _e32_vi : @@ -1366,7 +1373,7 @@ multiclass VOP2be_Real_e32e64_vi_only <bits<6> op, string OpName, string AsmName } } -let AssemblerPredicates = [isGFX9Only] in { +let AssemblerPredicate = isGFX9Only in { multiclass VOP2be_Real_e32e64_gfx9 <bits<6> op, string OpName, string AsmName> { def _e32_gfx9 : @@ -1424,7 +1431,7 @@ multiclass VOP2_Real_e32e64_gfx9 <bits<6> op> { } } -} // AssemblerPredicates = [isGFX9Only] +} // AssemblerPredicate = isGFX9Only multiclass VOP2_Real_e32e64_vi <bits<6> op> : Base_VOP2_Real_e32e64_vi<op>, VOP2_SDWA_Real<op>, VOP2_SDWA9_Real<op> { diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td index 605425972b1c..67c8b926302d 100644 --- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td @@ -258,15 +258,15 @@ class getInterp16Ins <bit HasSrc2, bit HasOMod, (ins Src0Mod:$src0_modifiers, VRegSrc_32:$src0, Attr:$attr, AttrChan:$attrchan, Src2Mod:$src2_modifiers, VRegSrc_32:$src2, - highmod:$high, clampmod:$clamp, omod:$omod), + highmod:$high, clampmod0:$clamp, omod0:$omod), (ins Src0Mod:$src0_modifiers, VRegSrc_32:$src0, Attr:$attr, AttrChan:$attrchan, Src2Mod:$src2_modifiers, VRegSrc_32:$src2, - highmod:$high, clampmod:$clamp) + highmod:$high, clampmod0:$clamp) ), (ins Src0Mod:$src0_modifiers, VRegSrc_32:$src0, Attr:$attr, AttrChan:$attrchan, - highmod:$high, clampmod:$clamp, omod:$omod) + highmod:$high, clampmod0:$clamp, omod0:$omod) ); } @@ -452,14 +452,16 @@ def V_MAD_I16 : VOP3Inst <"v_mad_i16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_CL let FPDPRounding = 1 in { def V_MAD_F16 : VOP3Inst <"v_mad_f16", VOP3_Profile<VOP_F16_F16_F16_F16>, fmad>; let Uses = [M0, EXEC] in { +// For some reason the intrinsic operands are in a different order +// from the instruction operands. def V_INTERP_P2_F16 : VOP3Interp <"v_interp_p2_f16", VOP3_INTERP16<[f16, f32, i32, f32]>, - [(set f16:$vdst, (AMDGPUinterp_p2_f16 f32:$src0, (i32 timm:$attrchan), - (i32 timm:$attr), - (i32 timm:$src0_modifiers), - (f32 VRegSrc_32:$src2), - (i32 timm:$src2_modifiers), - (i1 timm:$high), - (i1 timm:$clamp)))]>; + [(set f16:$vdst, + (int_amdgcn_interp_p2_f16 (VOP3Mods f32:$src2, i32:$src2_modifiers), + (VOP3Mods f32:$src0, i32:$src0_modifiers), + (i32 timm:$attrchan), + (i32 timm:$attr), + (i1 timm:$high), + M0))]>; } // End Uses = [M0, EXEC] } // End FPDPRounding = 1 } // End renamedInGFX9 = 1 @@ -497,11 +499,11 @@ def V_INTERP_P1LV_F16 : VOP3Interp <"v_interp_p1lv_f16", VOP3_INTERP16<[f32, f32 } // End SubtargetPredicate = Has16BitInsts, isCommutable = 1 -let SubtargetPredicate = isGFX8GFX9 in { +let SubtargetPredicate = isGFX8Plus, Uses = [M0, EXEC] in { def V_INTERP_P1_F32_e64 : VOP3Interp <"v_interp_p1_f32", VOP3_INTERP>; def V_INTERP_P2_F32_e64 : VOP3Interp <"v_interp_p2_f32", VOP3_INTERP>; def V_INTERP_MOV_F32_e64 : VOP3Interp <"v_interp_mov_f32", VOP3_INTERP_MOV>; -} // End SubtargetPredicate = isGFX8GFX9 +} // End SubtargetPredicate = isGFX8Plus, Uses = [M0, EXEC] let Predicates = [Has16BitInsts, isGFX6GFX7GFX8GFX9] in { @@ -770,6 +772,10 @@ defm V_SUB_NC_I32 : defm V_ADD_NC_I32 : VOP3_Real_gfx10_with_name<0x37f, "V_ADD_I32_gfx9", "v_add_nc_i32">; +defm V_INTERP_P1_F32_e64 : VOP3Interp_Real_gfx10<0x200>; +defm V_INTERP_P2_F32_e64 : VOP3Interp_Real_gfx10<0x201>; +defm V_INTERP_MOV_F32_e64 : VOP3Interp_Real_gfx10<0x202>; + defm V_INTERP_P1LL_F16 : VOP3Interp_Real_gfx10<0x342>; defm V_INTERP_P1LV_F16 : VOP3Interp_Real_gfx10<0x343>; defm V_INTERP_P2_F16 : VOP3Interp_Real_gfx10<0x35a>; @@ -923,7 +929,7 @@ defm V_DIV_SCALE_F64 : VOP3be_Real_gfx6_gfx7_gfx10<0x16e>; // GFX8, GFX9 (VI). //===----------------------------------------------------------------------===// -let AssemblerPredicates = [isGFX8GFX9], DecoderNamespace = "GFX8" in { +let AssemblerPredicate = isGFX8GFX9, DecoderNamespace = "GFX8" in { multiclass VOP3_Real_vi<bits<10> op> { def _vi : VOP3_Real<!cast<VOP_Pseudo>(NAME), SIEncodingFamily.VI>, @@ -945,9 +951,9 @@ multiclass VOP3Interp_Real_vi<bits<10> op> { VOP3Interp_vi <op, !cast<VOP_Pseudo>(NAME).Pfl>; } -} // End AssemblerPredicates = [isGFX8GFX9], DecoderNamespace = "GFX8" +} // End AssemblerPredicate = isGFX8GFX9, DecoderNamespace = "GFX8" -let AssemblerPredicates = [isGFX8Only], DecoderNamespace = "GFX8" in { +let AssemblerPredicate = isGFX8Only, DecoderNamespace = "GFX8" in { multiclass VOP3_F16_Real_vi<bits<10> op> { def _vi : VOP3_Real<!cast<VOP3_Pseudo>(NAME), SIEncodingFamily.VI>, @@ -959,9 +965,9 @@ multiclass VOP3Interp_F16_Real_vi<bits<10> op> { VOP3Interp_vi <op, !cast<VOP3_Pseudo>(NAME).Pfl>; } -} // End AssemblerPredicates = [isGFX8Only], DecoderNamespace = "GFX8" +} // End AssemblerPredicate = isGFX8Only, DecoderNamespace = "GFX8" -let AssemblerPredicates = [isGFX9Only], DecoderNamespace = "GFX9" in { +let AssemblerPredicate = isGFX9Only, DecoderNamespace = "GFX9" in { multiclass VOP3_F16_Real_gfx9<bits<10> op, string OpName, string AsmName> { def _gfx9 : VOP3_Real<!cast<VOP3_Pseudo>(OpName), SIEncodingFamily.GFX9>, @@ -995,7 +1001,7 @@ multiclass VOP3_Real_gfx9<bits<10> op, string AsmName> { } } -} // End AssemblerPredicates = [isGFX9Only], DecoderNamespace = "GFX9" +} // End AssemblerPredicate = isGFX9Only, DecoderNamespace = "GFX9" defm V_MAD_U64_U32 : VOP3be_Real_vi <0x1E8>; defm V_MAD_I64_I32 : VOP3be_Real_vi <0x1E9>; diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td index 0c13f39fec02..933acc2278fd 100644 --- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td @@ -404,7 +404,7 @@ def : MnemonicAlias<"v_accvgpr_write", "v_accvgpr_write_b32">; multiclass VOP3P_Real_vi<bits<10> op> { def _vi : VOP3P_Real<!cast<VOP3_Pseudo>(NAME), SIEncodingFamily.VI>, VOP3Pe <op, !cast<VOP3_Pseudo>(NAME).Pfl> { - let AssemblerPredicates = [HasVOP3PInsts]; + let AssemblerPredicate = HasVOP3PInsts; let DecoderNamespace = "GFX8"; } } @@ -412,7 +412,7 @@ multiclass VOP3P_Real_vi<bits<10> op> { multiclass VOP3P_Real_MAI<bits<10> op> { def _vi : VOP3P_Real<!cast<VOP3_Pseudo>(NAME), SIEncodingFamily.VI>, VOP3Pe_MAI <op, !cast<VOP3_Pseudo>(NAME).Pfl> { - let AssemblerPredicates = [HasMAIInsts]; + let AssemblerPredicate = HasMAIInsts; let DecoderNamespace = "GFX8"; } } diff --git a/llvm/lib/Target/AMDGPU/VOPCInstructions.td b/llvm/lib/Target/AMDGPU/VOPCInstructions.td index 8ef0ec7b71f4..39d18794f947 100644 --- a/llvm/lib/Target/AMDGPU/VOPCInstructions.td +++ b/llvm/lib/Target/AMDGPU/VOPCInstructions.td @@ -650,7 +650,7 @@ class getVOPCClassPat64 <VOPProfile P> { list<dag> ret = [(set i1:$sdst, (AMDGPUfp_class - (P.Src0VT (VOP3Mods0Clamp0OMod P.Src0VT:$src0, i32:$src0_modifiers)), + (P.Src0VT (VOP3Mods P.Src0VT:$src0, i32:$src0_modifiers)), P.Src1VT:$src1))]; } @@ -1218,7 +1218,7 @@ defm V_CMPX_T_U64 : VOPCX_Real_gfx6_gfx7_gfx10<0x0f7>; //===----------------------------------------------------------------------===// multiclass VOPC_Real_vi <bits<10> op> { - let AssemblerPredicates = [isGFX8GFX9], DecoderNamespace = "GFX8" in { + let AssemblerPredicate = isGFX8GFX9, DecoderNamespace = "GFX8" in { def _e32_vi : VOPC_Real<!cast<VOPC_Pseudo>(NAME#"_e32"), SIEncodingFamily.VI>, VOPCe<op{7-0}>; |