summaryrefslogtreecommitdiff
path: root/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp')
-rw-r--r--lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp469
1 files changed, 379 insertions, 90 deletions
diff --git a/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 1239dfb235efa..cfe6346fb6b17 100644
--- a/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -28,8 +28,10 @@
#include "R600RegisterInfo.h"
#include "SIDefines.h"
#include "SIMachineFunctionInfo.h"
+#include "SIInstrInfo.h"
#include "SIRegisterInfo.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCSectionELF.h"
#include "llvm/MC/MCStreamer.h"
@@ -37,7 +39,9 @@
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/TargetRegistry.h"
#include "llvm/Target/TargetLoweringObjectFile.h"
+#include "AMDGPURuntimeMetadata.h"
+using namespace ::AMDGPU;
using namespace llvm;
// TODO: This should get the default rounding mode from the kernel. We just set
@@ -61,7 +65,7 @@ using namespace llvm;
// instructions to run at the double precision rate for the device so it's
// probably best to just report no single precision denormals.
static uint32_t getFPMode(const MachineFunction &F) {
- const AMDGPUSubtarget& ST = F.getSubtarget<AMDGPUSubtarget>();
+ const SISubtarget& ST = F.getSubtarget<SISubtarget>();
// TODO: Is there any real use for the flush in only / flush out only modes?
uint32_t FP32Denormals =
@@ -104,10 +108,12 @@ void AMDGPUAsmPrinter::EmitStartOfAsmFile(Module &M) {
AMDGPUTargetStreamer *TS =
static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
- TS->EmitDirectiveHSACodeObjectVersion(1, 0);
+ TS->EmitDirectiveHSACodeObjectVersion(2, 1);
+
AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(STI->getFeatureBits());
TS->EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor, ISA.Stepping,
"AMD", "AMDGPU");
+ emitStartOfRuntimeMetadata(M);
}
void AMDGPUAsmPrinter::EmitFunctionBodyStart() {
@@ -132,54 +138,13 @@ void AMDGPUAsmPrinter::EmitFunctionEntryLabel() {
AsmPrinter::EmitFunctionEntryLabel();
}
-static bool isModuleLinkage(const GlobalValue *GV) {
- switch (GV->getLinkage()) {
- case GlobalValue::InternalLinkage:
- case GlobalValue::CommonLinkage:
- return true;
- case GlobalValue::ExternalLinkage:
- return false;
- default: llvm_unreachable("unknown linkage type");
- }
-}
-
void AMDGPUAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
- if (TM.getTargetTriple().getOS() != Triple::AMDHSA) {
- AsmPrinter::EmitGlobalVariable(GV);
- return;
- }
-
- if (GV->isDeclaration() || GV->getLinkage() == GlobalValue::PrivateLinkage) {
- AsmPrinter::EmitGlobalVariable(GV);
- return;
- }
-
// Group segment variables aren't emitted in HSA.
if (AMDGPU::isGroupSegment(GV))
return;
- AMDGPUTargetStreamer *TS =
- static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
- if (isModuleLinkage(GV)) {
- TS->EmitAMDGPUHsaModuleScopeGlobal(GV->getName());
- } else {
- TS->EmitAMDGPUHsaProgramScopeGlobal(GV->getName());
- }
-
- MCSymbolELF *GVSym = cast<MCSymbolELF>(getSymbol(GV));
- const DataLayout &DL = getDataLayout();
-
- // Emit the size
- uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType());
- OutStreamer->emitELFSize(GVSym, MCConstantExpr::create(Size, OutContext));
- OutStreamer->PushSection();
- OutStreamer->SwitchSection(
- getObjFileLowering().SectionForGlobal(GV, *Mang, TM));
- const Constant *C = GV->getInitializer();
- OutStreamer->EmitLabel(GVSym);
- EmitGlobalConstant(DL, C);
- OutStreamer->PopSection();
+ AsmPrinter::EmitGlobalVariable(GV);
}
bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
@@ -230,6 +195,20 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
false);
OutStreamer->emitRawComment(" ScratchSize: " + Twine(KernelInfo.ScratchSize),
false);
+ OutStreamer->emitRawComment(" LDSByteSize: " + Twine(KernelInfo.LDSSize) +
+ " bytes/workgroup (compile time only)", false);
+
+ OutStreamer->emitRawComment(" ReservedVGPRFirst: " + Twine(KernelInfo.ReservedVGPRFirst),
+ false);
+ OutStreamer->emitRawComment(" ReservedVGPRCount: " + Twine(KernelInfo.ReservedVGPRCount),
+ false);
+
+ if (MF.getSubtarget<SISubtarget>().debuggerEmitPrologue()) {
+ OutStreamer->emitRawComment(" DebuggerWavefrontPrivateSegmentOffsetSGPR: s" +
+ Twine(KernelInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR), false);
+ OutStreamer->emitRawComment(" DebuggerPrivateSegmentBufferSGPR: s" +
+ Twine(KernelInfo.DebuggerPrivateSegmentBufferSGPR), false);
+ }
OutStreamer->emitRawComment(" COMPUTE_PGM_RSRC2:USER_SGPR: " +
Twine(G_00B84C_USER_SGPR(KernelInfo.ComputePGMRSrc2)),
@@ -268,15 +247,16 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
}
}
+ emitRuntimeMetadata(*MF.getFunction());
+
return false;
}
void AMDGPUAsmPrinter::EmitProgramInfoR600(const MachineFunction &MF) {
unsigned MaxGPR = 0;
bool killPixel = false;
- const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
- const R600RegisterInfo *RI =
- static_cast<const R600RegisterInfo *>(STM.getRegisterInfo());
+ const R600Subtarget &STM = MF.getSubtarget<R600Subtarget>();
+ const R600RegisterInfo *RI = STM.getRegisterInfo();
const R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
for (const MachineBasicBlock &MBB : MF) {
@@ -299,23 +279,23 @@ void AMDGPUAsmPrinter::EmitProgramInfoR600(const MachineFunction &MF) {
}
unsigned RsrcReg;
- if (STM.getGeneration() >= AMDGPUSubtarget::EVERGREEN) {
+ if (STM.getGeneration() >= R600Subtarget::EVERGREEN) {
// Evergreen / Northern Islands
- switch (MFI->getShaderType()) {
+ switch (MF.getFunction()->getCallingConv()) {
default: // Fall through
- case ShaderType::COMPUTE: RsrcReg = R_0288D4_SQ_PGM_RESOURCES_LS; break;
- case ShaderType::GEOMETRY: RsrcReg = R_028878_SQ_PGM_RESOURCES_GS; break;
- case ShaderType::PIXEL: RsrcReg = R_028844_SQ_PGM_RESOURCES_PS; break;
- case ShaderType::VERTEX: RsrcReg = R_028860_SQ_PGM_RESOURCES_VS; break;
+ case CallingConv::AMDGPU_CS: RsrcReg = R_0288D4_SQ_PGM_RESOURCES_LS; break;
+ case CallingConv::AMDGPU_GS: RsrcReg = R_028878_SQ_PGM_RESOURCES_GS; break;
+ case CallingConv::AMDGPU_PS: RsrcReg = R_028844_SQ_PGM_RESOURCES_PS; break;
+ case CallingConv::AMDGPU_VS: RsrcReg = R_028860_SQ_PGM_RESOURCES_VS; break;
}
} else {
// R600 / R700
- switch (MFI->getShaderType()) {
+ switch (MF.getFunction()->getCallingConv()) {
default: // Fall through
- case ShaderType::GEOMETRY: // Fall through
- case ShaderType::COMPUTE: // Fall through
- case ShaderType::VERTEX: RsrcReg = R_028868_SQ_PGM_RESOURCES_VS; break;
- case ShaderType::PIXEL: RsrcReg = R_028850_SQ_PGM_RESOURCES_PS; break;
+ case CallingConv::AMDGPU_GS: // Fall through
+ case CallingConv::AMDGPU_CS: // Fall through
+ case CallingConv::AMDGPU_VS: RsrcReg = R_028868_SQ_PGM_RESOURCES_VS; break;
+ case CallingConv::AMDGPU_PS: RsrcReg = R_028850_SQ_PGM_RESOURCES_PS; break;
}
}
@@ -325,23 +305,23 @@ void AMDGPUAsmPrinter::EmitProgramInfoR600(const MachineFunction &MF) {
OutStreamer->EmitIntValue(R_02880C_DB_SHADER_CONTROL, 4);
OutStreamer->EmitIntValue(S_02880C_KILL_ENABLE(killPixel), 4);
- if (MFI->getShaderType() == ShaderType::COMPUTE) {
+ if (AMDGPU::isCompute(MF.getFunction()->getCallingConv())) {
OutStreamer->EmitIntValue(R_0288E8_SQ_LDS_ALLOC, 4);
- OutStreamer->EmitIntValue(RoundUpToAlignment(MFI->LDSSize, 4) >> 2, 4);
+ OutStreamer->EmitIntValue(alignTo(MFI->LDSSize, 4) >> 2, 4);
}
}
void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
const MachineFunction &MF) const {
- const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
+ const SISubtarget &STM = MF.getSubtarget<SISubtarget>();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
uint64_t CodeSize = 0;
unsigned MaxSGPR = 0;
unsigned MaxVGPR = 0;
bool VCCUsed = false;
bool FlatUsed = false;
- const SIRegisterInfo *RI =
- static_cast<const SIRegisterInfo *>(STM.getRegisterInfo());
+ const SIRegisterInfo *RI = STM.getRegisterInfo();
+ const SIInstrInfo *TII = STM.getInstrInfo();
for (const MachineBasicBlock &MBB : MF) {
for (const MachineInstr &MI : MBB) {
@@ -351,8 +331,7 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
if (MI.isDebugValue())
continue;
- // FIXME: This is reporting 0 for many instructions.
- CodeSize += MI.getDesc().Size;
+ CodeSize += TII->getInstSizeInBytes(MI);
unsigned numOperands = MI.getNumOperands();
for (unsigned op_idx = 0; op_idx < numOperands; op_idx++) {
@@ -366,6 +345,8 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
unsigned reg = MO.getReg();
switch (reg) {
case AMDGPU::EXEC:
+ case AMDGPU::EXEC_LO:
+ case AMDGPU::EXEC_HI:
case AMDGPU::SCC:
case AMDGPU::M0:
continue;
@@ -382,17 +363,32 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
FlatUsed = true;
continue;
+ case AMDGPU::TBA:
+ case AMDGPU::TBA_LO:
+ case AMDGPU::TBA_HI:
+ case AMDGPU::TMA:
+ case AMDGPU::TMA_LO:
+ case AMDGPU::TMA_HI:
+ llvm_unreachable("Trap Handler registers should not be used");
+ continue;
+
default:
break;
}
if (AMDGPU::SReg_32RegClass.contains(reg)) {
+ if (AMDGPU::TTMP_32RegClass.contains(reg)) {
+ llvm_unreachable("Trap Handler registers should not be used");
+ }
isSGPR = true;
width = 1;
} else if (AMDGPU::VGPR_32RegClass.contains(reg)) {
isSGPR = false;
width = 1;
} else if (AMDGPU::SReg_64RegClass.contains(reg)) {
+ if (AMDGPU::TTMP_64RegClass.contains(reg)) {
+ llvm_unreachable("Trap Handler registers should not be used");
+ }
isSGPR = true;
width = 2;
} else if (AMDGPU::VReg_64RegClass.contains(reg)) {
@@ -438,7 +434,7 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
if (VCCUsed)
ExtraSGPRs = 2;
- if (STM.getGeneration() < AMDGPUSubtarget::VOLCANIC_ISLANDS) {
+ if (STM.getGeneration() < SISubtarget::VOLCANIC_ISLANDS) {
if (FlatUsed)
ExtraSGPRs = 4;
} else {
@@ -451,23 +447,54 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
MaxSGPR += ExtraSGPRs;
+ // Record first reserved register and reserved register count fields, and
+ // update max register counts if "amdgpu-debugger-reserve-regs" attribute was
+ // specified.
+ if (STM.debuggerReserveRegs()) {
+ ProgInfo.ReservedVGPRFirst = MaxVGPR + 1;
+ ProgInfo.ReservedVGPRCount = MFI->getDebuggerReservedVGPRCount();
+ MaxVGPR += MFI->getDebuggerReservedVGPRCount();
+ }
+
+ // Update DebuggerWavefrontPrivateSegmentOffsetSGPR and
+ // DebuggerPrivateSegmentBufferSGPR fields if "amdgpu-debugger-emit-prologue"
+ // attribute was specified.
+ if (STM.debuggerEmitPrologue()) {
+ ProgInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR =
+ RI->getHWRegIndex(MFI->getScratchWaveOffsetReg());
+ ProgInfo.DebuggerPrivateSegmentBufferSGPR =
+ RI->getHWRegIndex(MFI->getScratchRSrcReg());
+ }
+
// We found the maximum register index. They start at 0, so add one to get the
// number of registers.
ProgInfo.NumVGPR = MaxVGPR + 1;
ProgInfo.NumSGPR = MaxSGPR + 1;
if (STM.hasSGPRInitBug()) {
- if (ProgInfo.NumSGPR > AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG) {
+ if (ProgInfo.NumSGPR > SISubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG) {
LLVMContext &Ctx = MF.getFunction()->getContext();
- Ctx.emitError("too many SGPRs used with the SGPR init bug");
+ DiagnosticInfoResourceLimit Diag(*MF.getFunction(),
+ "SGPRs with SGPR init bug",
+ ProgInfo.NumSGPR, DS_Error);
+ Ctx.diagnose(Diag);
}
- ProgInfo.NumSGPR = AMDGPUSubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG;
+ ProgInfo.NumSGPR = SISubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG;
}
if (MFI->NumUserSGPRs > STM.getMaxNumUserSGPRs()) {
LLVMContext &Ctx = MF.getFunction()->getContext();
- Ctx.emitError("too many user SGPRs used");
+ DiagnosticInfoResourceLimit Diag(*MF.getFunction(), "user SGPRs",
+ MFI->NumUserSGPRs, DS_Error);
+ Ctx.diagnose(Diag);
+ }
+
+ if (MFI->LDSSize > static_cast<unsigned>(STM.getLocalMemorySize())) {
+ LLVMContext &Ctx = MF.getFunction()->getContext();
+ DiagnosticInfoResourceLimit Diag(*MF.getFunction(), "local memory",
+ MFI->LDSSize, DS_Error);
+ Ctx.diagnose(Diag);
}
ProgInfo.VGPRBlocks = (ProgInfo.NumVGPR - 1) / 4;
@@ -476,21 +503,20 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
// register.
ProgInfo.FloatMode = getFPMode(MF);
- // XXX: Not quite sure what this does, but sc seems to unset this.
ProgInfo.IEEEMode = 0;
- // Do not clamp NAN to 0.
- ProgInfo.DX10Clamp = 0;
+ // Make clamp modifier on NaN input returns 0.
+ ProgInfo.DX10Clamp = 1;
const MachineFrameInfo *FrameInfo = MF.getFrameInfo();
- ProgInfo.ScratchSize = FrameInfo->estimateStackSize(MF);
+ ProgInfo.ScratchSize = FrameInfo->getStackSize();
ProgInfo.FlatUsed = FlatUsed;
ProgInfo.VCCUsed = VCCUsed;
ProgInfo.CodeLen = CodeSize;
unsigned LDSAlignShift;
- if (STM.getGeneration() < AMDGPUSubtarget::SEA_ISLANDS) {
+ if (STM.getGeneration() < SISubtarget::SEA_ISLANDS) {
// LDS is allocated in 64 dword blocks.
LDSAlignShift = 8;
} else {
@@ -503,7 +529,7 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
ProgInfo.LDSSize = MFI->LDSSize + LDSSpillSize;
ProgInfo.LDSBlocks =
- RoundUpToAlignment(ProgInfo.LDSSize, 1 << LDSAlignShift) >> LDSAlignShift;
+ alignTo(ProgInfo.LDSSize, 1ULL << LDSAlignShift) >> LDSAlignShift;
// Scratch is allocated in 256 dword blocks.
unsigned ScratchAlignShift = 10;
@@ -511,8 +537,9 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
// is used by the entire wave. ProgInfo.ScratchSize is the amount of
// scratch memory used per thread.
ProgInfo.ScratchBlocks =
- RoundUpToAlignment(ProgInfo.ScratchSize * STM.getWavefrontSize(),
- 1 << ScratchAlignShift) >> ScratchAlignShift;
+ alignTo(ProgInfo.ScratchSize * STM.getWavefrontSize(),
+ 1ULL << ScratchAlignShift) >>
+ ScratchAlignShift;
ProgInfo.ComputePGMRSrc1 =
S_00B848_VGPRS(ProgInfo.VGPRBlocks) |
@@ -544,23 +571,23 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo,
S_00B84C_EXCP_EN(0);
}
-static unsigned getRsrcReg(unsigned ShaderType) {
- switch (ShaderType) {
+static unsigned getRsrcReg(CallingConv::ID CallConv) {
+ switch (CallConv) {
default: // Fall through
- case ShaderType::COMPUTE: return R_00B848_COMPUTE_PGM_RSRC1;
- case ShaderType::GEOMETRY: return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
- case ShaderType::PIXEL: return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
- case ShaderType::VERTEX: return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
+ case CallingConv::AMDGPU_CS: return R_00B848_COMPUTE_PGM_RSRC1;
+ case CallingConv::AMDGPU_GS: return R_00B228_SPI_SHADER_PGM_RSRC1_GS;
+ case CallingConv::AMDGPU_PS: return R_00B028_SPI_SHADER_PGM_RSRC1_PS;
+ case CallingConv::AMDGPU_VS: return R_00B128_SPI_SHADER_PGM_RSRC1_VS;
}
}
void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF,
const SIProgramInfo &KernelInfo) {
- const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
+ const SISubtarget &STM = MF.getSubtarget<SISubtarget>();
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
- unsigned RsrcReg = getRsrcReg(MFI->getShaderType());
+ unsigned RsrcReg = getRsrcReg(MF.getFunction()->getCallingConv());
- if (MFI->getShaderType() == ShaderType::COMPUTE) {
+ if (AMDGPU::isCompute(MF.getFunction()->getCallingConv())) {
OutStreamer->EmitIntValue(R_00B848_COMPUTE_PGM_RSRC1, 4);
OutStreamer->EmitIntValue(KernelInfo.ComputePGMRSrc1, 4);
@@ -577,13 +604,13 @@ void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF,
OutStreamer->EmitIntValue(RsrcReg, 4);
OutStreamer->EmitIntValue(S_00B028_VGPRS(KernelInfo.VGPRBlocks) |
S_00B028_SGPRS(KernelInfo.SGPRBlocks), 4);
- if (STM.isVGPRSpillingEnabled(MFI)) {
+ if (STM.isVGPRSpillingEnabled(*MF.getFunction())) {
OutStreamer->EmitIntValue(R_0286E8_SPI_TMPRING_SIZE, 4);
OutStreamer->EmitIntValue(S_0286E8_WAVESIZE(KernelInfo.ScratchBlocks), 4);
}
}
- if (MFI->getShaderType() == ShaderType::PIXEL) {
+ if (MF.getFunction()->getCallingConv() == CallingConv::AMDGPU_PS) {
OutStreamer->EmitIntValue(R_00B02C_SPI_SHADER_PGM_RSRC2_PS, 4);
OutStreamer->EmitIntValue(S_00B02C_EXTRA_LDS_SIZE(KernelInfo.LDSBlocks), 4);
OutStreamer->EmitIntValue(R_0286CC_SPI_PS_INPUT_ENA, 4);
@@ -591,12 +618,31 @@ void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF,
OutStreamer->EmitIntValue(R_0286D0_SPI_PS_INPUT_ADDR, 4);
OutStreamer->EmitIntValue(MFI->getPSInputAddr(), 4);
}
+
+ OutStreamer->EmitIntValue(R_SPILLED_SGPRS, 4);
+ OutStreamer->EmitIntValue(MFI->getNumSpilledSGPRs(), 4);
+ OutStreamer->EmitIntValue(R_SPILLED_VGPRS, 4);
+ OutStreamer->EmitIntValue(MFI->getNumSpilledVGPRs(), 4);
+}
+
+// This is supposed to be log2(Size)
+static amd_element_byte_size_t getElementByteSizeValue(unsigned Size) {
+ switch (Size) {
+ case 4:
+ return AMD_ELEMENT_4_BYTES;
+ case 8:
+ return AMD_ELEMENT_8_BYTES;
+ case 16:
+ return AMD_ELEMENT_16_BYTES;
+ default:
+ llvm_unreachable("invalid private_element_size");
+ }
}
void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF,
const SIProgramInfo &KernelInfo) const {
const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
- const AMDGPUSubtarget &STM = MF.getSubtarget<AMDGPUSubtarget>();
+ const SISubtarget &STM = MF.getSubtarget<SISubtarget>();
amd_kernel_code_t header;
AMDGPU::initDefaultAMDKernelCodeT(header, STM.getFeatureBits());
@@ -606,6 +652,11 @@ void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF,
(KernelInfo.ComputePGMRSrc2 << 32);
header.code_properties = AMD_CODE_PROPERTY_IS_PTR64;
+
+ AMD_HSA_BITS_SET(header.code_properties,
+ AMD_CODE_PROPERTY_PRIVATE_ELEMENT_SIZE,
+ getElementByteSizeValue(STM.getMaxPrivateElementSize()));
+
if (MFI->hasPrivateSegmentBuffer()) {
header.code_properties |=
AMD_CODE_PROPERTY_ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER;
@@ -646,6 +697,9 @@ void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF,
if (MFI->hasDispatchPtr())
header.code_properties |= AMD_CODE_PROPERTY_ENABLE_SGPR_DISPATCH_PTR;
+ if (STM.debuggerSupported())
+ header.code_properties |= AMD_CODE_PROPERTY_IS_DEBUG_SUPPORTED;
+
if (STM.isXNACKEnabled())
header.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED;
@@ -654,9 +708,20 @@ void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF,
header.workitem_vgpr_count = KernelInfo.NumVGPR;
header.workitem_private_segment_byte_size = KernelInfo.ScratchSize;
header.workgroup_group_segment_byte_size = KernelInfo.LDSSize;
+ header.reserved_vgpr_first = KernelInfo.ReservedVGPRFirst;
+ header.reserved_vgpr_count = KernelInfo.ReservedVGPRCount;
+
+ if (STM.debuggerEmitPrologue()) {
+ header.debug_wavefront_private_segment_offset_sgpr =
+ KernelInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR;
+ header.debug_private_segment_buffer_sgpr =
+ KernelInfo.DebuggerPrivateSegmentBufferSGPR;
+ }
AMDGPUTargetStreamer *TS =
static_cast<AMDGPUTargetStreamer *>(OutStreamer->getTargetStreamer());
+
+ OutStreamer->SwitchSection(getObjFileLowering().getTextSection());
TS->EmitAMDKernelCodeT(header);
}
@@ -680,3 +745,227 @@ bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
*TM.getSubtargetImpl(*MF->getFunction())->getRegisterInfo());
return false;
}
+
+// Emit a key and an integer value for runtime metadata.
+static void emitRuntimeMDIntValue(std::unique_ptr<MCStreamer> &Streamer,
+ RuntimeMD::Key K, uint64_t V,
+ unsigned Size) {
+ Streamer->EmitIntValue(K, 1);
+ Streamer->EmitIntValue(V, Size);
+}
+
+// Emit a key and a string value for runtime metadata.
+static void emitRuntimeMDStringValue(std::unique_ptr<MCStreamer> &Streamer,
+ RuntimeMD::Key K, StringRef S) {
+ Streamer->EmitIntValue(K, 1);
+ Streamer->EmitIntValue(S.size(), 4);
+ Streamer->EmitBytes(S);
+}
+
+// Emit a key and three integer values for runtime metadata.
+// The three integer values are obtained from MDNode \p Node;
+static void emitRuntimeMDThreeIntValues(std::unique_ptr<MCStreamer> &Streamer,
+ RuntimeMD::Key K, MDNode *Node,
+ unsigned Size) {
+ Streamer->EmitIntValue(K, 1);
+ Streamer->EmitIntValue(mdconst::extract<ConstantInt>(
+ Node->getOperand(0))->getZExtValue(), Size);
+ Streamer->EmitIntValue(mdconst::extract<ConstantInt>(
+ Node->getOperand(1))->getZExtValue(), Size);
+ Streamer->EmitIntValue(mdconst::extract<ConstantInt>(
+ Node->getOperand(2))->getZExtValue(), Size);
+}
+
+void AMDGPUAsmPrinter::emitStartOfRuntimeMetadata(const Module &M) {
+ OutStreamer->SwitchSection(getObjFileLowering().getContext()
+ .getELFSection(RuntimeMD::SectionName, ELF::SHT_PROGBITS, 0));
+
+ emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyMDVersion,
+ RuntimeMD::MDVersion << 8 | RuntimeMD::MDRevision, 2);
+ if (auto MD = M.getNamedMetadata("opencl.ocl.version")) {
+ emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyLanguage,
+ RuntimeMD::OpenCL_C, 1);
+ auto Node = MD->getOperand(0);
+ unsigned short Major = mdconst::extract<ConstantInt>(Node->getOperand(0))
+ ->getZExtValue();
+ unsigned short Minor = mdconst::extract<ConstantInt>(Node->getOperand(1))
+ ->getZExtValue();
+ emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyLanguageVersion,
+ Major * 100 + Minor * 10, 2);
+ }
+}
+
+static std::string getOCLTypeName(Type *Ty, bool isSigned) {
+ if (VectorType* VecTy = dyn_cast<VectorType>(Ty)) {
+ Type* EleTy = VecTy->getElementType();
+ unsigned Size = VecTy->getVectorNumElements();
+ return (Twine(getOCLTypeName(EleTy, isSigned)) + Twine(Size)).str();
+ }
+ switch (Ty->getTypeID()) {
+ case Type::HalfTyID: return "half";
+ case Type::FloatTyID: return "float";
+ case Type::DoubleTyID: return "double";
+ case Type::IntegerTyID: {
+ if (!isSigned)
+ return (Twine('u') + Twine(getOCLTypeName(Ty, true))).str();
+ auto IntTy = cast<IntegerType>(Ty);
+ auto BW = IntTy->getIntegerBitWidth();
+ switch (BW) {
+ case 8:
+ return "char";
+ case 16:
+ return "short";
+ case 32:
+ return "int";
+ case 64:
+ return "long";
+ default:
+ return (Twine('i') + Twine(BW)).str();
+ }
+ }
+ default:
+ llvm_unreachable("invalid type");
+ }
+}
+
+static RuntimeMD::KernelArg::ValueType getRuntimeMDValueType(
+ Type *Ty, StringRef TypeName) {
+ if (auto VT = dyn_cast<VectorType>(Ty))
+ return getRuntimeMDValueType(VT->getElementType(), TypeName);
+ else if (auto PT = dyn_cast<PointerType>(Ty))
+ return getRuntimeMDValueType(PT->getElementType(), TypeName);
+ else if (Ty->isHalfTy())
+ return RuntimeMD::KernelArg::F16;
+ else if (Ty->isFloatTy())
+ return RuntimeMD::KernelArg::F32;
+ else if (Ty->isDoubleTy())
+ return RuntimeMD::KernelArg::F64;
+ else if (IntegerType* intTy = dyn_cast<IntegerType>(Ty)) {
+ bool Signed = !TypeName.startswith("u");
+ switch (intTy->getIntegerBitWidth()) {
+ case 8:
+ return Signed ? RuntimeMD::KernelArg::I8 : RuntimeMD::KernelArg::U8;
+ case 16:
+ return Signed ? RuntimeMD::KernelArg::I16 : RuntimeMD::KernelArg::U16;
+ case 32:
+ return Signed ? RuntimeMD::KernelArg::I32 : RuntimeMD::KernelArg::U32;
+ case 64:
+ return Signed ? RuntimeMD::KernelArg::I64 : RuntimeMD::KernelArg::U64;
+ default:
+ // Runtime does not recognize other integer types. Report as
+ // struct type.
+ return RuntimeMD::KernelArg::Struct;
+ }
+ } else
+ return RuntimeMD::KernelArg::Struct;
+}
+
+void AMDGPUAsmPrinter::emitRuntimeMetadata(const Function &F) {
+ if (!F.getMetadata("kernel_arg_type"))
+ return;
+
+ MCContext &Context = getObjFileLowering().getContext();
+ OutStreamer->SwitchSection(
+ Context.getELFSection(RuntimeMD::SectionName, ELF::SHT_PROGBITS, 0));
+ OutStreamer->EmitIntValue(RuntimeMD::KeyKernelBegin, 1);
+ emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyKernelName, F.getName());
+
+ for (auto &Arg:F.args()) {
+ // Emit KeyArgBegin.
+ unsigned I = Arg.getArgNo();
+ OutStreamer->EmitIntValue(RuntimeMD::KeyArgBegin, 1);
+
+ // Emit KeyArgSize and KeyArgAlign.
+ auto T = Arg.getType();
+ auto DL = F.getParent()->getDataLayout();
+ emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgSize,
+ DL.getTypeAllocSize(T), 4);
+ emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgAlign,
+ DL.getABITypeAlignment(T), 4);
+
+ // Emit KeyArgTypeName.
+ auto TypeName = dyn_cast<MDString>(F.getMetadata(
+ "kernel_arg_type")->getOperand(I))->getString();
+ emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyArgTypeName, TypeName);
+
+ // Emit KeyArgName.
+ if (auto ArgNameMD = F.getMetadata("kernel_arg_name")) {
+ auto ArgName = cast<MDString>(ArgNameMD->getOperand(
+ I))->getString();
+ emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyArgName, ArgName);
+ }
+
+ // Emit KeyArgIsVolatile, KeyArgIsRestrict, KeyArgIsConst and KeyArgIsPipe.
+ auto TypeQual = cast<MDString>(F.getMetadata(
+ "kernel_arg_type_qual")->getOperand(I))->getString();
+ SmallVector<StringRef, 1> SplitQ;
+ TypeQual.split(SplitQ, " ", -1, false/* drop empty entry*/);
+ for (auto &I:SplitQ) {
+ auto Key = StringSwitch<RuntimeMD::Key>(I)
+ .Case("volatile", RuntimeMD::KeyArgIsVolatile)
+ .Case("restrict", RuntimeMD::KeyArgIsRestrict)
+ .Case("const", RuntimeMD::KeyArgIsConst)
+ .Case("pipe", RuntimeMD::KeyArgIsPipe)
+ .Default(RuntimeMD::KeyNull);
+ OutStreamer->EmitIntValue(Key, 1);
+ }
+
+ // Emit KeyArgTypeKind.
+ auto BaseTypeName = cast<MDString>(
+ F.getMetadata("kernel_arg_base_type")->getOperand(I))->getString();
+ auto TypeKind = StringSwitch<RuntimeMD::KernelArg::TypeKind>(BaseTypeName)
+ .Case("sampler_t", RuntimeMD::KernelArg::Sampler)
+ .Case("queue_t", RuntimeMD::KernelArg::Queue)
+ .Cases("image1d_t", "image1d_array_t", "image1d_buffer_t",
+ "image2d_t" , "image2d_array_t", RuntimeMD::KernelArg::Image)
+ .Cases("image2d_depth_t", "image2d_array_depth_t",
+ "image2d_msaa_t", "image2d_array_msaa_t",
+ "image2d_msaa_depth_t", RuntimeMD::KernelArg::Image)
+ .Cases("image2d_array_msaa_depth_t", "image3d_t",
+ RuntimeMD::KernelArg::Image)
+ .Default(isa<PointerType>(T) ? RuntimeMD::KernelArg::Pointer :
+ RuntimeMD::KernelArg::Value);
+ emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgTypeKind, TypeKind, 1);
+
+ // Emit KeyArgValueType.
+ emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgValueType,
+ getRuntimeMDValueType(T, BaseTypeName), 2);
+
+ // Emit KeyArgAccQual.
+ auto AccQual = cast<MDString>(F.getMetadata(
+ "kernel_arg_access_qual")->getOperand(I))->getString();
+ auto AQ = StringSwitch<RuntimeMD::KernelArg::AccessQualifer>(AccQual)
+ .Case("read_only", RuntimeMD::KernelArg::ReadOnly)
+ .Case("write_only", RuntimeMD::KernelArg::WriteOnly)
+ .Case("read_write", RuntimeMD::KernelArg::ReadWrite)
+ .Default(RuntimeMD::KernelArg::None);
+ emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgAccQual,
+ AQ, 1);
+
+ // Emit KeyArgAddrQual.
+ if (isa<PointerType>(T))
+ emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgAddrQual,
+ T->getPointerAddressSpace(), 1);
+
+ // Emit KeyArgEnd
+ OutStreamer->EmitIntValue(RuntimeMD::KeyArgEnd, 1);
+ }
+
+ // Emit KeyReqdWorkGroupSize, KeyWorkGroupSizeHint, and KeyVecTypeHint.
+ if (auto RWGS = F.getMetadata("reqd_work_group_size"))
+ emitRuntimeMDThreeIntValues(OutStreamer, RuntimeMD::KeyReqdWorkGroupSize,
+ RWGS, 4);
+ if (auto WGSH = F.getMetadata("work_group_size_hint"))
+ emitRuntimeMDThreeIntValues(OutStreamer, RuntimeMD::KeyWorkGroupSizeHint,
+ WGSH, 4);
+ if (auto VTH = F.getMetadata("vec_type_hint")) {
+ auto TypeName = getOCLTypeName(cast<ValueAsMetadata>(
+ VTH->getOperand(0))->getType(), mdconst::extract<ConstantInt>(
+ VTH->getOperand(1))->getZExtValue());
+ emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyVecTypeHint,
+ TypeName);
+ }
+
+ // Emit KeyKernelEnd
+ OutStreamer->EmitIntValue(RuntimeMD::KeyKernelEnd, 1);
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-18 06:44:31 +0000