summaryrefslogtreecommitdiff
path: root/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp')
-rw-r--r--llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp869
1 files changed, 542 insertions, 327 deletions
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 9f359c232981..c1a9b30a509e 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -18,7 +18,9 @@
#include "AMDGPUInstrInfo.h"
#include "AMDGPUTargetMachine.h"
#include "SIMachineFunctionInfo.h"
+#include "Utils/AMDGPUBaseInfo.h"
#include "llvm/ADT/ScopeExit.h"
+#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
@@ -47,7 +49,7 @@ static constexpr unsigned MaxRegisterSize = 1024;
static LLT getPow2VectorType(LLT Ty) {
unsigned NElts = Ty.getNumElements();
unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts);
- return Ty.changeNumElements(Pow2NElts);
+ return Ty.changeElementCount(ElementCount::getFixed(Pow2NElts));
}
// Round the number of bits to the next power of two bits
@@ -93,7 +95,8 @@ static LegalizeMutation oneMoreElement(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
const LLT EltTy = Ty.getElementType();
- return std::make_pair(TypeIdx, LLT::vector(Ty.getNumElements() + 1, EltTy));
+ return std::make_pair(TypeIdx,
+ LLT::fixed_vector(Ty.getNumElements() + 1, EltTy));
};
}
@@ -104,7 +107,9 @@ static LegalizeMutation fewerEltsToSize64Vector(unsigned TypeIdx) {
unsigned Size = Ty.getSizeInBits();
unsigned Pieces = (Size + 63) / 64;
unsigned NewNumElts = (Ty.getNumElements() + 1) / Pieces;
- return std::make_pair(TypeIdx, LLT::scalarOrVector(NewNumElts, EltTy));
+ return std::make_pair(
+ TypeIdx,
+ LLT::scalarOrVector(ElementCount::getFixed(NewNumElts), EltTy));
};
}
@@ -122,7 +127,7 @@ static LegalizeMutation moreEltsToNext32Bit(unsigned TypeIdx) {
assert(EltSize < 32);
const int NewNumElts = (32 * NextMul32 + EltSize - 1) / EltSize;
- return std::make_pair(TypeIdx, LLT::vector(NewNumElts, EltTy));
+ return std::make_pair(TypeIdx, LLT::fixed_vector(NewNumElts, EltTy));
};
}
@@ -136,7 +141,7 @@ static LLT getBitcastRegisterType(const LLT Ty) {
return LLT::scalar(Size);
}
- return LLT::scalarOrVector(Size / 32, 32);
+ return LLT::scalarOrVector(ElementCount::getFixed(Size / 32), 32);
}
static LegalizeMutation bitcastToRegisterType(unsigned TypeIdx) {
@@ -151,7 +156,8 @@ static LegalizeMutation bitcastToVectorElement32(unsigned TypeIdx) {
const LLT Ty = Query.Types[TypeIdx];
unsigned Size = Ty.getSizeInBits();
assert(Size % 32 == 0);
- return std::make_pair(TypeIdx, LLT::scalarOrVector(Size / 32, 32));
+ return std::make_pair(
+ TypeIdx, LLT::scalarOrVector(ElementCount::getFixed(Size / 32), 32));
};
}
@@ -220,11 +226,13 @@ static LegalityPredicate elementTypeIsLegal(unsigned TypeIdx) {
};
}
-static LegalityPredicate isWideScalarTruncStore(unsigned TypeIdx) {
+// If we have a truncating store or an extending load with a data size larger
+// than 32-bits, we need to reduce to a 32-bit type.
+static LegalityPredicate isWideScalarExtLoadTruncStore(unsigned TypeIdx) {
return [=](const LegalityQuery &Query) {
const LLT Ty = Query.Types[TypeIdx];
return !Ty.isVector() && Ty.getSizeInBits() > 32 &&
- Query.MMODescrs[0].SizeInBits < Ty.getSizeInBits();
+ Query.MMODescrs[0].MemoryTy.getSizeInBits() < Ty.getSizeInBits();
};
}
@@ -257,15 +265,14 @@ static unsigned maxSizeForAddrSpace(const GCNSubtarget &ST, unsigned AS,
}
static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
- const LegalityQuery &Query,
- unsigned Opcode) {
+ const LegalityQuery &Query) {
const LLT Ty = Query.Types[0];
// Handle G_LOAD, G_ZEXTLOAD, G_SEXTLOAD
- const bool IsLoad = Opcode != AMDGPU::G_STORE;
+ const bool IsLoad = Query.Opcode != AMDGPU::G_STORE;
unsigned RegSize = Ty.getSizeInBits();
- unsigned MemSize = Query.MMODescrs[0].SizeInBits;
+ unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
unsigned AlignBits = Query.MMODescrs[0].AlignInBits;
unsigned AS = Query.Types[1].getAddressSpace();
@@ -273,6 +280,10 @@ static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
if (AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT)
return false;
+ // Do not handle extending vector loads.
+ if (Ty.isVector() && MemSize != RegSize)
+ return false;
+
// TODO: We should be able to widen loads if the alignment is high enough, but
// we also need to modify the memory access size.
#if 0
@@ -341,33 +352,37 @@ static bool loadStoreBitcastWorkaround(const LLT Ty) {
return EltSize != 32 && EltSize != 64;
}
-static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query,
- unsigned Opcode) {
+static bool isLoadStoreLegal(const GCNSubtarget &ST, const LegalityQuery &Query) {
const LLT Ty = Query.Types[0];
- return isRegisterType(Ty) && isLoadStoreSizeLegal(ST, Query, Opcode) &&
+ return isRegisterType(Ty) && isLoadStoreSizeLegal(ST, Query) &&
!loadStoreBitcastWorkaround(Ty);
}
/// Return true if a load or store of the type should be lowered with a bitcast
/// to a different type.
static bool shouldBitcastLoadStoreType(const GCNSubtarget &ST, const LLT Ty,
- const unsigned MemSizeInBits) {
+ const LLT MemTy) {
+ const unsigned MemSizeInBits = MemTy.getSizeInBits();
const unsigned Size = Ty.getSizeInBits();
- if (Size != MemSizeInBits)
- return Size <= 32 && Ty.isVector();
+ if (Size != MemSizeInBits)
+ return Size <= 32 && Ty.isVector();
if (loadStoreBitcastWorkaround(Ty) && isRegisterType(Ty))
return true;
- return Ty.isVector() && (Size <= 32 || isRegisterSize(Size)) &&
+
+ // Don't try to handle bitcasting vector ext loads for now.
+ return Ty.isVector() && (!MemTy.isVector() || MemTy == Ty) &&
+ (Size <= 32 || isRegisterSize(Size)) &&
!isRegisterVectorElementType(Ty.getElementType());
}
/// Return true if we should legalize a load by widening an odd sized memory
/// access up to the alignment. Note this case when the memory access itself
/// changes, not the size of the result register.
-static bool shouldWidenLoad(const GCNSubtarget &ST, unsigned SizeInBits,
+static bool shouldWidenLoad(const GCNSubtarget &ST, LLT MemoryTy,
unsigned AlignInBits, unsigned AddrSpace,
unsigned Opcode) {
+ unsigned SizeInBits = MemoryTy.getSizeInBits();
// We don't want to widen cases that are naturally legal.
if (isPowerOf2_32(SizeInBits))
return false;
@@ -403,7 +418,7 @@ static bool shouldWidenLoad(const GCNSubtarget &ST, const LegalityQuery &Query,
if (Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic)
return false;
- return shouldWidenLoad(ST, Query.MMODescrs[0].SizeInBits,
+ return shouldWidenLoad(ST, Query.MMODescrs[0].MemoryTy,
Query.MMODescrs[0].AlignInBits,
Query.Types[1].getAddressSpace(), Opcode);
}
@@ -427,35 +442,35 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const LLT S512 = LLT::scalar(512);
const LLT MaxScalar = LLT::scalar(MaxRegisterSize);
- const LLT V2S8 = LLT::vector(2, 8);
- const LLT V2S16 = LLT::vector(2, 16);
- const LLT V4S16 = LLT::vector(4, 16);
+ const LLT V2S8 = LLT::fixed_vector(2, 8);
+ const LLT V2S16 = LLT::fixed_vector(2, 16);
+ const LLT V4S16 = LLT::fixed_vector(4, 16);
- const LLT V2S32 = LLT::vector(2, 32);
- const LLT V3S32 = LLT::vector(3, 32);
- const LLT V4S32 = LLT::vector(4, 32);
- const LLT V5S32 = LLT::vector(5, 32);
- const LLT V6S32 = LLT::vector(6, 32);
- const LLT V7S32 = LLT::vector(7, 32);
- const LLT V8S32 = LLT::vector(8, 32);
- const LLT V9S32 = LLT::vector(9, 32);
- const LLT V10S32 = LLT::vector(10, 32);
- const LLT V11S32 = LLT::vector(11, 32);
- const LLT V12S32 = LLT::vector(12, 32);
- const LLT V13S32 = LLT::vector(13, 32);
- const LLT V14S32 = LLT::vector(14, 32);
- const LLT V15S32 = LLT::vector(15, 32);
- const LLT V16S32 = LLT::vector(16, 32);
- const LLT V32S32 = LLT::vector(32, 32);
+ const LLT V2S32 = LLT::fixed_vector(2, 32);
+ const LLT V3S32 = LLT::fixed_vector(3, 32);
+ const LLT V4S32 = LLT::fixed_vector(4, 32);
+ const LLT V5S32 = LLT::fixed_vector(5, 32);
+ const LLT V6S32 = LLT::fixed_vector(6, 32);
+ const LLT V7S32 = LLT::fixed_vector(7, 32);
+ const LLT V8S32 = LLT::fixed_vector(8, 32);
+ const LLT V9S32 = LLT::fixed_vector(9, 32);
+ const LLT V10S32 = LLT::fixed_vector(10, 32);
+ const LLT V11S32 = LLT::fixed_vector(11, 32);
+ const LLT V12S32 = LLT::fixed_vector(12, 32);
+ const LLT V13S32 = LLT::fixed_vector(13, 32);
+ const LLT V14S32 = LLT::fixed_vector(14, 32);
+ const LLT V15S32 = LLT::fixed_vector(15, 32);
+ const LLT V16S32 = LLT::fixed_vector(16, 32);
+ const LLT V32S32 = LLT::fixed_vector(32, 32);
- const LLT V2S64 = LLT::vector(2, 64);
- const LLT V3S64 = LLT::vector(3, 64);
- const LLT V4S64 = LLT::vector(4, 64);
- const LLT V5S64 = LLT::vector(5, 64);
- const LLT V6S64 = LLT::vector(6, 64);
- const LLT V7S64 = LLT::vector(7, 64);
- const LLT V8S64 = LLT::vector(8, 64);
- const LLT V16S64 = LLT::vector(16, 64);
+ const LLT V2S64 = LLT::fixed_vector(2, 64);
+ const LLT V3S64 = LLT::fixed_vector(3, 64);
+ const LLT V4S64 = LLT::fixed_vector(4, 64);
+ const LLT V5S64 = LLT::fixed_vector(5, 64);
+ const LLT V6S64 = LLT::fixed_vector(6, 64);
+ const LLT V7S64 = LLT::fixed_vector(7, 64);
+ const LLT V8S64 = LLT::fixed_vector(8, 64);
+ const LLT V16S64 = LLT::fixed_vector(16, 64);
std::initializer_list<LLT> AllS32Vectors =
{V2S32, V3S32, V4S32, V5S32, V6S32, V7S32, V8S32,
@@ -495,8 +510,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const LLT MinScalarFPTy = ST.has16BitInsts() ? S16 : S32;
- setAction({G_BRCOND, S1}, Legal); // VCC branches
- setAction({G_BRCOND, S32}, Legal); // SCC branches
+ // s1 for VCC branches, s32 for SCC branches.
+ getActionDefinitionsBuilder(G_BRCOND).legalFor({S1, S32});
// TODO: All multiples of 32, vectors of pointers, all v2s16 pairs, more
// elements for v3s16
@@ -579,11 +594,12 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.lower();
}
- getActionDefinitionsBuilder({G_SDIV, G_UDIV, G_SREM, G_UREM})
- .customFor({S32, S64})
- .clampScalar(0, S32, S64)
- .widenScalarToNextPow2(0, 32)
- .scalarize(0);
+ getActionDefinitionsBuilder(
+ {G_SDIV, G_UDIV, G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
+ .customFor({S32, S64})
+ .clampScalar(0, S32, S64)
+ .widenScalarToNextPow2(0, 32)
+ .scalarize(0);
auto &Mulh = getActionDefinitionsBuilder({G_UMULH, G_SMULH})
.legalFor({S32})
@@ -643,7 +659,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.widenScalarToNextPow2(0, 32)
.clampMaxNumElements(0, S32, 16);
- setAction({G_FRAME_INDEX, PrivatePtr}, Legal);
+ getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({PrivatePtr});
// If the amount is divergent, we have to do a wave reduction to get the
// maximum value, so this is expanded during RegBankSelect.
@@ -653,7 +669,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
getActionDefinitionsBuilder(G_GLOBAL_VALUE)
.customIf(typeIsNot(0, PrivatePtr));
- setAction({G_BLOCK_ADDR, CodePtr}, Legal);
+ getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({CodePtr});
auto &FPOpActions = getActionDefinitionsBuilder(
{ G_FADD, G_FMUL, G_FMA, G_FCANONICALIZE})
@@ -809,7 +825,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
auto &FPToI = getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
.legalFor({{S32, S32}, {S32, S64}, {S32, S16}})
- .customFor({{S64, S64}})
+ .customFor({{S64, S32}, {S64, S64}})
.narrowScalarFor({{S64, S16}}, changeTo(0, S32));
if (ST.has16BitInsts())
FPToI.legalFor({{S16, S16}});
@@ -817,6 +833,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
FPToI.minScalar(1, S32);
FPToI.minScalar(0, S32)
+ .widenScalarToNextPow2(0, 32)
.scalarize(0)
.lower();
@@ -935,10 +952,13 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.widenScalarToNextPow2(0, 32)
.widenScalarToNextPow2(1, 32);
+ // S64 is only legal on SALU, and needs to be broken into 32-bit elements in
+ // RegBankSelect.
getActionDefinitionsBuilder(G_BITREVERSE)
- .legalFor({S32})
- .clampScalar(0, S32, S32)
- .scalarize(0);
+ .legalFor({S32, S64})
+ .clampScalar(0, S32, S64)
+ .scalarize(0)
+ .widenScalarToNextPow2(0);
if (ST.has16BitInsts()) {
getActionDefinitionsBuilder(G_BSWAP)
@@ -951,7 +971,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.scalarize(0);
if (ST.hasVOP3PInsts()) {
- getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
+ getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
.legalFor({S32, S16, V2S16})
.moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
.clampMaxNumElements(0, S16, 2)
@@ -960,7 +980,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.scalarize(0)
.lower();
} else {
- getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
+ getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
.legalFor({S32, S16})
.widenScalarToNextPow2(0)
.minScalar(0, S16)
@@ -979,7 +999,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.scalarize(0)
.lower();
- getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
+ getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX, G_ABS})
.legalFor({S32})
.minScalar(0, S32)
.widenScalarToNextPow2(0)
@@ -1029,7 +1049,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const LLT DstTy = Query.Types[0];
// Split vector extloads.
- unsigned MemSize = Query.MMODescrs[0].SizeInBits;
+ unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
unsigned AlignBits = Query.MMODescrs[0].AlignInBits;
if (MemSize < DstTy.getSizeInBits())
@@ -1078,35 +1098,35 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
auto &Actions = getActionDefinitionsBuilder(Op);
// Explicitly list some common cases.
// TODO: Does this help compile time at all?
- Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, 32, GlobalAlign32},
- {V2S32, GlobalPtr, 64, GlobalAlign32},
- {V4S32, GlobalPtr, 128, GlobalAlign32},
- {S64, GlobalPtr, 64, GlobalAlign32},
- {V2S64, GlobalPtr, 128, GlobalAlign32},
- {V2S16, GlobalPtr, 32, GlobalAlign32},
- {S32, GlobalPtr, 8, GlobalAlign8},
- {S32, GlobalPtr, 16, GlobalAlign16},
+ Actions.legalForTypesWithMemDesc({{S32, GlobalPtr, S32, GlobalAlign32},
+ {V2S32, GlobalPtr, V2S32, GlobalAlign32},
+ {V4S32, GlobalPtr, V4S32, GlobalAlign32},
+ {S64, GlobalPtr, S64, GlobalAlign32},
+ {V2S64, GlobalPtr, V2S64, GlobalAlign32},
+ {V2S16, GlobalPtr, V2S16, GlobalAlign32},
+ {S32, GlobalPtr, S8, GlobalAlign8},
+ {S32, GlobalPtr, S16, GlobalAlign16},
- {S32, LocalPtr, 32, 32},
- {S64, LocalPtr, 64, 32},
- {V2S32, LocalPtr, 64, 32},
- {S32, LocalPtr, 8, 8},
- {S32, LocalPtr, 16, 16},
- {V2S16, LocalPtr, 32, 32},
+ {S32, LocalPtr, S32, 32},
+ {S64, LocalPtr, S64, 32},
+ {V2S32, LocalPtr, V2S32, 32},
+ {S32, LocalPtr, S8, 8},
+ {S32, LocalPtr, S16, 16},
+ {V2S16, LocalPtr, S32, 32},
- {S32, PrivatePtr, 32, 32},
- {S32, PrivatePtr, 8, 8},
- {S32, PrivatePtr, 16, 16},
- {V2S16, PrivatePtr, 32, 32},
+ {S32, PrivatePtr, S32, 32},
+ {S32, PrivatePtr, S8, 8},
+ {S32, PrivatePtr, S16, 16},
+ {V2S16, PrivatePtr, S32, 32},
- {S32, ConstantPtr, 32, GlobalAlign32},
- {V2S32, ConstantPtr, 64, GlobalAlign32},
- {V4S32, ConstantPtr, 128, GlobalAlign32},
- {S64, ConstantPtr, 64, GlobalAlign32},
- {V2S32, ConstantPtr, 32, GlobalAlign32}});
+ {S32, ConstantPtr, S32, GlobalAlign32},
+ {V2S32, ConstantPtr, V2S32, GlobalAlign32},
+ {V4S32, ConstantPtr, V4S32, GlobalAlign32},
+ {S64, ConstantPtr, S64, GlobalAlign32},
+ {V2S32, ConstantPtr, V2S32, GlobalAlign32}});
Actions.legalIf(
[=](const LegalityQuery &Query) -> bool {
- return isLoadStoreLegal(ST, Query, Op);
+ return isLoadStoreLegal(ST, Query);
});
// Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
@@ -1125,7 +1145,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
Actions.bitcastIf(
[=](const LegalityQuery &Query) -> bool {
return shouldBitcastLoadStoreType(ST, Query.Types[0],
- Query.MMODescrs[0].SizeInBits);
+ Query.MMODescrs[0].MemoryTy);
}, bitcastToRegisterType(0));
if (!IsStore) {
@@ -1148,7 +1168,7 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const LLT PtrTy = Query.Types[1];
const unsigned DstSize = DstTy.getSizeInBits();
- unsigned MemSize = Query.MMODescrs[0].SizeInBits;
+ unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
// Split extloads.
if (DstSize > MemSize)
@@ -1196,16 +1216,18 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// FIXME: 3 element stores scalarized on SI
// Split if it's too large for the address space.
- if (Query.MMODescrs[0].SizeInBits > MaxSize) {
+ unsigned MemSize = Query.MMODescrs[0].MemoryTy.getSizeInBits();
+ if (MemSize > MaxSize) {
unsigned NumElts = DstTy.getNumElements();
unsigned EltSize = EltTy.getSizeInBits();
if (MaxSize % EltSize == 0) {
return std::make_pair(
- 0, LLT::scalarOrVector(MaxSize / EltSize, EltTy));
+ 0, LLT::scalarOrVector(
+ ElementCount::getFixed(MaxSize / EltSize), EltTy));
}
- unsigned NumPieces = Query.MMODescrs[0].SizeInBits / MaxSize;
+ unsigned NumPieces = MemSize / MaxSize;
// FIXME: Refine when odd breakdowns handled
// The scalars will need to be re-legalized.
@@ -1213,12 +1235,11 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
NumElts % NumPieces != 0)
return std::make_pair(0, EltTy);
- return std::make_pair(0,
- LLT::vector(NumElts / NumPieces, EltTy));
+ return std::make_pair(
+ 0, LLT::fixed_vector(NumElts / NumPieces, EltTy));
}
// FIXME: We could probably handle weird extending loads better.
- unsigned MemSize = Query.MMODescrs[0].SizeInBits;
if (DstTy.getSizeInBits() > MemSize)
return std::make_pair(0, EltTy);
@@ -1230,48 +1251,58 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// should be OK, since the new parts will be further legalized.
unsigned FloorSize = PowerOf2Floor(DstSize);
return std::make_pair(
- 0, LLT::scalarOrVector(FloorSize / EltSize, EltTy));
+ 0, LLT::scalarOrVector(
+ ElementCount::getFixed(FloorSize / EltSize), EltTy));
}
// Need to split because of alignment.
unsigned Align = Query.MMODescrs[0].AlignInBits;
if (EltSize > Align &&
(EltSize / Align < DstTy.getNumElements())) {
- return std::make_pair(0, LLT::vector(EltSize / Align, EltTy));
+ return std::make_pair(
+ 0, LLT::fixed_vector(EltSize / Align, EltTy));
}
// May need relegalization for the scalars.
return std::make_pair(0, EltTy);
})
.lowerIfMemSizeNotPow2()
- .minScalar(0, S32);
-
- if (IsStore)
- Actions.narrowScalarIf(isWideScalarTruncStore(0), changeTo(0, S32));
-
- Actions
- .widenScalarToNextPow2(0)
- .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
- .lower();
+ .minScalar(0, S32)
+ .narrowScalarIf(isWideScalarExtLoadTruncStore(0), changeTo(0, S32))
+ .widenScalarToNextPow2(0)
+ .moreElementsIf(vectorSmallerThan(0, 32), moreEltsToNext32Bit(0))
+ .lower();
}
+ // FIXME: Unaligned accesses not lowered.
auto &ExtLoads = getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
- .legalForTypesWithMemDesc({{S32, GlobalPtr, 8, 8},
- {S32, GlobalPtr, 16, 2 * 8},
- {S32, LocalPtr, 8, 8},
- {S32, LocalPtr, 16, 16},
- {S32, PrivatePtr, 8, 8},
- {S32, PrivatePtr, 16, 16},
- {S32, ConstantPtr, 8, 8},
- {S32, ConstantPtr, 16, 2 * 8}});
+ .legalForTypesWithMemDesc({{S32, GlobalPtr, S8, 8},
+ {S32, GlobalPtr, S16, 2 * 8},
+ {S32, LocalPtr, S8, 8},
+ {S32, LocalPtr, S16, 16},
+ {S32, PrivatePtr, S8, 8},
+ {S32, PrivatePtr, S16, 16},
+ {S32, ConstantPtr, S8, 8},
+ {S32, ConstantPtr, S16, 2 * 8}})
+ .legalIf(
+ [=](const LegalityQuery &Query) -> bool {
+ return isLoadStoreLegal(ST, Query);
+ });
+
if (ST.hasFlatAddressSpace()) {
ExtLoads.legalForTypesWithMemDesc(
- {{S32, FlatPtr, 8, 8}, {S32, FlatPtr, 16, 16}});
+ {{S32, FlatPtr, S8, 8}, {S32, FlatPtr, S16, 16}});
}
+ // Constant 32-bit is handled by addrspacecasting the 32-bit pointer to
+ // 64-bits.
+ //
+ // TODO: Should generalize bitcast action into coerce, which will also cover
+ // inserting addrspacecasts.
+ ExtLoads.customIf(typeIs(1, Constant32Ptr));
+
ExtLoads.clampScalar(0, S32, S32)
.widenScalarToNextPow2(0)
- .unsupportedIfMemSizeNotPow2()
.lower();
auto &Atomics = getActionDefinitionsBuilder(
@@ -1286,10 +1317,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
Atomics.legalFor({{S32, FlatPtr}, {S64, FlatPtr}});
}
+ auto &Atomic = getActionDefinitionsBuilder(G_ATOMICRMW_FADD);
if (ST.hasLDSFPAtomics()) {
- getActionDefinitionsBuilder(G_ATOMICRMW_FADD)
- .legalFor({{S32, LocalPtr}, {S32, RegionPtr}});
+ Atomic.legalFor({{S32, LocalPtr}, {S32, RegionPtr}});
+ if (ST.hasGFX90AInsts())
+ Atomic.legalFor({{S64, LocalPtr}});
}
+ if (ST.hasAtomicFaddInsts())
+ Atomic.legalFor({{S32, GlobalPtr}});
// BUFFER/FLAT_ATOMIC_CMP_SWAP on GCN GPUs needs input marshalling, and output
// demarshalling
@@ -1302,19 +1337,21 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
// Condition should be s32 for scalar, s1 for vector.
getActionDefinitionsBuilder(G_SELECT)
- .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16,
- GlobalPtr, LocalPtr, FlatPtr, PrivatePtr,
- LLT::vector(2, LocalPtr), LLT::vector(2, PrivatePtr)}, {S1, S32})
- .clampScalar(0, S16, S64)
- .scalarize(1)
- .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
- .fewerElementsIf(numElementsNotEven(0), scalarize(0))
- .clampMaxNumElements(0, S32, 2)
- .clampMaxNumElements(0, LocalPtr, 2)
- .clampMaxNumElements(0, PrivatePtr, 2)
- .scalarize(0)
- .widenScalarToNextPow2(0)
- .legalIf(all(isPointer(0), typeInSet(1, {S1, S32})));
+ .legalForCartesianProduct({S32, S64, S16, V2S32, V2S16, V4S16, GlobalPtr,
+ LocalPtr, FlatPtr, PrivatePtr,
+ LLT::fixed_vector(2, LocalPtr),
+ LLT::fixed_vector(2, PrivatePtr)},
+ {S1, S32})
+ .clampScalar(0, S16, S64)
+ .scalarize(1)
+ .moreElementsIf(isSmallOddVector(0), oneMoreElement(0))
+ .fewerElementsIf(numElementsNotEven(0), scalarize(0))
+ .clampMaxNumElements(0, S32, 2)
+ .clampMaxNumElements(0, LocalPtr, 2)
+ .clampMaxNumElements(0, PrivatePtr, 2)
+ .scalarize(0)
+ .widenScalarToNextPow2(0)
+ .legalIf(all(isPointer(0), typeInSet(1, {S1, S32})));
// TODO: Only the low 4/5/6 bits of the shift amount are observed, so we can
// be more flexible with the shift amount type.
@@ -1393,7 +1430,8 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
const unsigned TargetEltSize = DstEltSize % 64 == 0 ? 64 : 32;
return std::make_pair(
- VecTypeIdx, LLT::vector(VecSize / TargetEltSize, TargetEltSize));
+ VecTypeIdx,
+ LLT::fixed_vector(VecSize / TargetEltSize, TargetEltSize));
})
.clampScalar(EltTypeIdx, S32, S64)
.clampScalar(VecTypeIdx, S32, S64)
@@ -1590,17 +1628,44 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
.clampScalar(0, S32, S64)
.lower();
+ // TODO: Only Try to form v2s16 with legal packed instructions.
getActionDefinitionsBuilder(G_FSHR)
.legalFor({{S32, S32}})
+ .lowerFor({{V2S16, V2S16}})
+ .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16))
.scalarize(0)
.lower();
+ if (ST.hasVOP3PInsts()) {
+ getActionDefinitionsBuilder(G_FSHL)
+ .lowerFor({{V2S16, V2S16}})
+ .fewerElementsIf(elementTypeIs(0, S16), changeTo(0, V2S16))
+ .scalarize(0)
+ .lower();
+ } else {
+ getActionDefinitionsBuilder(G_FSHL)
+ .scalarize(0)
+ .lower();
+ }
+
getActionDefinitionsBuilder(G_READCYCLECOUNTER)
.legalFor({S64});
getActionDefinitionsBuilder(G_FENCE)
.alwaysLegal();
+ getActionDefinitionsBuilder({G_SMULO, G_UMULO})
+ .scalarize(0)
+ .minScalar(0, S32)
+ .lower();
+
+ getActionDefinitionsBuilder({G_SBFX, G_UBFX})
+ .legalFor({{S32, S32}, {S64, S32}})
+ .clampScalar(1, S32, S32)
+ .clampScalar(0, S32, S64)
+ .widenScalarToNextPow2(0)
+ .scalarize(0);
+
getActionDefinitionsBuilder({
// TODO: Verify V_BFI_B32 is generated from expanded bit ops
G_FCOPYSIGN,
@@ -1614,16 +1679,14 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
G_SADDO, G_SSUBO,
// TODO: Implement
- G_FMINIMUM, G_FMAXIMUM,
- G_FSHL
- }).lower();
+ G_FMINIMUM, G_FMAXIMUM}).lower();
getActionDefinitionsBuilder({G_VASTART, G_VAARG, G_BRJT, G_JUMP_TABLE,
G_INDEXED_LOAD, G_INDEXED_SEXTLOAD,
G_INDEXED_ZEXTLOAD, G_INDEXED_STORE})
.unsupported();
- computeTables();
+ getLegacyLegalizerInfo().computeTables();
verify(*ST.getInstrInfo());
}
@@ -1668,6 +1731,8 @@ bool AMDGPULegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
case TargetOpcode::G_GLOBAL_VALUE:
return legalizeGlobalValue(MI, MRI, B);
case TargetOpcode::G_LOAD:
+ case TargetOpcode::G_SEXTLOAD:
+ case TargetOpcode::G_ZEXTLOAD:
return legalizeLoad(Helper, MI);
case TargetOpcode::G_FMAD:
return legalizeFMad(MI, MRI, B);
@@ -1675,10 +1740,12 @@ bool AMDGPULegalizerInfo::legalizeCustom(LegalizerHelper &Helper,
return legalizeFDIV(MI, MRI, B);
case TargetOpcode::G_UDIV:
case TargetOpcode::G_UREM:
- return legalizeUDIV_UREM(MI, MRI, B);
+ case TargetOpcode::G_UDIVREM:
+ return legalizeUnsignedDIV_REM(MI, MRI, B);
case TargetOpcode::G_SDIV:
case TargetOpcode::G_SREM:
- return legalizeSDIV_SREM(MI, MRI, B);
+ case TargetOpcode::G_SDIVREM:
+ return legalizeSignedDIV_REM(MI, MRI, B);
case TargetOpcode::G_ATOMIC_CMPXCHG:
return legalizeAtomicCmpXChg(MI, MRI, B);
case TargetOpcode::G_FLOG:
@@ -1751,7 +1818,7 @@ Register AMDGPULegalizerInfo::getSegmentAperture(
PtrInfo,
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant,
- 4, commonAlignment(Align(64), StructOffset));
+ LLT::scalar(32), commonAlignment(Align(64), StructOffset));
Register LoadAddr;
@@ -2021,9 +2088,10 @@ bool AMDGPULegalizerInfo::legalizeITOFP(
// TODO: Copied from DAG implementation. Verify logic and document how this
// actually works.
-bool AMDGPULegalizerInfo::legalizeFPTOI(
- MachineInstr &MI, MachineRegisterInfo &MRI,
- MachineIRBuilder &B, bool Signed) const {
+bool AMDGPULegalizerInfo::legalizeFPTOI(MachineInstr &MI,
+ MachineRegisterInfo &MRI,
+ MachineIRBuilder &B,
+ bool Signed) const {
Register Dst = MI.getOperand(0).getReg();
Register Src = MI.getOperand(1).getReg();
@@ -2031,24 +2099,57 @@ bool AMDGPULegalizerInfo::legalizeFPTOI(
const LLT S64 = LLT::scalar(64);
const LLT S32 = LLT::scalar(32);
- assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S64);
+ const LLT SrcLT = MRI.getType(Src);
+ assert((SrcLT == S32 || SrcLT == S64) && MRI.getType(Dst) == S64);
unsigned Flags = MI.getFlags();
- auto Trunc = B.buildIntrinsicTrunc(S64, Src, Flags);
- auto K0 = B.buildFConstant(S64, BitsToDouble(UINT64_C(0x3df0000000000000)));
- auto K1 = B.buildFConstant(S64, BitsToDouble(UINT64_C(0xc1f0000000000000)));
+ // The basic idea of converting a floating point number into a pair of 32-bit
+ // integers is illustrated as follows:
+ //
+ // tf := trunc(val);
+ // hif := floor(tf * 2^-32);
+ // lof := tf - hif * 2^32; // lof is always positive due to floor.
+ // hi := fptoi(hif);
+ // lo := fptoi(lof);
+ //
+ auto Trunc = B.buildIntrinsicTrunc(SrcLT, Src, Flags);
+ MachineInstrBuilder Sign;
+ if (Signed && SrcLT == S32) {
+ // However, a 32-bit floating point number has only 23 bits mantissa and
+ // it's not enough to hold all the significant bits of `lof` if val is
+ // negative. To avoid the loss of precision, We need to take the absolute
+ // value after truncating and flip the result back based on the original
+ // signedness.
+ Sign = B.buildAShr(S32, Src, B.buildConstant(S32, 31));
+ Trunc = B.buildFAbs(S32, Trunc, Flags);
+ }
+ MachineInstrBuilder K0, K1;
+ if (SrcLT == S64) {
+ K0 = B.buildFConstant(S64,
+ BitsToDouble(UINT64_C(/*2^-32*/ 0x3df0000000000000)));
+ K1 = B.buildFConstant(S64,
+ BitsToDouble(UINT64_C(/*-2^32*/ 0xc1f0000000000000)));
+ } else {
+ K0 = B.buildFConstant(S32, BitsToFloat(UINT32_C(/*2^-32*/ 0x2f800000)));
+ K1 = B.buildFConstant(S32, BitsToFloat(UINT32_C(/*-2^32*/ 0xcf800000)));
+ }
- auto Mul = B.buildFMul(S64, Trunc, K0, Flags);
- auto FloorMul = B.buildFFloor(S64, Mul, Flags);
- auto Fma = B.buildFMA(S64, FloorMul, K1, Trunc, Flags);
+ auto Mul = B.buildFMul(SrcLT, Trunc, K0, Flags);
+ auto FloorMul = B.buildFFloor(SrcLT, Mul, Flags);
+ auto Fma = B.buildFMA(SrcLT, FloorMul, K1, Trunc, Flags);
- auto Hi = Signed ?
- B.buildFPTOSI(S32, FloorMul) :
- B.buildFPTOUI(S32, FloorMul);
+ auto Hi = (Signed && SrcLT == S64) ? B.buildFPTOSI(S32, FloorMul)
+ : B.buildFPTOUI(S32, FloorMul);
auto Lo = B.buildFPTOUI(S32, Fma);
- B.buildMerge(Dst, { Lo, Hi });
+ if (Signed && SrcLT == S32) {
+ // Flip the result based on the signedness, which is either all 0s or 1s.
+ Sign = B.buildMerge(S64, {Sign, Sign});
+ // r := xor({lo, hi}, sign) - sign;
+ B.buildSub(Dst, B.buildXor(S64, B.buildMerge(S64, {Lo, Hi}), Sign), Sign);
+ } else
+ B.buildMerge(Dst, {Lo, Hi});
MI.eraseFromParent();
return true;
@@ -2141,7 +2242,7 @@ bool AMDGPULegalizerInfo::legalizeInsertVectorElt(
bool AMDGPULegalizerInfo::legalizeShuffleVector(
MachineInstr &MI, MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
- const LLT V2S16 = LLT::vector(2, 16);
+ const LLT V2S16 = LLT::fixed_vector(2, 16);
Register Dst = MI.getOperand(0).getReg();
Register Src0 = MI.getOperand(1).getReg();
@@ -2258,7 +2359,8 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue(
SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
if (AS == AMDGPUAS::LOCAL_ADDRESS || AS == AMDGPUAS::REGION_ADDRESS) {
- if (!MFI->isModuleEntryFunction()) {
+ if (!MFI->isModuleEntryFunction() &&
+ !GV->getName().equals("llvm.amdgcn.module.lds")) {
const Function &Fn = MF.getFunction();
DiagnosticInfoUnsupported BadLDSDecl(
Fn, "local memory global used by non-kernel function", MI.getDebugLoc(),
@@ -2334,11 +2436,12 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue(
LLT PtrTy = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
Register GOTAddr = MRI.createGenericVirtualRegister(PtrTy);
+ LLT LoadTy = Ty.getSizeInBits() == 32 ? PtrTy : Ty;
MachineMemOperand *GOTMMO = MF.getMachineMemOperand(
MachinePointerInfo::getGOT(MF),
MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
MachineMemOperand::MOInvariant,
- 8 /*Size*/, Align(8));
+ LoadTy, Align(8));
buildPCRelGlobalAddress(GOTAddr, PtrTy, B, GV, 0, SIInstrInfo::MO_GOTPCREL32);
@@ -2355,7 +2458,8 @@ bool AMDGPULegalizerInfo::legalizeGlobalValue(
static LLT widenToNextPowerOf2(LLT Ty) {
if (Ty.isVector())
- return Ty.changeNumElements(PowerOf2Ceil(Ty.getNumElements()));
+ return Ty.changeElementCount(
+ ElementCount::getFixed(PowerOf2Ceil(Ty.getNumElements())));
return LLT::scalar(PowerOf2Ceil(Ty.getSizeInBits()));
}
@@ -2378,17 +2482,21 @@ bool AMDGPULegalizerInfo::legalizeLoad(LegalizerHelper &Helper,
return true;
}
+ if (MI.getOpcode() != AMDGPU::G_LOAD)
+ return false;
+
Register ValReg = MI.getOperand(0).getReg();
LLT ValTy = MRI.getType(ValReg);
MachineMemOperand *MMO = *MI.memoperands_begin();
const unsigned ValSize = ValTy.getSizeInBits();
- const unsigned MemSize = 8 * MMO->getSize();
+ const LLT MemTy = MMO->getMemoryType();
const Align MemAlign = MMO->getAlign();
+ const unsigned MemSize = MemTy.getSizeInBits();
const unsigned AlignInBits = 8 * MemAlign.value();
// Widen non-power-of-2 loads to the alignment if needed
- if (shouldWidenLoad(ST, MemSize, AlignInBits, AddrSpace, MI.getOpcode())) {
+ if (shouldWidenLoad(ST, MemTy, AlignInBits, AddrSpace, MI.getOpcode())) {
const unsigned WideMemSize = PowerOf2Ceil(MemSize);
// This was already the correct extending load result type, so just adjust
@@ -2472,7 +2580,7 @@ bool AMDGPULegalizerInfo::legalizeAtomicCmpXChg(
"this should not have been custom lowered");
LLT ValTy = MRI.getType(CmpVal);
- LLT VecTy = LLT::vector(2, ValTy);
+ LLT VecTy = LLT::fixed_vector(2, ValTy);
Register PackedVal = B.buildBuildVector(VecTy, { NewVal, CmpVal }).getReg(0);
@@ -2624,7 +2732,7 @@ bool AMDGPULegalizerInfo::legalizeBuildVector(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
Register Dst = MI.getOperand(0).getReg();
const LLT S32 = LLT::scalar(32);
- assert(MRI.getType(Dst) == LLT::vector(2, 16));
+ assert(MRI.getType(Dst) == LLT::fixed_vector(2, 16));
Register Src0 = MI.getOperand(1).getReg();
Register Src1 = MI.getOperand(2).getReg();
@@ -2762,11 +2870,11 @@ bool AMDGPULegalizerInfo::legalizeFDIV(MachineInstr &MI,
return false;
}
-void AMDGPULegalizerInfo::legalizeUDIV_UREM32Impl(MachineIRBuilder &B,
- Register DstReg,
- Register X,
- Register Y,
- bool IsDiv) const {
+void AMDGPULegalizerInfo::legalizeUnsignedDIV_REM32Impl(MachineIRBuilder &B,
+ Register DstDivReg,
+ Register DstRemReg,
+ Register X,
+ Register Y) const {
const LLT S1 = LLT::scalar(1);
const LLT S32 = LLT::scalar(32);
@@ -2792,28 +2900,17 @@ void AMDGPULegalizerInfo::legalizeUDIV_UREM32Impl(MachineIRBuilder &B,
// First quotient/remainder refinement.
auto One = B.buildConstant(S32, 1);
auto Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
- if (IsDiv)
+ if (DstDivReg)
Q = B.buildSelect(S32, Cond, B.buildAdd(S32, Q, One), Q);
R = B.buildSelect(S32, Cond, B.buildSub(S32, R, Y), R);
// Second quotient/remainder refinement.
Cond = B.buildICmp(CmpInst::ICMP_UGE, S1, R, Y);
- if (IsDiv)
- B.buildSelect(DstReg, Cond, B.buildAdd(S32, Q, One), Q);
- else
- B.buildSelect(DstReg, Cond, B.buildSub(S32, R, Y), R);
-}
+ if (DstDivReg)
+ B.buildSelect(DstDivReg, Cond, B.buildAdd(S32, Q, One), Q);
-bool AMDGPULegalizerInfo::legalizeUDIV_UREM32(MachineInstr &MI,
- MachineRegisterInfo &MRI,
- MachineIRBuilder &B) const {
- const bool IsDiv = MI.getOpcode() == AMDGPU::G_UDIV;
- Register DstReg = MI.getOperand(0).getReg();
- Register Num = MI.getOperand(1).getReg();
- Register Den = MI.getOperand(2).getReg();
- legalizeUDIV_UREM32Impl(B, DstReg, Num, Den, IsDiv);
- MI.eraseFromParent();
- return true;
+ if (DstRemReg)
+ B.buildSelect(DstRemReg, Cond, B.buildSub(S32, R, Y), R);
}
// Build integer reciprocal sequence arounud V_RCP_IFLAG_F32
@@ -2859,11 +2956,11 @@ static std::pair<Register, Register> emitReciprocalU64(MachineIRBuilder &B,
return {ResultLo.getReg(0), ResultHi.getReg(0)};
}
-void AMDGPULegalizerInfo::legalizeUDIV_UREM64Impl(MachineIRBuilder &B,
- Register DstReg,
- Register Numer,
- Register Denom,
- bool IsDiv) const {
+void AMDGPULegalizerInfo::legalizeUnsignedDIV_REM64Impl(MachineIRBuilder &B,
+ Register DstDivReg,
+ Register DstRemReg,
+ Register Numer,
+ Register Denom) const {
const LLT S32 = LLT::scalar(32);
const LLT S64 = LLT::scalar(64);
const LLT S1 = LLT::scalar(1);
@@ -2959,57 +3056,74 @@ void AMDGPULegalizerInfo::legalizeUDIV_UREM64Impl(MachineIRBuilder &B,
// endif C6
// endif C3
- if (IsDiv) {
+ if (DstDivReg) {
auto Sel1 = B.buildSelect(
S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Add4, Add3);
- B.buildSelect(DstReg,
- B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32), Sel1, MulHi3);
- } else {
+ B.buildSelect(DstDivReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
+ Sel1, MulHi3);
+ }
+
+ if (DstRemReg) {
auto Sel2 = B.buildSelect(
S64, B.buildICmp(CmpInst::ICMP_NE, S1, C6, Zero32), Sub3, Sub2);
- B.buildSelect(DstReg,
- B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32), Sel2, Sub1);
+ B.buildSelect(DstRemReg, B.buildICmp(CmpInst::ICMP_NE, S1, C3, Zero32),
+ Sel2, Sub1);
}
}
-bool AMDGPULegalizerInfo::legalizeUDIV_UREM(MachineInstr &MI,
- MachineRegisterInfo &MRI,
- MachineIRBuilder &B) const {
+bool AMDGPULegalizerInfo::legalizeUnsignedDIV_REM(MachineInstr &MI,
+ MachineRegisterInfo &MRI,
+ MachineIRBuilder &B) const {
+ Register DstDivReg, DstRemReg;
+ switch (MI.getOpcode()) {
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case AMDGPU::G_UDIV: {
+ DstDivReg = MI.getOperand(0).getReg();
+ break;
+ }
+ case AMDGPU::G_UREM: {
+ DstRemReg = MI.getOperand(0).getReg();
+ break;
+ }
+ case AMDGPU::G_UDIVREM: {
+ DstDivReg = MI.getOperand(0).getReg();
+ DstRemReg = MI.getOperand(1).getReg();
+ break;
+ }
+ }
+
const LLT S64 = LLT::scalar(64);
const LLT S32 = LLT::scalar(32);
- const bool IsDiv = MI.getOpcode() == AMDGPU::G_UDIV;
- Register DstReg = MI.getOperand(0).getReg();
- Register Num = MI.getOperand(1).getReg();
- Register Den = MI.getOperand(2).getReg();
- LLT Ty = MRI.getType(DstReg);
+ const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
+ Register Num = MI.getOperand(FirstSrcOpIdx).getReg();
+ Register Den = MI.getOperand(FirstSrcOpIdx + 1).getReg();
+ LLT Ty = MRI.getType(MI.getOperand(0).getReg());
if (Ty == S32)
- legalizeUDIV_UREM32Impl(B, DstReg, Num, Den, IsDiv);
+ legalizeUnsignedDIV_REM32Impl(B, DstDivReg, DstRemReg, Num, Den);
else if (Ty == S64)
- legalizeUDIV_UREM64Impl(B, DstReg, Num, Den, IsDiv);
+ legalizeUnsignedDIV_REM64Impl(B, DstDivReg, DstRemReg, Num, Den);
else
return false;
MI.eraseFromParent();
return true;
-
}
-bool AMDGPULegalizerInfo::legalizeSDIV_SREM(MachineInstr &MI,
- MachineRegisterInfo &MRI,
- MachineIRBuilder &B) const {
+bool AMDGPULegalizerInfo::legalizeSignedDIV_REM(MachineInstr &MI,
+ MachineRegisterInfo &MRI,
+ MachineIRBuilder &B) const {
const LLT S64 = LLT::scalar(64);
const LLT S32 = LLT::scalar(32);
- Register DstReg = MI.getOperand(0).getReg();
- const LLT Ty = MRI.getType(DstReg);
+ LLT Ty = MRI.getType(MI.getOperand(0).getReg());
if (Ty != S32 && Ty != S64)
return false;
- const bool IsDiv = MI.getOpcode() == AMDGPU::G_SDIV;
-
- Register LHS = MI.getOperand(1).getReg();
- Register RHS = MI.getOperand(2).getReg();
+ const unsigned FirstSrcOpIdx = MI.getNumExplicitDefs();
+ Register LHS = MI.getOperand(FirstSrcOpIdx).getReg();
+ Register RHS = MI.getOperand(FirstSrcOpIdx + 1).getReg();
auto SignBitOffset = B.buildConstant(S32, Ty.getSizeInBits() - 1);
auto LHSign = B.buildAShr(Ty, LHS, SignBitOffset);
@@ -3021,20 +3135,45 @@ bool AMDGPULegalizerInfo::legalizeSDIV_SREM(MachineInstr &MI,
LHS = B.buildXor(Ty, LHS, LHSign).getReg(0);
RHS = B.buildXor(Ty, RHS, RHSign).getReg(0);
- Register UDivRem = MRI.createGenericVirtualRegister(Ty);
+ Register DstDivReg, DstRemReg, TmpDivReg, TmpRemReg;
+ switch (MI.getOpcode()) {
+ default:
+ llvm_unreachable("Unexpected opcode!");
+ case AMDGPU::G_SDIV: {
+ DstDivReg = MI.getOperand(0).getReg();
+ TmpDivReg = MRI.createGenericVirtualRegister(Ty);
+ break;
+ }
+ case AMDGPU::G_SREM: {
+ DstRemReg = MI.getOperand(0).getReg();
+ TmpRemReg = MRI.createGenericVirtualRegister(Ty);
+ break;
+ }
+ case AMDGPU::G_SDIVREM: {
+ DstDivReg = MI.getOperand(0).getReg();
+ DstRemReg = MI.getOperand(1).getReg();
+ TmpDivReg = MRI.createGenericVirtualRegister(Ty);
+ TmpRemReg = MRI.createGenericVirtualRegister(Ty);
+ break;
+ }
+ }
+
if (Ty == S32)
- legalizeUDIV_UREM32Impl(B, UDivRem, LHS, RHS, IsDiv);
+ legalizeUnsignedDIV_REM32Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);
else
- legalizeUDIV_UREM64Impl(B, UDivRem, LHS, RHS, IsDiv);
+ legalizeUnsignedDIV_REM64Impl(B, TmpDivReg, TmpRemReg, LHS, RHS);
- Register Sign;
- if (IsDiv)
- Sign = B.buildXor(Ty, LHSign, RHSign).getReg(0);
- else
- Sign = LHSign.getReg(0); // Remainder sign is the same as LHS
+ if (DstDivReg) {
+ auto Sign = B.buildXor(Ty, LHSign, RHSign).getReg(0);
+ auto SignXor = B.buildXor(Ty, TmpDivReg, Sign).getReg(0);
+ B.buildSub(DstDivReg, SignXor, Sign);
+ }
- UDivRem = B.buildXor(Ty, UDivRem, Sign).getReg(0);
- B.buildSub(DstReg, UDivRem, Sign);
+ if (DstRemReg) {
+ auto Sign = LHSign.getReg(0); // Remainder sign is the same as LHS
+ auto SignXor = B.buildXor(Ty, TmpRemReg, Sign).getReg(0);
+ B.buildSub(DstRemReg, SignXor, Sign);
+ }
MI.eraseFromParent();
return true;
@@ -3511,18 +3650,21 @@ bool AMDGPULegalizerInfo::legalizeIsAddrSpace(MachineInstr &MI,
// (the offset that is excluded from bounds checking and swizzling, to go in
// the instruction's soffset field). This function takes the first kind of
// offset and figures out how to split it between voffset and immoffset.
-std::tuple<Register, unsigned, unsigned>
+std::pair<Register, unsigned>
AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
Register OrigOffset) const {
const unsigned MaxImm = 4095;
Register BaseReg;
- unsigned TotalConstOffset;
+ unsigned ImmOffset;
const LLT S32 = LLT::scalar(32);
+ MachineRegisterInfo &MRI = *B.getMRI();
- std::tie(BaseReg, TotalConstOffset) =
- AMDGPU::getBaseWithConstantOffset(*B.getMRI(), OrigOffset);
+ std::tie(BaseReg, ImmOffset) =
+ AMDGPU::getBaseWithConstantOffset(MRI, OrigOffset);
- unsigned ImmOffset = TotalConstOffset;
+ // If BaseReg is a pointer, convert it to int.
+ if (MRI.getType(BaseReg).isPointer())
+ BaseReg = B.buildPtrToInt(MRI.getType(OrigOffset), BaseReg).getReg(0);
// If the immediate value is too big for the immoffset field, put the value
// and -4096 into the immoffset field so that the value that is copied/added
@@ -3550,7 +3692,32 @@ AMDGPULegalizerInfo::splitBufferOffsets(MachineIRBuilder &B,
if (!BaseReg)
BaseReg = B.buildConstant(S32, 0).getReg(0);
- return std::make_tuple(BaseReg, ImmOffset, TotalConstOffset);
+ return std::make_pair(BaseReg, ImmOffset);
+}
+
+/// Update \p MMO based on the offset inputs to a raw/struct buffer intrinsic.
+void AMDGPULegalizerInfo::updateBufferMMO(MachineMemOperand *MMO,
+ Register VOffset, Register SOffset,
+ unsigned ImmOffset, Register VIndex,
+ MachineRegisterInfo &MRI) const {
+ Optional<ValueAndVReg> MaybeVOffsetVal =
+ getConstantVRegValWithLookThrough(VOffset, MRI);
+ Optional<ValueAndVReg> MaybeSOffsetVal =
+ getConstantVRegValWithLookThrough(SOffset, MRI);
+ Optional<ValueAndVReg> MaybeVIndexVal =
+ getConstantVRegValWithLookThrough(VIndex, MRI);
+ // If the combined VOffset + SOffset + ImmOffset + strided VIndex is constant,
+ // update the MMO with that offset. The stride is unknown so we can only do
+ // this if VIndex is constant 0.
+ if (MaybeVOffsetVal && MaybeSOffsetVal && MaybeVIndexVal &&
+ MaybeVIndexVal->Value == 0) {
+ uint64_t TotalOffset = MaybeVOffsetVal->Value.getZExtValue() +
+ MaybeSOffsetVal->Value.getZExtValue() + ImmOffset;
+ MMO->setOffset(TotalOffset);
+ } else {
+ // We don't have a constant combined offset to use in the MMO. Give up.
+ MMO->setValue((Value *)nullptr);
+ }
}
/// Handle register layout difference for f16 images for some subtargets.
@@ -3572,7 +3739,8 @@ Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
int NumElts = StoreVT.getNumElements();
- return B.buildBuildVector(LLT::vector(NumElts, S32), WideRegs).getReg(0);
+ return B.buildBuildVector(LLT::fixed_vector(NumElts, S32), WideRegs)
+ .getReg(0);
}
if (ImageStore && ST.hasImageStoreD16Bug()) {
@@ -3581,7 +3749,8 @@ Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
Reg = B.buildBitcast(S32, Reg).getReg(0);
PackedRegs.push_back(Reg);
PackedRegs.resize(2, B.buildUndef(S32).getReg(0));
- return B.buildBuildVector(LLT::vector(2, S32), PackedRegs).getReg(0);
+ return B.buildBuildVector(LLT::fixed_vector(2, S32), PackedRegs)
+ .getReg(0);
}
if (StoreVT.getNumElements() == 3) {
@@ -3590,18 +3759,19 @@ Register AMDGPULegalizerInfo::handleD16VData(MachineIRBuilder &B,
for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
PackedRegs.push_back(Unmerge.getReg(I));
PackedRegs.resize(6, B.buildUndef(S16).getReg(0));
- Reg = B.buildBuildVector(LLT::vector(6, S16), PackedRegs).getReg(0);
- return B.buildBitcast(LLT::vector(3, S32), Reg).getReg(0);
+ Reg = B.buildBuildVector(LLT::fixed_vector(6, S16), PackedRegs).getReg(0);
+ return B.buildBitcast(LLT::fixed_vector(3, S32), Reg).getReg(0);
}
if (StoreVT.getNumElements() == 4) {
SmallVector<Register, 4> PackedRegs;
- Reg = B.buildBitcast(LLT::vector(2, S32), Reg).getReg(0);
+ Reg = B.buildBitcast(LLT::fixed_vector(2, S32), Reg).getReg(0);
auto Unmerge = B.buildUnmerge(S32, Reg);
for (int I = 0, E = Unmerge->getNumOperands() - 1; I != E; ++I)
PackedRegs.push_back(Unmerge.getReg(I));
PackedRegs.resize(4, B.buildUndef(S32).getReg(0));
- return B.buildBuildVector(LLT::vector(4, S32), PackedRegs).getReg(0);
+ return B.buildBuildVector(LLT::fixed_vector(4, S32), PackedRegs)
+ .getReg(0);
}
llvm_unreachable("invalid data type");
@@ -3651,7 +3821,6 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
const int MemSize = MMO->getSize();
unsigned ImmOffset;
- unsigned TotalOffset;
// The typed intrinsics add an immediate after the registers.
const unsigned NumVIndexOps = IsTyped ? 8 : 7;
@@ -3663,6 +3832,8 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
if (HasVIndex) {
VIndex = MI.getOperand(3).getReg();
OpOffset = 1;
+ } else {
+ VIndex = B.buildConstant(S32, 0).getReg(0);
}
Register VOffset = MI.getOperand(3 + OpOffset).getReg();
@@ -3676,9 +3847,8 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
- std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
- if (TotalOffset != 0)
- MMO = B.getMF().getMachineMemOperand(MMO, TotalOffset, MemSize);
+ std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+ updateBufferMMO(MMO, VOffset, SOffset, ImmOffset, VIndex, MRI);
unsigned Opc;
if (IsTyped) {
@@ -3701,9 +3871,6 @@ bool AMDGPULegalizerInfo::legalizeBufferStore(MachineInstr &MI,
}
}
- if (!VIndex)
- VIndex = B.buildConstant(S32, 0).getReg(0);
-
auto MIB = B.buildInstr(Opc)
.addUse(VData) // vdata
.addUse(RSrc) // rsrc
@@ -3730,7 +3897,7 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
bool IsTyped) const {
// FIXME: Verifier should enforce 1 MMO for these intrinsics.
MachineMemOperand *MMO = *MI.memoperands_begin();
- const int MemSize = MMO->getSize();
+ const LLT MemTy = MMO->getMemoryType();
const LLT S32 = LLT::scalar(32);
Register Dst = MI.getOperand(0).getReg();
@@ -3746,6 +3913,8 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
if (HasVIndex) {
VIndex = MI.getOperand(3).getReg();
OpOffset = 1;
+ } else {
+ VIndex = B.buildConstant(S32, 0).getReg(0);
}
Register VOffset = MI.getOperand(3 + OpOffset).getReg();
@@ -3759,16 +3928,14 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
unsigned AuxiliaryData = MI.getOperand(5 + OpOffset).getImm();
unsigned ImmOffset;
- unsigned TotalOffset;
LLT Ty = MRI.getType(Dst);
LLT EltTy = Ty.getScalarType();
const bool IsD16 = IsFormat && (EltTy.getSizeInBits() == 16);
const bool Unpacked = ST.hasUnpackedD16VMem();
- std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
- if (TotalOffset != 0)
- MMO = B.getMF().getMachineMemOperand(MMO, TotalOffset, MemSize);
+ std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+ updateBufferMMO(MMO, VOffset, SOffset, ImmOffset, VIndex, MRI);
unsigned Opc;
@@ -3779,11 +3946,11 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
Opc = IsD16 ? AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT_D16 :
AMDGPU::G_AMDGPU_BUFFER_LOAD_FORMAT;
} else {
- switch (MemSize) {
- case 1:
+ switch (MemTy.getSizeInBits()) {
+ case 8:
Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_UBYTE;
break;
- case 2:
+ case 16:
Opc = AMDGPU::G_AMDGPU_BUFFER_LOAD_USHORT;
break;
default:
@@ -3794,7 +3961,8 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
Register LoadDstReg;
- bool IsExtLoad = (!IsD16 && MemSize < 4) || (IsD16 && !Ty.isVector());
+ bool IsExtLoad =
+ (!IsD16 && MemTy.getSizeInBits() < 32) || (IsD16 && !Ty.isVector());
LLT UnpackedTy = Ty.changeElementSize(32);
if (IsExtLoad)
@@ -3804,9 +3972,6 @@ bool AMDGPULegalizerInfo::legalizeBufferLoad(MachineInstr &MI,
else
LoadDstReg = Dst;
- if (!VIndex)
- VIndex = B.buildConstant(S32, 0).getReg(0);
-
auto MIB = B.buildInstr(Opc)
.addDef(LoadDstReg) // vdata
.addUse(RSrc) // rsrc
@@ -3898,9 +4063,16 @@ static unsigned getBufferAtomicPseudo(Intrinsic::ID IntrID) {
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_CMPSWAP;
+ case Intrinsic::amdgcn_buffer_atomic_fadd:
case Intrinsic::amdgcn_raw_buffer_atomic_fadd:
case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FADD;
+ case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
+ return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMIN;
+ case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
+ return AMDGPU::G_AMDGPU_BUFFER_ATOMIC_FMAX;
default:
llvm_unreachable("unhandled atomic opcode");
}
@@ -3940,6 +4112,8 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
if (HasVIndex) {
VIndex = MI.getOperand(4 + OpOffset).getReg();
++OpOffset;
+ } else {
+ VIndex = B.buildConstant(LLT::scalar(32), 0).getReg(0);
}
Register VOffset = MI.getOperand(4 + OpOffset).getReg();
@@ -3949,13 +4123,8 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
MachineMemOperand *MMO = *MI.memoperands_begin();
unsigned ImmOffset;
- unsigned TotalOffset;
- std::tie(VOffset, ImmOffset, TotalOffset) = splitBufferOffsets(B, VOffset);
- if (TotalOffset != 0)
- MMO = B.getMF().getMachineMemOperand(MMO, TotalOffset, MMO->getSize());
-
- if (!VIndex)
- VIndex = B.buildConstant(LLT::scalar(32), 0).getReg(0);
+ std::tie(VOffset, ImmOffset) = splitBufferOffsets(B, VOffset);
+ updateBufferMMO(MMO, VOffset, SOffset, ImmOffset, VIndex, *B.getMRI());
auto MIB = B.buildInstr(getBufferAtomicPseudo(IID));
@@ -3980,14 +4149,16 @@ bool AMDGPULegalizerInfo::legalizeBufferAtomic(MachineInstr &MI,
return true;
}
-/// Turn a set of s16 typed registers in \p A16AddrRegs into a dword sized
+/// Turn a set of s16 typed registers in \p AddrRegs into a dword sized
/// vector with s16 typed elements.
-static void packImageA16AddressToDwords(
- MachineIRBuilder &B, MachineInstr &MI,
- SmallVectorImpl<Register> &PackedAddrs, unsigned ArgOffset,
- const AMDGPU::ImageDimIntrinsicInfo *Intr, unsigned EndIdx) {
+static void packImage16bitOpsToDwords(MachineIRBuilder &B, MachineInstr &MI,
+ SmallVectorImpl<Register> &PackedAddrs,
+ unsigned ArgOffset,
+ const AMDGPU::ImageDimIntrinsicInfo *Intr,
+ bool IsA16, bool IsG16) {
const LLT S16 = LLT::scalar(16);
- const LLT V2S16 = LLT::vector(2, 16);
+ const LLT V2S16 = LLT::fixed_vector(2, 16);
+ auto EndIdx = Intr->VAddrEnd;
for (unsigned I = Intr->VAddrStart; I < EndIdx; I++) {
MachineOperand &SrcOp = MI.getOperand(ArgOffset + I);
@@ -3996,7 +4167,10 @@ static void packImageA16AddressToDwords(
Register AddrReg = SrcOp.getReg();
- if (I < Intr->GradientStart) {
+ if ((I < Intr->GradientStart) ||
+ (I >= Intr->GradientStart && I < Intr->CoordStart && !IsG16) ||
+ (I >= Intr->CoordStart && !IsA16)) {
+ // Handle any gradient or coordinate operands that should not be packed
AddrReg = B.buildBitcast(V2S16, AddrReg).getReg(0);
PackedAddrs.push_back(AddrReg);
} else {
@@ -4041,16 +4215,16 @@ static void convertImageAddrToPacked(MachineIRBuilder &B, MachineInstr &MI,
int NumAddrRegs = AddrRegs.size();
if (NumAddrRegs != 1) {
- // Round up to 8 elements for v5-v7
- // FIXME: Missing intermediate sized register classes and instructions.
- if (NumAddrRegs > 4 && !isPowerOf2_32(NumAddrRegs)) {
+ // Above 8 elements round up to next power of 2 (i.e. 16).
+ if (NumAddrRegs > 8 && !isPowerOf2_32(NumAddrRegs)) {
const int RoundedNumRegs = NextPowerOf2(NumAddrRegs);
auto Undef = B.buildUndef(S32);
AddrRegs.append(RoundedNumRegs - NumAddrRegs, Undef.getReg(0));
NumAddrRegs = RoundedNumRegs;
}
- auto VAddr = B.buildBuildVector(LLT::vector(NumAddrRegs, 32), AddrRegs);
+ auto VAddr =
+ B.buildBuildVector(LLT::fixed_vector(NumAddrRegs, 32), AddrRegs);
MI.getOperand(DimIdx).setReg(VAddr.getReg(0));
}
@@ -4091,7 +4265,7 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
MachineRegisterInfo *MRI = B.getMRI();
const LLT S32 = LLT::scalar(32);
const LLT S16 = LLT::scalar(16);
- const LLT V2S16 = LLT::vector(2, 16);
+ const LLT V2S16 = LLT::fixed_vector(2, 16);
unsigned DMask = 0;
@@ -4146,7 +4320,7 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
if (BaseOpcode->AtomicX2) {
Register VData1 = MI.getOperand(3).getReg();
// The two values are packed in one register.
- LLT PackedTy = LLT::vector(2, Ty);
+ LLT PackedTy = LLT::fixed_vector(2, Ty);
auto Concat = B.buildBuildVector(PackedTy, {VData0, VData1});
MI.getOperand(2).setReg(Concat.getReg(0));
MI.getOperand(3).setReg(AMDGPU::NoRegister);
@@ -4194,35 +4368,30 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
}
// Rewrite the addressing register layout before doing anything else.
- if (IsA16 || IsG16) {
- if (IsA16) {
- // Target must support the feature and gradients need to be 16 bit too
- if (!ST.hasA16() || !IsG16)
- return false;
- } else if (!ST.hasG16())
- return false;
+ if (BaseOpcode->Gradients && !ST.hasG16() && (IsA16 != IsG16)) {
+ // 16 bit gradients are supported, but are tied to the A16 control
+ // so both gradients and addresses must be 16 bit
+ return false;
+ }
+ if (IsA16 && !ST.hasA16()) {
+ // A16 not supported
+ return false;
+ }
+
+ if (IsA16 || IsG16) {
if (Intr->NumVAddrs > 1) {
SmallVector<Register, 4> PackedRegs;
- // Don't compress addresses for G16
- const int PackEndIdx = IsA16 ? Intr->VAddrEnd : Intr->CoordStart;
- packImageA16AddressToDwords(B, MI, PackedRegs, ArgOffset, Intr,
- PackEndIdx);
- if (!IsA16) {
- // Add uncompressed address
- for (unsigned I = Intr->CoordStart; I < Intr->VAddrEnd; I++) {
- int AddrReg = MI.getOperand(ArgOffset + I).getReg();
- assert(B.getMRI()->getType(AddrReg) == LLT::scalar(32));
- PackedRegs.push_back(AddrReg);
- }
- }
+ packImage16bitOpsToDwords(B, MI, PackedRegs, ArgOffset, Intr, IsA16,
+ IsG16);
// See also below in the non-a16 branch
- const bool UseNSA = PackedRegs.size() >= 3 && ST.hasNSAEncoding();
+ const bool UseNSA = ST.hasNSAEncoding() && PackedRegs.size() >= 3 &&
+ PackedRegs.size() <= ST.getNSAMaxSize();
if (!UseNSA && PackedRegs.size() > 1) {
- LLT PackedAddrTy = LLT::vector(2 * PackedRegs.size(), 16);
+ LLT PackedAddrTy = LLT::fixed_vector(2 * PackedRegs.size(), 16);
auto Concat = B.buildConcatVectors(PackedAddrTy, PackedRegs);
PackedRegs[0] = Concat.getReg(0);
PackedRegs.resize(1);
@@ -4256,7 +4425,8 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
//
// SIShrinkInstructions will convert NSA encodings to non-NSA after register
// allocation when possible.
- const bool UseNSA = CorrectedNumVAddrs >= 3 && ST.hasNSAEncoding();
+ const bool UseNSA = ST.hasNSAEncoding() && CorrectedNumVAddrs >= 3 &&
+ CorrectedNumVAddrs <= ST.getNSAMaxSize();
if (!UseNSA && Intr->NumVAddrs > 1)
convertImageAddrToPacked(B, MI, ArgOffset + Intr->VAddrStart,
@@ -4299,7 +4469,8 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
return false;
const unsigned AdjustedNumElts = DMaskLanes == 0 ? 1 : DMaskLanes;
- const LLT AdjustedTy = Ty.changeNumElements(AdjustedNumElts);
+ const LLT AdjustedTy =
+ Ty.changeElementCount(ElementCount::getFixed(AdjustedNumElts));
// The raw dword aligned data component of the load. The only legal cases
// where this matters should be when using the packed D16 format, for
@@ -4313,15 +4484,17 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
LLT RegTy;
if (IsD16 && ST.hasUnpackedD16VMem()) {
- RoundedTy = LLT::scalarOrVector(AdjustedNumElts, 32);
- TFETy = LLT::vector(AdjustedNumElts + 1, 32);
+ RoundedTy =
+ LLT::scalarOrVector(ElementCount::getFixed(AdjustedNumElts), 32);
+ TFETy = LLT::fixed_vector(AdjustedNumElts + 1, 32);
RegTy = S32;
} else {
unsigned EltSize = EltTy.getSizeInBits();
unsigned RoundedElts = (AdjustedTy.getSizeInBits() + 31) / 32;
unsigned RoundedSize = 32 * RoundedElts;
- RoundedTy = LLT::scalarOrVector(RoundedSize / EltSize, EltSize);
- TFETy = LLT::vector(RoundedSize / 32 + 1, S32);
+ RoundedTy = LLT::scalarOrVector(
+ ElementCount::getFixed(RoundedSize / EltSize), EltSize);
+ TFETy = LLT::fixed_vector(RoundedSize / 32 + 1, S32);
RegTy = !IsTFE && EltSize == 16 ? V2S16 : S32;
}
@@ -4435,10 +4608,10 @@ bool AMDGPULegalizerInfo::legalizeImageIntrinsic(
const int RegsToCover = (Ty.getSizeInBits() + 31) / 32;
// Deal with the one annoying legal case.
- const LLT V3S16 = LLT::vector(3, 16);
+ const LLT V3S16 = LLT::fixed_vector(3, 16);
if (Ty == V3S16) {
padWithUndef(ResTy, RegsToCover - ResultRegs.size() + 1);
- auto Concat = B.buildConcatVectors(LLT::vector(6, 16), ResultRegs);
+ auto Concat = B.buildConcatVectors(LLT::fixed_vector(6, 16), ResultRegs);
B.buildUnmerge({DstReg, MRI->createGenericVirtualRegister(V3S16)}, Concat);
return true;
}
@@ -4460,7 +4633,7 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(
Observer.changingInstr(MI);
- if (shouldBitcastLoadStoreType(ST, Ty, Size)) {
+ if (shouldBitcastLoadStoreType(ST, Ty, LLT::scalar(Size))) {
Ty = getBitcastRegisterType(Ty);
Helper.bitcastDst(MI, Ty, 0);
Dst = MI.getOperand(0).getReg();
@@ -4502,27 +4675,55 @@ bool AMDGPULegalizerInfo::legalizeSBufferLoad(
bool AMDGPULegalizerInfo::legalizeTrapIntrinsic(MachineInstr &MI,
MachineRegisterInfo &MRI,
MachineIRBuilder &B) const {
- // Is non-HSA path or trap-handler disabled? then, insert s_endpgm instruction
- if (ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
- !ST.isTrapHandlerEnabled()) {
- B.buildInstr(AMDGPU::S_ENDPGM).addImm(0);
- } else {
- // Pass queue pointer to trap handler as input, and insert trap instruction
- // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
- MachineRegisterInfo &MRI = *B.getMRI();
-
- Register LiveIn =
- MRI.createGenericVirtualRegister(LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
- if (!loadInputValue(LiveIn, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
- return false;
+ if (!ST.isTrapHandlerEnabled() ||
+ ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA)
+ return legalizeTrapEndpgm(MI, MRI, B);
- Register SGPR01(AMDGPU::SGPR0_SGPR1);
- B.buildCopy(SGPR01, LiveIn);
- B.buildInstr(AMDGPU::S_TRAP)
- .addImm(GCNSubtarget::TrapIDLLVMTrap)
- .addReg(SGPR01, RegState::Implicit);
+ if (Optional<uint8_t> HsaAbiVer = AMDGPU::getHsaAbiVersion(&ST)) {
+ switch (*HsaAbiVer) {
+ case ELF::ELFABIVERSION_AMDGPU_HSA_V2:
+ case ELF::ELFABIVERSION_AMDGPU_HSA_V3:
+ return legalizeTrapHsaQueuePtr(MI, MRI, B);
+ case ELF::ELFABIVERSION_AMDGPU_HSA_V4:
+ return ST.supportsGetDoorbellID() ?
+ legalizeTrapHsa(MI, MRI, B) :
+ legalizeTrapHsaQueuePtr(MI, MRI, B);
+ }
}
+ llvm_unreachable("Unknown trap handler");
+}
+
+bool AMDGPULegalizerInfo::legalizeTrapEndpgm(
+ MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
+ B.buildInstr(AMDGPU::S_ENDPGM).addImm(0);
+ MI.eraseFromParent();
+ return true;
+}
+
+bool AMDGPULegalizerInfo::legalizeTrapHsaQueuePtr(
+ MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
+ // Pass queue pointer to trap handler as input, and insert trap instruction
+ // Reference: https://llvm.org/docs/AMDGPUUsage.html#trap-handler-abi
+ Register LiveIn =
+ MRI.createGenericVirtualRegister(LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64));
+ if (!loadInputValue(LiveIn, B, AMDGPUFunctionArgInfo::QUEUE_PTR))
+ return false;
+
+ Register SGPR01(AMDGPU::SGPR0_SGPR1);
+ B.buildCopy(SGPR01, LiveIn);
+ B.buildInstr(AMDGPU::S_TRAP)
+ .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap))
+ .addReg(SGPR01, RegState::Implicit);
+
+ MI.eraseFromParent();
+ return true;
+}
+
+bool AMDGPULegalizerInfo::legalizeTrapHsa(
+ MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
+ B.buildInstr(AMDGPU::S_TRAP)
+ .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSATrap));
MI.eraseFromParent();
return true;
}
@@ -4531,8 +4732,8 @@ bool AMDGPULegalizerInfo::legalizeDebugTrapIntrinsic(
MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) const {
// Is non-HSA path or trap-handler disabled? then, report a warning
// accordingly
- if (ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbiHsa ||
- !ST.isTrapHandlerEnabled()) {
+ if (!ST.isTrapHandlerEnabled() ||
+ ST.getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) {
DiagnosticInfoUnsupported NoTrap(B.getMF().getFunction(),
"debugtrap handler not supported",
MI.getDebugLoc(), DS_Warning);
@@ -4540,7 +4741,8 @@ bool AMDGPULegalizerInfo::legalizeDebugTrapIntrinsic(
Ctx.diagnose(NoTrap);
} else {
// Insert debug-trap instruction
- B.buildInstr(AMDGPU::S_TRAP).addImm(GCNSubtarget::TrapIDLLVMDebugTrap);
+ B.buildInstr(AMDGPU::S_TRAP)
+ .addImm(static_cast<unsigned>(GCNSubtarget::TrapID::LLVMAMDHSADebugTrap));
}
MI.eraseFromParent();
@@ -4561,6 +4763,14 @@ bool AMDGPULegalizerInfo::legalizeBVHIntrinsic(MachineInstr &MI,
Register RayInvDir = MI.getOperand(6).getReg();
Register TDescr = MI.getOperand(7).getReg();
+ if (!ST.hasGFX10_AEncoding()) {
+ DiagnosticInfoUnsupported BadIntrin(B.getMF().getFunction(),
+ "intrinsic not supported on subtarget",
+ MI.getDebugLoc());
+ B.getMF().getFunction().getContext().diagnose(BadIntrin);
+ return false;
+ }
+
bool IsA16 = MRI.getType(RayDir).getElementType().getSizeInBits() == 16;
bool Is64 = MRI.getType(NodePtr).getSizeInBits() == 64;
unsigned Opcode = IsA16 ? Is64 ? AMDGPU::IMAGE_BVH64_INTERSECT_RAY_a16_nsa
@@ -4810,6 +5020,11 @@ bool AMDGPULegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper,
case Intrinsic::amdgcn_struct_buffer_atomic_fadd:
case Intrinsic::amdgcn_raw_buffer_atomic_cmpswap:
case Intrinsic::amdgcn_struct_buffer_atomic_cmpswap:
+ case Intrinsic::amdgcn_buffer_atomic_fadd:
+ case Intrinsic::amdgcn_raw_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_struct_buffer_atomic_fmin:
+ case Intrinsic::amdgcn_raw_buffer_atomic_fmax:
+ case Intrinsic::amdgcn_struct_buffer_atomic_fmax:
return legalizeBufferAtomic(MI, B, IntrID);
case Intrinsic::amdgcn_atomic_inc:
return legalizeAtomicIncDec(MI, B, true);
generated by cgit v1.3 (git 2.53.0) at 2026-07-30 18:18:26 +0000