aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp')
-rw-r--r--llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp495
1 files changed, 338 insertions, 157 deletions
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index cca9fa72d0ca..3414fe758eff 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -19,7 +19,7 @@
#include "AMDGPUMemoryUtils.h"
#include "SIMachineFunctionInfo.h"
#include "llvm/CodeGen/Analysis.h"
-#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
+#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
@@ -423,6 +423,11 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM,
setOperationAction({ISD::FLOG10, ISD::FLOG, ISD::FEXP, ISD::FEXP10}, MVT::f16,
Custom);
+ setOperationAction(ISD::FCANONICALIZE, {MVT::f32, MVT::f64}, Legal);
+ if (Subtarget->has16BitInsts()) {
+ setOperationAction(ISD::FCANONICALIZE, MVT::f16, Legal);
+ }
+
// FIXME: These IS_FPCLASS vector fp types are marked custom so it reaches
// scalarization code. Can be removed when IS_FPCLASS expand isn't called by
// default unless marked custom/legal.
@@ -661,6 +666,8 @@ static bool fnegFoldsIntoOpcode(unsigned Opc) {
case ISD::FMAXNUM_IEEE:
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM:
case ISD::SELECT:
case ISD::FSIN:
case ISD::FTRUNC:
@@ -796,8 +803,8 @@ EVT AMDGPUTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
return EVT::getIntegerVT(Context, 32 * ((Size + 31) / 32));
}
-MVT AMDGPUTargetLowering::getVectorIdxTy(const DataLayout &) const {
- return MVT::i32;
+unsigned AMDGPUTargetLowering::getVectorIdxWidth(const DataLayout &) const {
+ return 32;
}
bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const {
@@ -819,11 +826,11 @@ bool AMDGPUTargetLowering::ShouldShrinkFPConstant(EVT VT) const {
return (ScalarVT != MVT::f32 && ScalarVT != MVT::f64);
}
-bool AMDGPUTargetLowering::shouldReduceLoadWidth(SDNode *N,
- ISD::LoadExtType ExtTy,
- EVT NewVT) const {
+bool AMDGPUTargetLowering::shouldReduceLoadWidth(
+ SDNode *N, ISD::LoadExtType ExtTy, EVT NewVT,
+ std::optional<unsigned> ByteOffset) const {
// TODO: This may be worth removing. Check regression tests for diffs.
- if (!TargetLoweringBase::shouldReduceLoadWidth(N, ExtTy, NewVT))
+ if (!TargetLoweringBase::shouldReduceLoadWidth(N, ExtTy, NewVT, ByteOffset))
return false;
unsigned NewSize = NewVT.getStoreSizeInBits();
@@ -846,7 +853,7 @@ bool AMDGPUTargetLowering::shouldReduceLoadWidth(SDNode *N,
AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
(isa<LoadSDNode>(N) && AS == AMDGPUAS::GLOBAL_ADDRESS &&
MN->isInvariant())) &&
- AMDGPUInstrInfo::isUniformMMO(MN->getMemOperand()))
+ AMDGPU::isUniformMMO(MN->getMemOperand()))
return false;
// Don't produce extloads from sub 32-bit types. SI doesn't have scalar
@@ -1040,8 +1047,12 @@ bool AMDGPUTargetLowering::isNarrowingProfitable(SDNode *N, EVT SrcVT,
case ISD::MUL:
case ISD::SETCC:
case ISD::SELECT:
+ case ISD::SMIN:
+ case ISD::SMAX:
+ case ISD::UMIN:
+ case ISD::UMAX:
if (Subtarget->has16BitInsts() &&
- (DestVT.isVector() ? !Subtarget->hasVOP3PInsts() : true)) {
+ (!DestVT.isVector() || !Subtarget->hasVOP3PInsts())) {
// Don't narrow back down to i16 if promoted to i32 already.
if (!N->isDivergent() && DestVT.isInteger() &&
DestVT.getScalarSizeInBits() > 1 &&
@@ -1136,7 +1147,7 @@ CCAssignFn *AMDGPUCallLowering::CCAssignFnForCall(CallingConv::ID CC,
case CallingConv::AMDGPU_KERNEL:
case CallingConv::SPIR_KERNEL:
default:
- report_fatal_error("Unsupported calling convention for call");
+ reportFatalUsageError("unsupported calling convention for call");
}
}
@@ -1163,7 +1174,7 @@ CCAssignFn *AMDGPUCallLowering::CCAssignFnForReturn(CallingConv::ID CC,
case CallingConv::Cold:
return RetCC_AMDGPU_Func;
default:
- report_fatal_error("Unsupported calling convention.");
+ reportFatalUsageError("unsupported calling convention");
}
}
@@ -1383,13 +1394,12 @@ SDValue AMDGPUTargetLowering::lowerUnhandledCall(CallLoweringInfo &CLI,
else if (const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
FuncName = G->getGlobal()->getName();
- DiagnosticInfoUnsupported NoCalls(
- Fn, Reason + FuncName, CLI.DL.getDebugLoc());
- DAG.getContext()->diagnose(NoCalls);
+ DAG.getContext()->diagnose(
+ DiagnosticInfoUnsupported(Fn, Reason + FuncName, CLI.DL.getDebugLoc()));
if (!CLI.IsTailCall) {
for (ISD::InputArg &Arg : CLI.Ins)
- InVals.push_back(DAG.getUNDEF(Arg.VT));
+ InVals.push_back(DAG.getPOISON(Arg.VT));
}
return DAG.getEntryNode();
@@ -1404,9 +1414,8 @@ SDValue AMDGPUTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
SelectionDAG &DAG) const {
const Function &Fn = DAG.getMachineFunction().getFunction();
- DiagnosticInfoUnsupported NoDynamicAlloca(Fn, "unsupported dynamic alloca",
- SDLoc(Op).getDebugLoc());
- DAG.getContext()->diagnose(NoDynamicAlloca);
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ Fn, "unsupported dynamic alloca", SDLoc(Op).getDebugLoc()));
auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)};
return DAG.getMergeValues(Ops, SDLoc());
}
@@ -1521,10 +1530,9 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
!AMDGPU::isNamedBarrier(*cast<GlobalVariable>(GV))) {
SDLoc DL(Op);
const Function &Fn = DAG.getMachineFunction().getFunction();
- DiagnosticInfoUnsupported BadLDSDecl(
- Fn, "local memory global used by non-kernel function",
- DL.getDebugLoc(), DS_Warning);
- DAG.getContext()->diagnose(BadLDSDecl);
+ DAG.getContext()->diagnose(DiagnosticInfoUnsupported(
+ Fn, "local memory global used by non-kernel function",
+ DL.getDebugLoc(), DS_Warning));
// We currently don't have a way to correctly allocate LDS objects that
// aren't directly associated with a kernel. We do force inlining of
@@ -1535,7 +1543,7 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI,
SDValue OutputChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
Trap, DAG.getRoot());
DAG.setRoot(OutputChain);
- return DAG.getUNDEF(Op.getValueType());
+ return DAG.getPOISON(Op.getValueType());
}
// XXX: What does the value of G->getOffset() mean?
@@ -1857,7 +1865,7 @@ SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op,
// This is the case that the vector is power of two so was evenly split.
Join = DAG.getNode(ISD::CONCAT_VECTORS, SL, VT, LoLoad, HiLoad);
} else {
- Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getUNDEF(VT), LoLoad,
+ Join = DAG.getNode(ISD::INSERT_SUBVECTOR, SL, VT, DAG.getPOISON(VT), LoLoad,
DAG.getVectorIdxConstant(0, SL));
Join = DAG.getNode(
HiVT.isVector() ? ISD::INSERT_SUBVECTOR : ISD::INSERT_VECTOR_ELT, SL,
@@ -2971,7 +2979,8 @@ SDValue AMDGPUTargetLowering::lowerFEXP10Unsafe(SDValue X, const SDLoc &SL,
SelectionDAG &DAG,
SDNodeFlags Flags) const {
const EVT VT = X.getValueType();
- const unsigned Exp2Op = VT == MVT::f32 ? AMDGPUISD::EXP : ISD::FEXP2;
+ const unsigned Exp2Op = VT == MVT::f32 ? static_cast<unsigned>(AMDGPUISD::EXP)
+ : static_cast<unsigned>(ISD::FEXP2);
if (VT != MVT::f32 || !needsDenormHandlingF32(DAG, X, Flags)) {
// exp2(x * 0x1.a92000p+1f) * exp2(x * 0x1.4f0978p-11f);
@@ -3578,15 +3587,22 @@ SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) con
return SDValue();
}
- assert(N0.getSimpleValueType() == MVT::f64);
+ return LowerF64ToF16Safe(N0, DL, DAG);
+}
+
+// return node in i32
+SDValue AMDGPUTargetLowering::LowerF64ToF16Safe(SDValue Src, const SDLoc &DL,
+ SelectionDAG &DAG) const {
+ assert(Src.getSimpleValueType() == MVT::f64);
// f64 -> f16 conversion using round-to-nearest-even rounding mode.
+ // TODO: We can generate better code for True16.
const unsigned ExpMask = 0x7ff;
const unsigned ExpBiasf64 = 1023;
const unsigned ExpBiasf16 = 15;
SDValue Zero = DAG.getConstant(0, DL, MVT::i32);
SDValue One = DAG.getConstant(1, DL, MVT::i32);
- SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0);
+ SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Src);
SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U,
DAG.getConstant(32, DL, MVT::i64));
UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32);
@@ -3660,8 +3676,7 @@ SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) con
Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign,
DAG.getConstant(0x8000, DL, MVT::i32));
- V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V);
- return DAG.getZExtOrTrunc(V, DL, Op.getValueType());
+ return DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V);
}
SDValue AMDGPUTargetLowering::LowerFP_TO_INT(const SDValue Op,
@@ -3843,7 +3858,7 @@ bool AMDGPUTargetLowering::shouldCombineMemoryType(EVT VT) const {
return true;
}
-// Replace load of an illegal type with a store of a bitcast to a friendlier
+// Replace load of an illegal type with a bitcast from a load of a friendlier
// type.
SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
@@ -3991,7 +4006,8 @@ SDValue AMDGPUTargetLowering::performIntrinsicWOChainCombine(
case Intrinsic::amdgcn_rsq:
case Intrinsic::amdgcn_rcp_legacy:
case Intrinsic::amdgcn_rsq_legacy:
- case Intrinsic::amdgcn_rsq_clamp: {
+ case Intrinsic::amdgcn_rsq_clamp:
+ case Intrinsic::amdgcn_tanh: {
// FIXME: This is probably wrong. If src is an sNaN, it won't be quieted
SDValue Src = N->getOperand(1);
return Src.isUndef() ? Src : SDValue();
@@ -4040,152 +4056,308 @@ SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl(
SDValue AMDGPUTargetLowering::performShlCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
EVT VT = N->getValueType(0);
-
- ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
- if (!RHS)
- return SDValue();
-
SDValue LHS = N->getOperand(0);
- unsigned RHSVal = RHS->getZExtValue();
- if (!RHSVal)
- return LHS;
-
+ SDValue RHS = N->getOperand(1);
+ ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS);
SDLoc SL(N);
SelectionDAG &DAG = DCI.DAG;
- switch (LHS->getOpcode()) {
- default:
- break;
- case ISD::ZERO_EXTEND:
- case ISD::SIGN_EXTEND:
- case ISD::ANY_EXTEND: {
- SDValue X = LHS->getOperand(0);
+ unsigned RHSVal;
+ if (CRHS) {
+ RHSVal = CRHS->getZExtValue();
+ if (!RHSVal)
+ return LHS;
- if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
- isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) {
- // Prefer build_vector as the canonical form if packed types are legal.
- // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
- SDValue Vec = DAG.getBuildVector(MVT::v2i16, SL,
- { DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0) });
- return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
- }
-
- // shl (ext x) => zext (shl x), if shift does not overflow int
- if (VT != MVT::i64)
- break;
- KnownBits Known = DAG.computeKnownBits(X);
- unsigned LZ = Known.countMinLeadingZeros();
- if (LZ < RHSVal)
+ switch (LHS->getOpcode()) {
+ default:
break;
- EVT XVT = X.getValueType();
- SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(RHS, 0));
- return DAG.getZExtOrTrunc(Shl, SL, VT);
- }
+ case ISD::ZERO_EXTEND:
+ case ISD::SIGN_EXTEND:
+ case ISD::ANY_EXTEND: {
+ SDValue X = LHS->getOperand(0);
+
+ if (VT == MVT::i32 && RHSVal == 16 && X.getValueType() == MVT::i16 &&
+ isOperationLegal(ISD::BUILD_VECTOR, MVT::v2i16)) {
+ // Prefer build_vector as the canonical form if packed types are legal.
+ // (shl ([asz]ext i16:x), 16 -> build_vector 0, x
+ SDValue Vec = DAG.getBuildVector(
+ MVT::v2i16, SL,
+ {DAG.getConstant(0, SL, MVT::i16), LHS->getOperand(0)});
+ return DAG.getNode(ISD::BITCAST, SL, MVT::i32, Vec);
+ }
+
+ // shl (ext x) => zext (shl x), if shift does not overflow int
+ if (VT != MVT::i64)
+ break;
+ KnownBits Known = DAG.computeKnownBits(X);
+ unsigned LZ = Known.countMinLeadingZeros();
+ if (LZ < RHSVal)
+ break;
+ EVT XVT = X.getValueType();
+ SDValue Shl = DAG.getNode(ISD::SHL, SL, XVT, X, SDValue(CRHS, 0));
+ return DAG.getZExtOrTrunc(Shl, SL, VT);
+ }
+ }
}
- if (VT != MVT::i64)
+ if (VT.getScalarType() != MVT::i64)
return SDValue();
- // i64 (shl x, C) -> (build_pair 0, (shl x, C -32))
+ // i64 (shl x, C) -> (build_pair 0, (shl x, C - 32))
// On some subtargets, 64-bit shift is a quarter rate instruction. In the
// common case, splitting this into a move and a 32-bit shift is faster and
// the same code size.
- if (RHSVal < 32)
+ KnownBits Known = DAG.computeKnownBits(RHS);
+
+ EVT ElementType = VT.getScalarType();
+ EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(*DAG.getContext());
+ EVT TargetType = VT.isVector() ? VT.changeVectorElementType(TargetScalarType)
+ : TargetScalarType;
+
+ if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
return SDValue();
+ SDValue ShiftAmt;
- SDValue ShiftAmt = DAG.getConstant(RHSVal - 32, SL, MVT::i32);
+ if (CRHS) {
+ ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL,
+ TargetType);
+ } else {
+ SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
+ const SDValue ShiftMask =
+ DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
+ // This AND instruction will clamp out of bounds shift values.
+ // It will also be removed during later instruction selection.
+ ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
+ }
- SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, MVT::i32, LHS);
- SDValue NewShift = DAG.getNode(ISD::SHL, SL, MVT::i32, Lo, ShiftAmt);
+ SDValue Lo = DAG.getNode(ISD::TRUNCATE, SL, TargetType, LHS);
+ SDValue NewShift =
+ DAG.getNode(ISD::SHL, SL, TargetType, Lo, ShiftAmt, N->getFlags());
- const SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
+ const SDValue Zero = DAG.getConstant(0, SL, TargetScalarType);
+ SDValue Vec;
- SDValue Vec = DAG.getBuildVector(MVT::v2i32, SL, {Zero, NewShift});
- return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Vec);
+ if (VT.isVector()) {
+ EVT ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.getContext());
+ unsigned NElts = TargetType.getVectorNumElements();
+ SmallVector<SDValue, 8> HiOps;
+ SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
+
+ DAG.ExtractVectorElements(NewShift, HiOps, 0, NElts);
+ for (unsigned I = 0; I != NElts; ++I)
+ HiAndLoOps[2 * I + 1] = HiOps[I];
+ Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, ConcatType, HiAndLoOps);
+ } else {
+ EVT ConcatType = EVT::getVectorVT(*DAG.getContext(), TargetType, 2);
+ Vec = DAG.getBuildVector(ConcatType, SL, {Zero, NewShift});
+ }
+ return DAG.getNode(ISD::BITCAST, SL, VT, Vec);
}
SDValue AMDGPUTargetLowering::performSraCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
- if (N->getValueType(0) != MVT::i64)
+ SDValue RHS = N->getOperand(1);
+ ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS);
+ EVT VT = N->getValueType(0);
+ SDValue LHS = N->getOperand(0);
+ SelectionDAG &DAG = DCI.DAG;
+ SDLoc SL(N);
+
+ if (VT.getScalarType() != MVT::i64)
return SDValue();
- const ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
- if (!RHS)
+ // For C >= 32
+ // i64 (sra x, C) -> (build_pair (sra hi_32(x), C - 32), sra hi_32(x), 31))
+
+ // On some subtargets, 64-bit shift is a quarter rate instruction. In the
+ // common case, splitting this into a move and a 32-bit shift is faster and
+ // the same code size.
+ KnownBits Known = DAG.computeKnownBits(RHS);
+
+ EVT ElementType = VT.getScalarType();
+ EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(*DAG.getContext());
+ EVT TargetType = VT.isVector() ? VT.changeVectorElementType(TargetScalarType)
+ : TargetScalarType;
+
+ if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
return SDValue();
- SelectionDAG &DAG = DCI.DAG;
- SDLoc SL(N);
- unsigned RHSVal = RHS->getZExtValue();
+ SDValue ShiftFullAmt =
+ DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
+ SDValue ShiftAmt;
+ if (CRHS) {
+ unsigned RHSVal = CRHS->getZExtValue();
+ ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL,
+ TargetType);
+ } else if (Known.getMinValue().getZExtValue() ==
+ (ElementType.getSizeInBits() - 1)) {
+ ShiftAmt = ShiftFullAmt;
+ } else {
+ SDValue truncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
+ const SDValue ShiftMask =
+ DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
+ // This AND instruction will clamp out of bounds shift values.
+ // It will also be removed during later instruction selection.
+ ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, truncShiftAmt, ShiftMask);
+ }
- // (sra i64:x, 32) -> build_pair x, (sra hi_32(x), 31)
- if (RHSVal == 32) {
- SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
- SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
- DAG.getConstant(31, SL, MVT::i32));
+ EVT ConcatType;
+ SDValue Hi;
+ SDLoc LHSSL(LHS);
+ // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
+ if (VT.isVector()) {
+ unsigned NElts = TargetType.getVectorNumElements();
+ ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.getContext());
+ SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
+ SmallVector<SDValue, 8> HiOps(NElts);
+ SmallVector<SDValue, 16> HiAndLoOps;
- SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {Hi, NewShift});
- return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
+ DAG.ExtractVectorElements(SplitLHS, HiAndLoOps, 0, NElts * 2);
+ for (unsigned I = 0; I != NElts; ++I) {
+ HiOps[I] = HiAndLoOps[2 * I + 1];
+ }
+ Hi = DAG.getNode(ISD::BUILD_VECTOR, LHSSL, TargetType, HiOps);
+ } else {
+ const SDValue One = DAG.getConstant(1, LHSSL, TargetScalarType);
+ ConcatType = EVT::getVectorVT(*DAG.getContext(), TargetType, 2);
+ SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
+ Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One);
}
- // (sra i64:x, 63) -> build_pair (sra hi_32(x), 31), (sra hi_32(x), 31)
- if (RHSVal == 63) {
- SDValue Hi = getHiHalf64(N->getOperand(0), DAG);
- SDValue NewShift = DAG.getNode(ISD::SRA, SL, MVT::i32, Hi,
- DAG.getConstant(31, SL, MVT::i32));
- SDValue BuildVec = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, NewShift});
- return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildVec);
+ KnownBits KnownLHS = DAG.computeKnownBits(LHS);
+ SDValue HiShift;
+ if (KnownLHS.isNegative()) {
+ HiShift = DAG.getAllOnesConstant(SL, TargetType);
+ } else {
+ Hi = DAG.getFreeze(Hi);
+ HiShift = DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftFullAmt);
}
+ SDValue NewShift =
+ DAG.getNode(ISD::SRA, SL, TargetType, Hi, ShiftAmt, N->getFlags());
- return SDValue();
+ SDValue Vec;
+ if (VT.isVector()) {
+ unsigned NElts = TargetType.getVectorNumElements();
+ SmallVector<SDValue, 8> HiOps;
+ SmallVector<SDValue, 8> LoOps;
+ SmallVector<SDValue, 16> HiAndLoOps(NElts * 2);
+
+ DAG.ExtractVectorElements(HiShift, HiOps, 0, NElts);
+ DAG.ExtractVectorElements(NewShift, LoOps, 0, NElts);
+ for (unsigned I = 0; I != NElts; ++I) {
+ HiAndLoOps[2 * I + 1] = HiOps[I];
+ HiAndLoOps[2 * I] = LoOps[I];
+ }
+ Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, ConcatType, HiAndLoOps);
+ } else {
+ Vec = DAG.getBuildVector(ConcatType, SL, {NewShift, HiShift});
+ }
+ return DAG.getNode(ISD::BITCAST, SL, VT, Vec);
}
SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N,
DAGCombinerInfo &DCI) const {
- auto *RHS = dyn_cast<ConstantSDNode>(N->getOperand(1));
- if (!RHS)
- return SDValue();
-
+ SDValue RHS = N->getOperand(1);
+ ConstantSDNode *CRHS = dyn_cast<ConstantSDNode>(RHS);
EVT VT = N->getValueType(0);
SDValue LHS = N->getOperand(0);
- unsigned ShiftAmt = RHS->getZExtValue();
SelectionDAG &DAG = DCI.DAG;
SDLoc SL(N);
+ unsigned RHSVal;
- // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
- // this improves the ability to match BFE patterns in isel.
- if (LHS.getOpcode() == ISD::AND) {
- if (auto *Mask = dyn_cast<ConstantSDNode>(LHS.getOperand(1))) {
- unsigned MaskIdx, MaskLen;
- if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
- MaskIdx == ShiftAmt) {
- return DAG.getNode(
- ISD::AND, SL, VT,
- DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0), N->getOperand(1)),
- DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1), N->getOperand(1)));
+ if (CRHS) {
+ RHSVal = CRHS->getZExtValue();
+
+ // fold (srl (and x, c1 << c2), c2) -> (and (srl(x, c2), c1)
+ // this improves the ability to match BFE patterns in isel.
+ if (LHS.getOpcode() == ISD::AND) {
+ if (auto *Mask = dyn_cast<ConstantSDNode>(LHS.getOperand(1))) {
+ unsigned MaskIdx, MaskLen;
+ if (Mask->getAPIntValue().isShiftedMask(MaskIdx, MaskLen) &&
+ MaskIdx == RHSVal) {
+ return DAG.getNode(ISD::AND, SL, VT,
+ DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(0),
+ N->getOperand(1)),
+ DAG.getNode(ISD::SRL, SL, VT, LHS.getOperand(1),
+ N->getOperand(1)));
+ }
}
}
}
- if (VT != MVT::i64)
+ if (VT.getScalarType() != MVT::i64)
return SDValue();
- if (ShiftAmt < 32)
+ // for C >= 32
+ // i64 (srl x, C) -> (build_pair (srl hi_32(x), C - 32), 0)
+
+ // On some subtargets, 64-bit shift is a quarter rate instruction. In the
+ // common case, splitting this into a move and a 32-bit shift is faster and
+ // the same code size.
+ KnownBits Known = DAG.computeKnownBits(RHS);
+
+ EVT ElementType = VT.getScalarType();
+ EVT TargetScalarType = ElementType.getHalfSizedIntegerVT(*DAG.getContext());
+ EVT TargetType = VT.isVector() ? VT.changeVectorElementType(TargetScalarType)
+ : TargetScalarType;
+
+ if (Known.getMinValue().getZExtValue() < TargetScalarType.getSizeInBits())
return SDValue();
- // srl i64:x, C for C >= 32
- // =>
- // build_pair (srl hi_32(x), C - 32), 0
- SDValue Zero = DAG.getConstant(0, SL, MVT::i32);
+ SDValue ShiftAmt;
+ if (CRHS) {
+ ShiftAmt = DAG.getConstant(RHSVal - TargetScalarType.getSizeInBits(), SL,
+ TargetType);
+ } else {
+ SDValue TruncShiftAmt = DAG.getNode(ISD::TRUNCATE, SL, TargetType, RHS);
+ const SDValue ShiftMask =
+ DAG.getConstant(TargetScalarType.getSizeInBits() - 1, SL, TargetType);
+ // This AND instruction will clamp out of bounds shift values.
+ // It will also be removed during later instruction selection.
+ ShiftAmt = DAG.getNode(ISD::AND, SL, TargetType, TruncShiftAmt, ShiftMask);
+ }
- SDValue Hi = getHiHalf64(LHS, DAG);
+ const SDValue Zero = DAG.getConstant(0, SL, TargetScalarType);
+ EVT ConcatType;
+ SDValue Hi;
+ SDLoc LHSSL(LHS);
+ // Bitcast LHS into ConcatType so hi-half of source can be extracted into Hi
+ if (VT.isVector()) {
+ unsigned NElts = TargetType.getVectorNumElements();
+ ConcatType = TargetType.getDoubleNumVectorElementsVT(*DAG.getContext());
+ SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
+ SmallVector<SDValue, 8> HiOps(NElts);
+ SmallVector<SDValue, 16> HiAndLoOps;
+
+ DAG.ExtractVectorElements(SplitLHS, HiAndLoOps, /*Start=*/0, NElts * 2);
+ for (unsigned I = 0; I != NElts; ++I)
+ HiOps[I] = HiAndLoOps[2 * I + 1];
+ Hi = DAG.getNode(ISD::BUILD_VECTOR, LHSSL, TargetType, HiOps);
+ } else {
+ const SDValue One = DAG.getConstant(1, LHSSL, TargetScalarType);
+ ConcatType = EVT::getVectorVT(*DAG.getContext(), TargetType, 2);
+ SDValue SplitLHS = DAG.getNode(ISD::BITCAST, LHSSL, ConcatType, LHS);
+ Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, LHSSL, TargetType, SplitLHS, One);
+ }
- SDValue NewConst = DAG.getConstant(ShiftAmt - 32, SL, MVT::i32);
- SDValue NewShift = DAG.getNode(ISD::SRL, SL, MVT::i32, Hi, NewConst);
+ SDValue NewShift =
+ DAG.getNode(ISD::SRL, SL, TargetType, Hi, ShiftAmt, N->getFlags());
- SDValue BuildPair = DAG.getBuildVector(MVT::v2i32, SL, {NewShift, Zero});
+ SDValue Vec;
+ if (VT.isVector()) {
+ unsigned NElts = TargetType.getVectorNumElements();
+ SmallVector<SDValue, 8> LoOps;
+ SmallVector<SDValue, 16> HiAndLoOps(NElts * 2, Zero);
- return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair);
+ DAG.ExtractVectorElements(NewShift, LoOps, 0, NElts);
+ for (unsigned I = 0; I != NElts; ++I)
+ HiAndLoOps[2 * I] = LoOps[I];
+ Vec = DAG.getNode(ISD::BUILD_VECTOR, SL, ConcatType, HiAndLoOps);
+ } else {
+ Vec = DAG.getBuildVector(ConcatType, SL, {NewShift, Zero});
+ }
+ return DAG.getNode(ISD::BITCAST, SL, VT, Vec);
}
SDValue AMDGPUTargetLowering::performTruncateCombine(
@@ -4217,18 +4389,21 @@ SDValue AMDGPUTargetLowering::performTruncateCombine(
// trunc (srl (bitcast (build_vector x, y))), 16 -> trunc (bitcast y)
if (Src.getOpcode() == ISD::SRL && !VT.isVector()) {
if (auto *K = isConstOrConstSplat(Src.getOperand(1))) {
- if (2 * K->getZExtValue() == Src.getValueType().getScalarSizeInBits()) {
- SDValue BV = stripBitcast(Src.getOperand(0));
- if (BV.getOpcode() == ISD::BUILD_VECTOR &&
- BV.getValueType().getVectorNumElements() == 2) {
- SDValue SrcElt = BV.getOperand(1);
- EVT SrcEltVT = SrcElt.getValueType();
- if (SrcEltVT.isFloatingPoint()) {
- SrcElt = DAG.getNode(ISD::BITCAST, SL,
- SrcEltVT.changeTypeToInteger(), SrcElt);
- }
+ SDValue BV = stripBitcast(Src.getOperand(0));
+ if (BV.getOpcode() == ISD::BUILD_VECTOR) {
+ EVT SrcEltVT = BV.getOperand(0).getValueType();
+ unsigned SrcEltSize = SrcEltVT.getSizeInBits();
+ unsigned BitIndex = K->getZExtValue();
+ unsigned PartIndex = BitIndex / SrcEltSize;
- return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
+ if (PartIndex * SrcEltSize == BitIndex &&
+ PartIndex < BV.getNumOperands()) {
+ if (SrcEltVT.getSizeInBits() == VT.getSizeInBits()) {
+ SDValue SrcElt =
+ DAG.getNode(ISD::BITCAST, SL, SrcEltVT.changeTypeToInteger(),
+ BV.getOperand(PartIndex));
+ return DAG.getNode(ISD::TRUNCATE, SL, VT, SrcElt);
+ }
}
}
}
@@ -4763,10 +4938,14 @@ static unsigned inverseMinMax(unsigned Opc) {
return ISD::FMINIMUM;
case ISD::FMINIMUM:
return ISD::FMAXIMUM;
+ case ISD::FMAXIMUMNUM:
+ return ISD::FMINIMUMNUM;
+ case ISD::FMINIMUMNUM:
+ return ISD::FMAXIMUMNUM;
case AMDGPUISD::FMAX_LEGACY:
return AMDGPUISD::FMIN_LEGACY;
case AMDGPUISD::FMIN_LEGACY:
- return AMDGPUISD::FMAX_LEGACY;
+ return AMDGPUISD::FMAX_LEGACY;
default:
llvm_unreachable("invalid min/max opcode");
}
@@ -4888,6 +5067,8 @@ SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
case ISD::FMINNUM_IEEE:
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
+ case ISD::FMINIMUMNUM:
+ case ISD::FMAXIMUMNUM:
case AMDGPUISD::FMAX_LEGACY:
case AMDGPUISD::FMIN_LEGACY: {
// fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
@@ -5161,24 +5342,23 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N,
break;
}
- case ISD::SHL: {
- if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
- break;
-
- return performShlCombine(N, DCI);
- }
+ case ISD::SHL:
+ case ISD::SRA:
case ISD::SRL: {
- if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
+ // Range metadata can be invalidated when loads are converted to legal types
+ // (e.g. v2i64 -> v4i32).
+ // Try to convert vector shl/sra/srl before type legalization so that range
+ // metadata can be utilized.
+ if (!(N->getValueType(0).isVector() &&
+ DCI.getDAGCombineLevel() == BeforeLegalizeTypes) &&
+ DCI.getDAGCombineLevel() < AfterLegalizeDAG)
break;
-
+ if (N->getOpcode() == ISD::SHL)
+ return performShlCombine(N, DCI);
+ if (N->getOpcode() == ISD::SRA)
+ return performSraCombine(N, DCI);
return performSrlCombine(N, DCI);
}
- case ISD::SRA: {
- if (DCI.getDAGCombineLevel() < AfterLegalizeDAG)
- break;
-
- return performSraCombine(N, DCI);
- }
case ISD::TRUNCATE:
return performTruncateCombine(N, DCI);
case ISD::MUL:
@@ -5472,6 +5652,7 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
NODE_NAME_CASE(TC_RETURN)
NODE_NAME_CASE(TC_RETURN_GFX)
NODE_NAME_CASE(TC_RETURN_CHAIN)
+ NODE_NAME_CASE(TC_RETURN_CHAIN_DVGPR)
NODE_NAME_CASE(TRAP)
NODE_NAME_CASE(RET_GLUE)
NODE_NAME_CASE(WAVE_ADDRESS)
@@ -5893,9 +6074,8 @@ unsigned AMDGPUTargetLowering::ComputeNumSignBitsForTargetNode(
}
unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
- GISelKnownBits &Analysis, Register R,
- const APInt &DemandedElts, const MachineRegisterInfo &MRI,
- unsigned Depth) const {
+ GISelValueTracking &Analysis, Register R, const APInt &DemandedElts,
+ const MachineRegisterInfo &MRI, unsigned Depth) const {
const MachineInstr *MI = MRI.getVRegDef(R);
if (!MI)
return 1;
@@ -5929,10 +6109,9 @@ unsigned AMDGPUTargetLowering::computeNumSignBitsForTargetInstr(
}
}
-bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
- const SelectionDAG &DAG,
- bool SNaN,
- unsigned Depth) const {
+bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(
+ SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN,
+ unsigned Depth) const {
unsigned Opcode = Op.getOpcode();
switch (Opcode) {
case AMDGPUISD::FMIN_LEGACY:
@@ -6000,6 +6179,7 @@ bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
// TODO: Handle more intrinsics
switch (IntrinsicID) {
case Intrinsic::amdgcn_cubeid:
+ case Intrinsic::amdgcn_cvt_off_f32_i4:
return true;
case Intrinsic::amdgcn_frexp_mant: {
@@ -6017,7 +6197,8 @@ bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
case Intrinsic::amdgcn_rsq:
case Intrinsic::amdgcn_rcp_legacy:
case Intrinsic::amdgcn_rsq_legacy:
- case Intrinsic::amdgcn_rsq_clamp: {
+ case Intrinsic::amdgcn_rsq_clamp:
+ case Intrinsic::amdgcn_tanh: {
if (SNaN)
return true;
generated by cgit v1.3 (git 2.53.0) at 2026-07-30 02:48:15 +0000