diff options
Diffstat (limited to 'llvm/lib/Target/AMDGPU/SIISelLowering.cpp')
| -rw-r--r-- | llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 1288 |
1 files changed, 908 insertions, 380 deletions
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index b632c50dae0e..6cf2055c8e56 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -27,7 +27,7 @@ #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/ByteProvider.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" -#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h" +#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h" #include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/MIPatternMatch.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -61,6 +61,14 @@ static cl::opt<bool> UseDivergentRegisterIndexing( cl::desc("Use indirect register addressing for divergent indexes"), cl::init(false)); +// TODO: This option should be removed once we switch to always using PTRADD in +// the SelectionDAG. +static cl::opt<bool> UseSelectionDAGPTRADD( + "amdgpu-use-sdag-ptradd", cl::Hidden, + cl::desc("Generate ISD::PTRADD nodes for 64-bit pointer arithmetic in the " + "SelectionDAG ISel"), + cl::init(false)); + static bool denormalModeIsFlushAllF32(const MachineFunction &MF) { const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); return Info->getMode().FP32Denormals == DenormalMode::getPreserveSign(); @@ -523,8 +531,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction({ISD::SADDSAT, ISD::SSUBSAT}, {MVT::i16, MVT::i32}, Legal); - setOperationAction({ISD::FMINNUM, ISD::FMAXNUM}, {MVT::f32, MVT::f64}, - Custom); + setOperationAction( + {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}, + {MVT::f32, MVT::f64}, Custom); // These are really only legal for ieee_mode functions. We should be avoiding // them for functions that don't have ieee_mode enabled, so just say they are @@ -582,7 +591,7 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction({ISD::FP_TO_SINT, ISD::FP_TO_UINT}, MVT::i16, Custom); setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom); - setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i16, Custom); + setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i1, Custom); setOperationAction({ISD::SINT_TO_FP, ISD::UINT_TO_FP}, MVT::i32, Custom); @@ -751,12 +760,21 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::BUILD_VECTOR, {MVT::v2i16, MVT::v2f16, MVT::v2bf16}, Subtarget->hasVOP3PInsts() ? Legal : Custom); - setOperationAction(ISD::FNEG, MVT::v2f16, Legal); + setOperationAction(ISD::FNEG, {MVT::v2f16, MVT::v2bf16}, Legal); // This isn't really legal, but this avoids the legalizer unrolling it (and // allows matching fneg (fabs x) patterns) - setOperationAction(ISD::FABS, MVT::v2f16, Legal); + setOperationAction(ISD::FABS, {MVT::v2f16, MVT::v2bf16}, Legal); + + // Can do this in one BFI plus a constant materialize. + setOperationAction(ISD::FCOPYSIGN, + {MVT::v2f16, MVT::v2bf16, MVT::v4f16, MVT::v4bf16, + MVT::v8f16, MVT::v8bf16, MVT::v16f16, MVT::v16bf16, + MVT::v32f16, MVT::v32bf16}, + Custom); - setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, MVT::f16, Custom); + setOperationAction( + {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}, + MVT::f16, Custom); setOperationAction({ISD::FMAXNUM_IEEE, ISD::FMINNUM_IEEE}, MVT::f16, Legal); setOperationAction({ISD::FMINNUM_IEEE, ISD::FMAXNUM_IEEE, ISD::FMINIMUMNUM, @@ -810,8 +828,9 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction({ISD::FADD, ISD::FMUL, ISD::FMA, ISD::FCANONICALIZE}, VT, Custom); - setOperationAction({ISD::FMAXNUM, ISD::FMINNUM}, {MVT::v2f16, MVT::v4f16}, - Custom); + setOperationAction( + {ISD::FMAXNUM, ISD::FMINNUM, ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM}, + {MVT::v2f16, MVT::v4f16}, Custom); setOperationAction(ISD::FEXP, MVT::v2f16, Custom); setOperationAction(ISD::SELECT, {MVT::v4i16, MVT::v4f16, MVT::v4bf16}, @@ -855,22 +874,31 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, if (Subtarget->hasMad64_32()) setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, Custom); - if (Subtarget->hasPrefetch()) + if (Subtarget->hasPrefetch() && Subtarget->hasSafeSmemPrefetch()) setOperationAction(ISD::PREFETCH, MVT::Other, Custom); - if (Subtarget->hasIEEEMinMax()) { + if (Subtarget->hasIEEEMinimumMaximumInsts()) { setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Legal); - setOperationAction({ISD::FMINIMUM, ISD::FMAXIMUM}, - {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, - Custom); } else { // FIXME: For nnan fmaximum, emit the fmaximum3 instead of fmaxnum if (Subtarget->hasMinimum3Maximum3F32()) setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f32, Legal); - if (Subtarget->hasMinimum3Maximum3PKF16()) + if (Subtarget->hasMinimum3Maximum3PKF16()) { setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::v2f16, Legal); + + // If only the vector form is available, we need to widen to a vector. + if (!Subtarget->hasMinimum3Maximum3F16()) + setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM}, MVT::f16, Custom); + } + } + + if (Subtarget->hasVOP3PInsts()) { + // We want to break these into v2f16 pieces, not scalarize. + setOperationAction({ISD::FMINIMUM, ISD::FMAXIMUM}, + {MVT::v4f16, MVT::v8f16, MVT::v16f16, MVT::v32f16}, + Custom); } setOperationAction(ISD::INTRINSIC_WO_CHAIN, @@ -906,16 +934,18 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, setOperationAction(ISD::MUL, MVT::i1, Promote); if (Subtarget->hasBF16ConversionInsts()) { - setOperationAction(ISD::FP_ROUND, MVT::v2bf16, Legal); - setOperationAction(ISD::FP_ROUND, MVT::bf16, Legal); + setOperationAction(ISD::FP_ROUND, {MVT::bf16, MVT::v2bf16}, Custom); setOperationAction(ISD::BUILD_VECTOR, MVT::v2bf16, Legal); } if (Subtarget->hasCvtPkF16F32Inst()) { - setOperationAction(ISD::FP_ROUND, MVT::v2f16, Legal); + setOperationAction(ISD::FP_ROUND, + {MVT::v2f16, MVT::v4f16, MVT::v8f16, MVT::v16f16}, + Custom); } setTargetDAGCombine({ISD::ADD, + ISD::PTRADD, ISD::UADDO_CARRY, ISD::SUB, ISD::USUBO_CARRY, @@ -930,6 +960,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM, ISD::FMAXNUM_IEEE, ISD::FMINIMUM, ISD::FMAXIMUM, + ISD::FMINIMUMNUM, + ISD::FMAXIMUMNUM, ISD::FMA, ISD::SMIN, ISD::SMAX, @@ -1175,19 +1207,20 @@ static EVT memVTFromLoadIntrReturn(const SITargetLowering &TLI, return memVTFromLoadIntrData(TLI, DL, ST->getContainedType(0), MaxNumLanes); } -/// Map address space 7 to MVT::v5i32 because that's its in-memory -/// representation. This return value is vector-typed because there is no -/// MVT::i160 and it is not clear if one can be added. While this could -/// cause issues during codegen, these address space 7 pointers will be -/// rewritten away by then. Therefore, we can return MVT::v5i32 in order -/// to allow pre-codegen passes that query TargetTransformInfo, often for cost -/// modeling, to work. +/// Map address space 7 to MVT::amdgpuBufferFatPointer because that's its +/// in-memory representation. This return value is a custom type because there +/// is no MVT::i160 and adding one breaks integer promotion logic. While this +/// could cause issues during codegen, these address space 7 pointers will be +/// rewritten away by then. Therefore, we can return MVT::amdgpuBufferFatPointer +/// in order to allow pre-codegen passes that query TargetTransformInfo, often +/// for cost modeling, to work. (This also sets us up decently for doing the +/// buffer lowering in GlobalISel if SelectionDAG ever goes away.) MVT SITargetLowering::getPointerTy(const DataLayout &DL, unsigned AS) const { if (AMDGPUAS::BUFFER_FAT_POINTER == AS && DL.getPointerSizeInBits(AS) == 160) - return MVT::v5i32; + return MVT::amdgpuBufferFatPointer; if (AMDGPUAS::BUFFER_STRIDED_POINTER == AS && DL.getPointerSizeInBits(AS) == 192) - return MVT::v6i32; + return MVT::amdgpuBufferStridedPointer; return AMDGPUTargetLowering::getPointerTy(DL, AS); } /// Similarly, the in-memory representation of a p7 is {p8, i32}, aka @@ -1216,8 +1249,8 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, if (const AMDGPU::RsrcIntrinsic *RsrcIntr = AMDGPU::lookupRsrcIntrinsic(IntrID)) { - AttributeList Attr = - Intrinsic::getAttributes(CI.getContext(), (Intrinsic::ID)IntrID); + AttributeSet Attr = + Intrinsic::getFnAttributes(CI.getContext(), (Intrinsic::ID)IntrID); MemoryEffects ME = Attr.getMemoryEffects(); if (ME.doesNotAccessMemory()) return false; @@ -1371,6 +1404,19 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, return true; } + case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64: + case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: { + Info.opc = (IntrID == Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64) + ? ISD::INTRINSIC_W_CHAIN + : ISD::INTRINSIC_VOID; + Info.memVT = MVT::getVT(CI.getType()); + Info.ptrVal = CI.getOperand(0); + Info.memVT = MVT::i64; + Info.size = 8; + Info.align.reset(); + Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + return true; + } case Intrinsic::amdgcn_global_atomic_csub: { Info.opc = ISD::INTRINSIC_W_CHAIN; Info.memVT = MVT::getVT(CI.getType()); @@ -1380,9 +1426,15 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, MachineMemOperand::MOVolatile; return true; } - case Intrinsic::amdgcn_image_bvh_intersect_ray: { + case Intrinsic::amdgcn_image_bvh_dual_intersect_ray: + case Intrinsic::amdgcn_image_bvh_intersect_ray: + case Intrinsic::amdgcn_image_bvh8_intersect_ray: { Info.opc = ISD::INTRINSIC_W_CHAIN; - Info.memVT = MVT::getVT(CI.getType()); // XXX: what is correct VT? + Info.memVT = + MVT::getVT(IntrID == Intrinsic::amdgcn_image_bvh_intersect_ray + ? CI.getType() + : cast<StructType>(CI.getType()) + ->getElementType(0)); // XXX: what is correct VT? Info.fallbackAddressSpace = AMDGPUAS::BUFFER_RESOURCE; Info.align.reset(); @@ -1405,6 +1457,12 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, MachineMemOperand::MOVolatile; return true; } + case Intrinsic::amdgcn_ds_load_tr6_b96: + case Intrinsic::amdgcn_ds_load_tr4_b64: + case Intrinsic::amdgcn_ds_load_tr8_b64: + case Intrinsic::amdgcn_ds_load_tr16_b128: + case Intrinsic::amdgcn_global_load_tr6_b96: + case Intrinsic::amdgcn_global_load_tr4_b64: case Intrinsic::amdgcn_global_load_tr_b64: case Intrinsic::amdgcn_global_load_tr_b128: case Intrinsic::amdgcn_ds_read_tr4_b64: @@ -1443,6 +1501,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, Info.flags |= MachineMemOperand::MOStore; return true; } + case Intrinsic::amdgcn_load_to_lds: case Intrinsic::amdgcn_global_load_lds: { Info.opc = ISD::INTRINSIC_VOID; unsigned Width = cast<ConstantInt>(CI.getArgOperand(2))->getZExtValue(); @@ -1451,7 +1510,10 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore; return true; } - case Intrinsic::amdgcn_ds_bvh_stack_rtn: { + case Intrinsic::amdgcn_ds_bvh_stack_rtn: + case Intrinsic::amdgcn_ds_bvh_stack_push4_pop1_rtn: + case Intrinsic::amdgcn_ds_bvh_stack_push8_pop1_rtn: + case Intrinsic::amdgcn_ds_bvh_stack_push8_pop2_rtn: { Info.opc = ISD::INTRINSIC_W_CHAIN; const GCNTargetMachine &TM = @@ -1505,12 +1567,18 @@ bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II, case Intrinsic::amdgcn_atomic_cond_sub_u32: case Intrinsic::amdgcn_ds_append: case Intrinsic::amdgcn_ds_consume: + case Intrinsic::amdgcn_ds_load_tr8_b64: + case Intrinsic::amdgcn_ds_load_tr16_b128: + case Intrinsic::amdgcn_ds_load_tr4_b64: + case Intrinsic::amdgcn_ds_load_tr6_b96: case Intrinsic::amdgcn_ds_read_tr4_b64: case Intrinsic::amdgcn_ds_read_tr6_b96: case Intrinsic::amdgcn_ds_read_tr8_b64: case Intrinsic::amdgcn_ds_read_tr16_b64: case Intrinsic::amdgcn_ds_ordered_add: case Intrinsic::amdgcn_ds_ordered_swap: + case Intrinsic::amdgcn_ds_atomic_async_barrier_arrive_b64: + case Intrinsic::amdgcn_ds_atomic_barrier_arrive_rtn_b64: case Intrinsic::amdgcn_flat_atomic_fmax_num: case Intrinsic::amdgcn_flat_atomic_fmin_num: case Intrinsic::amdgcn_global_atomic_csub: @@ -1519,8 +1587,11 @@ bool SITargetLowering::getAddrModeArguments(const IntrinsicInst *II, case Intrinsic::amdgcn_global_atomic_ordered_add_b64: case Intrinsic::amdgcn_global_load_tr_b64: case Intrinsic::amdgcn_global_load_tr_b128: + case Intrinsic::amdgcn_global_load_tr4_b64: + case Intrinsic::amdgcn_global_load_tr6_b96: Ptr = II->getArgOperand(0); break; + case Intrinsic::amdgcn_load_to_lds: case Intrinsic::amdgcn_global_load_lds: Ptr = II->getArgOperand(1); break; @@ -1877,6 +1948,20 @@ bool SITargetLowering::allowsMisalignedMemoryAccessesImpl( Subtarget->hasUnalignedBufferAccessEnabled(); } + // Ensure robust out-of-bounds guarantees for buffer accesses are met if + // RelaxedBufferOOBMode is disabled. Normally hardware will ensure proper + // out-of-bounds behavior, but in the edge case where an access starts + // out-of-bounds and then enter in-bounds, the entire access would be treated + // as out-of-bounds. Prevent misaligned memory accesses by requiring the + // natural alignment of buffer accesses. + if (AddrSpace == AMDGPUAS::BUFFER_FAT_POINTER || + AddrSpace == AMDGPUAS::BUFFER_RESOURCE || + AddrSpace == AMDGPUAS::BUFFER_STRIDED_POINTER) { + if (!Subtarget->hasRelaxedBufferOOBMode() && + Alignment < Align(PowerOf2Ceil(divideCeil(Size, 8)))) + return false; + } + // Smaller than dword value must be aligned. if (Size < 32) return false; @@ -1898,7 +1983,8 @@ bool SITargetLowering::allowsMisalignedMemoryAccesses( } EVT SITargetLowering::getOptimalMemOpType( - const MemOp &Op, const AttributeList &FuncAttributes) const { + LLVMContext &Context, const MemOp &Op, + const AttributeList &FuncAttributes) const { // FIXME: Should account for address space here. // The default fallback uses the private pointer size as a guess for a type to @@ -2202,7 +2288,7 @@ SDValue SITargetLowering::getPreloadedValue( // It's undefined behavior if a function marked with the amdgpu-no-* // attributes uses the corresponding intrinsic. - return DAG.getUNDEF(VT); + return DAG.getPOISON(VT); } return loadInputValue(DAG, RC, VT, SDLoc(DAG.getEntryNode()), *Reg); @@ -2402,7 +2488,7 @@ void SITargetLowering::allocateSpecialInputVGPRsFixed( SIMachineFunctionInfo &Info) const { Register Reg = CCInfo.AllocateReg(AMDGPU::VGPR31); if (!Reg) - report_fatal_error("failed to allocated VGPR for implicit arguments"); + report_fatal_error("failed to allocate VGPR for implicit arguments"); const unsigned Mask = 0x3ff; Info.setWorkItemIDX(ArgDescriptor::createRegister(Reg, Mask)); @@ -2822,12 +2908,12 @@ SDValue SITargetLowering::LowerFormalArguments( const Function &Fn = MF.getFunction(); FunctionType *FType = MF.getFunction().getFunctionType(); SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); + bool IsError = false; if (Subtarget->isAmdHsaOS() && AMDGPU::isGraphics(CallConv)) { - DiagnosticInfoUnsupported NoGraphicsHSA( - Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc()); - DAG.getContext()->diagnose(NoGraphicsHSA); - return DAG.getEntryNode(); + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( + Fn, "unsupported non-compute shaders with HSA", DL.getDebugLoc())); + IsError = true; } SmallVector<ISD::InputArg, 16> Splits; @@ -2936,8 +3022,8 @@ SDValue SITargetLowering::LowerFormalArguments( for (unsigned i = 0, e = Ins.size(), ArgIdx = 0; i != e; ++i) { const ISD::InputArg &Arg = Ins[i]; - if (Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) { - InVals.push_back(DAG.getUNDEF(Arg.VT)); + if ((Arg.isOrigArg() && Skipped[Arg.getOrigArgIndex()]) || IsError) { + InVals.push_back(DAG.getPOISON(Arg.VT)); continue; } @@ -3050,11 +3136,10 @@ SDValue SITargetLowering::LowerFormalArguments( if (Arg.isOrigArg()) { Argument *OrigArg = Fn.getArg(Arg.getOrigArgIndex()); if (OrigArg->hasAttribute("amdgpu-hidden-argument")) { - DiagnosticInfoUnsupported NonPreloadHiddenArg( + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( *OrigArg->getParent(), "hidden argument in kernel signature was not preloaded", - DL.getDebugLoc()); - DAG.getContext()->diagnose(NonPreloadHiddenArg); + DL.getDebugLoc())); } } @@ -3196,6 +3281,7 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, const SDLoc &DL, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); + const SIRegisterInfo *TRI = getSubtarget()->getRegisterInfo(); if (AMDGPU::isKernel(CallConv)) { return AMDGPUTargetLowering::LowerReturn(Chain, CallConv, isVarArg, Outs, @@ -3209,7 +3295,6 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, // CCValAssign - represent the assignment of the return value to a location. SmallVector<CCValAssign, 48> RVLocs; - SmallVector<ISD::OutputArg, 48> Splits; // CCState - Info about the registers and stack slots. CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs, @@ -3222,6 +3307,8 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, SmallVector<SDValue, 48> RetOps; RetOps.push_back(Chain); // Operand #0 = Chain (updated below) + SDValue ReadFirstLane = + DAG.getTargetConstant(Intrinsic::amdgcn_readfirstlane, DL, MVT::i32); // Copy the result values into the output registers. for (unsigned I = 0, RealRVLocIdx = 0, E = RVLocs.size(); I != E; ++I, ++RealRVLocIdx) { @@ -3249,7 +3336,9 @@ SITargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv, default: llvm_unreachable("Unknown loc info!"); } - + if (TRI->isSGPRPhysReg(VA.getLocReg())) + Arg = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Arg.getValueType(), + ReadFirstLane, Arg); Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Arg, Glue); Glue = Chain.getValue(1); RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT())); @@ -3416,12 +3505,12 @@ void SITargetLowering::passSpecialInputs( if (Id.has_value()) { InputReg = DAG.getConstant(*Id, DL, ArgVT); } else { - InputReg = DAG.getUNDEF(ArgVT); + InputReg = DAG.getPOISON(ArgVT); } } else { // We may have proven the input wasn't needed, although the ABI is // requiring it. We just need to allocate the register appropriately. - InputReg = DAG.getUNDEF(ArgVT); + InputReg = DAG.getPOISON(ArgVT); } if (OutgoingArg->isRegister()) { @@ -3501,7 +3590,7 @@ void SITargetLowering::passSpecialInputs( // ID, but the calling function does not have it (e.g a graphics function // calling a C calling convention function). This is illegal, but we need // to produce something. - InputReg = DAG.getUNDEF(MVT::i32); + InputReg = DAG.getPOISON(MVT::i32); } else { // Workitem ids are already packed, any of present incoming arguments // will carry all required fields. @@ -3529,21 +3618,6 @@ void SITargetLowering::passSpecialInputs( } } -static bool canGuaranteeTCO(CallingConv::ID CC) { - return CC == CallingConv::Fast; -} - -/// Return true if we might ever do TCO for calls with this calling convention. -static bool mayTailCallThisCC(CallingConv::ID CC) { - switch (CC) { - case CallingConv::C: - case CallingConv::AMDGPU_Gfx: - return true; - default: - return canGuaranteeTCO(CC); - } -} - bool SITargetLowering::isEligibleForTailCallOptimization( SDValue Callee, CallingConv::ID CalleeCC, bool IsVarArg, const SmallVectorImpl<ISD::OutputArg> &Outs, @@ -3552,7 +3626,7 @@ bool SITargetLowering::isEligibleForTailCallOptimization( if (AMDGPU::isChainCC(CalleeCC)) return true; - if (!mayTailCallThisCC(CalleeCC)) + if (!AMDGPU::mayTailCallThisCC(CalleeCC)) return false; // For a divergent call target, we need to do a waterfall loop over the @@ -3574,7 +3648,7 @@ bool SITargetLowering::isEligibleForTailCallOptimization( bool CCMatch = CallerCC == CalleeCC; if (DAG.getTarget().Options.GuaranteedTailCallOpt) { - if (canGuaranteeTCO(CalleeCC) && CCMatch) + if (AMDGPU::canGuaranteeTCO(CalleeCC) && CCMatch) return true; return false; } @@ -3650,6 +3724,19 @@ bool SITargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const { return true; } +namespace { +// Chain calls have special arguments that we need to handle. These are +// tagging along at the end of the arguments list(s), after the SGPR and VGPR +// arguments (index 0 and 1 respectively). +enum ChainCallArgIdx { + Exec = 2, + Flags, + NumVGPRs, + FallbackExec, + FallbackCallee +}; +} // anonymous namespace + // The wave scratch offset register is used as the global base pointer. SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, SmallVectorImpl<SDValue> &InVals) const { @@ -3658,37 +3745,75 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, SelectionDAG &DAG = CLI.DAG; - TargetLowering::ArgListEntry RequestedExec; + const SDLoc &DL = CLI.DL; + SDValue Chain = CLI.Chain; + SDValue Callee = CLI.Callee; + + llvm::SmallVector<SDValue, 6> ChainCallSpecialArgs; + bool UsesDynamicVGPRs = false; if (IsChainCallConv) { - // The last argument should be the value that we need to put in EXEC. - // Pop it out of CLI.Outs and CLI.OutVals before we do any processing so we - // don't treat it like the rest of the arguments. - RequestedExec = CLI.Args.back(); - assert(RequestedExec.Node && "No node for EXEC"); + // The last arguments should be the value that we need to put in EXEC, + // followed by the flags and any other arguments with special meanings. + // Pop them out of CLI.Outs and CLI.OutVals before we do any processing so + // we don't treat them like the "real" arguments. + auto RequestedExecIt = + llvm::find_if(CLI.Outs, [](const ISD::OutputArg &Arg) { + return Arg.OrigArgIndex == 2; + }); + assert(RequestedExecIt != CLI.Outs.end() && "No node for EXEC"); - if (!RequestedExec.Ty->isIntegerTy(Subtarget->getWavefrontSize())) + size_t SpecialArgsBeginIdx = RequestedExecIt - CLI.Outs.begin(); + CLI.OutVals.erase(CLI.OutVals.begin() + SpecialArgsBeginIdx, + CLI.OutVals.end()); + CLI.Outs.erase(RequestedExecIt, CLI.Outs.end()); + + assert(CLI.Outs.back().OrigArgIndex < 2 && + "Haven't popped all the special args"); + + TargetLowering::ArgListEntry RequestedExecArg = + CLI.Args[ChainCallArgIdx::Exec]; + if (!RequestedExecArg.Ty->isIntegerTy(Subtarget->getWavefrontSize())) return lowerUnhandledCall(CLI, InVals, "Invalid value for EXEC"); - assert(CLI.Outs.back().OrigArgIndex == 2 && "Unexpected last arg"); - CLI.Outs.pop_back(); - CLI.OutVals.pop_back(); + // Convert constants into TargetConstants, so they become immediate operands + // instead of being selected into S_MOV. + auto PushNodeOrTargetConstant = [&](TargetLowering::ArgListEntry Arg) { + if (const auto *ArgNode = dyn_cast<ConstantSDNode>(Arg.Node)) { + ChainCallSpecialArgs.push_back(DAG.getTargetConstant( + ArgNode->getAPIntValue(), DL, ArgNode->getValueType(0))); + } else + ChainCallSpecialArgs.push_back(Arg.Node); + }; - if (RequestedExec.Ty->isIntegerTy(64)) { - assert(CLI.Outs.back().OrigArgIndex == 2 && "Exec wasn't split up"); - CLI.Outs.pop_back(); - CLI.OutVals.pop_back(); - } + PushNodeOrTargetConstant(RequestedExecArg); + + // Process any other special arguments depending on the value of the flags. + TargetLowering::ArgListEntry Flags = CLI.Args[ChainCallArgIdx::Flags]; + + const APInt &FlagsValue = cast<ConstantSDNode>(Flags.Node)->getAPIntValue(); + if (FlagsValue.isZero()) { + if (CLI.Args.size() > ChainCallArgIdx::Flags + 1) + return lowerUnhandledCall(CLI, InVals, + "no additional args allowed if flags == 0"); + } else if (FlagsValue.isOneBitSet(0)) { + if (CLI.Args.size() != ChainCallArgIdx::FallbackCallee + 1) { + return lowerUnhandledCall(CLI, InVals, "expected 3 additional args"); + } - assert(CLI.Outs.back().OrigArgIndex != 2 && - "Haven't popped all the pieces of the EXEC mask"); + if (!Subtarget->isWave32()) { + return lowerUnhandledCall( + CLI, InVals, "dynamic VGPR mode is only supported for wave32"); + } + + UsesDynamicVGPRs = true; + std::for_each(CLI.Args.begin() + ChainCallArgIdx::NumVGPRs, + CLI.Args.end(), PushNodeOrTargetConstant); + } } - const SDLoc &DL = CLI.DL; SmallVector<ISD::OutputArg, 32> &Outs = CLI.Outs; SmallVector<SDValue, 32> &OutVals = CLI.OutVals; SmallVector<ISD::InputArg, 32> &Ins = CLI.Ins; - SDValue Chain = CLI.Chain; - SDValue Callee = CLI.Callee; bool &IsTailCall = CLI.IsTailCall; bool IsVarArg = CLI.IsVarArg; bool IsSibCall = false; @@ -3697,7 +3822,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, if (Callee.isUndef() || isNullConstant(Callee)) { if (!CLI.IsTailCall) { for (ISD::InputArg &Arg : CLI.Ins) - InVals.push_back(DAG.getUNDEF(Arg.VT)); + InVals.push_back(DAG.getPOISON(Arg.VT)); } return Chain; @@ -3709,7 +3834,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, } if (!CLI.CB) - report_fatal_error("unsupported libcall legalization"); + return lowerUnhandledCall(CLI, InVals, "unsupported libcall legalization"); if (IsTailCall && MF.getTarget().Options.GuaranteedTailCallOpt) { return lowerUnhandledCall(CLI, InVals, @@ -3768,7 +3893,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, // arguments to begin at SP+0. Completely unused for non-tail calls. int32_t FPDiff = 0; MachineFrameInfo &MFI = MF.getFrameInfo(); - auto *TRI = static_cast<const SIRegisterInfo *>(Subtarget->getRegisterInfo()); + auto *TRI = Subtarget->getRegisterInfo(); // Adjust the stack pointer for the new arguments... // These operations are automatically eliminated by the prolog/epilog pass @@ -3976,7 +4101,7 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, } if (IsChainCallConv) - Ops.push_back(RequestedExec.Node); + llvm::append_range(Ops, ChainCallSpecialArgs); // Add argument registers to the end of the list so that they are known live // into the call. @@ -4013,7 +4138,8 @@ SDValue SITargetLowering::LowerCall(CallLoweringInfo &CLI, break; case CallingConv::AMDGPU_CS_Chain: case CallingConv::AMDGPU_CS_ChainPreserve: - OPC = AMDGPUISD::TC_RETURN_CHAIN; + OPC = UsesDynamicVGPRs ? AMDGPUISD::TC_RETURN_CHAIN_DVGPR + : AMDGPUISD::TC_RETURN_CHAIN; break; } @@ -4384,6 +4510,8 @@ SDValue SITargetLowering::lowerSET_FPENV(SDValue Op, SelectionDAG &DAG) const { Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const { + const Function &Fn = MF.getFunction(); + Register Reg = StringSwitch<Register>(RegName) .Case("m0", AMDGPU::M0) .Case("exec", AMDGPU::EXEC) @@ -4393,16 +4521,13 @@ Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT, .Case("flat_scratch_lo", AMDGPU::FLAT_SCR_LO) .Case("flat_scratch_hi", AMDGPU::FLAT_SCR_HI) .Default(Register()); - - if (Reg == AMDGPU::NoRegister) { - report_fatal_error( - Twine("invalid register name \"" + StringRef(RegName) + "\".")); - } + if (!Reg) + return Reg; if (!Subtarget->hasFlatScrRegister() && Subtarget->getRegisterInfo()->regsOverlap(Reg, AMDGPU::FLAT_SCR)) { - report_fatal_error(Twine("invalid register \"" + StringRef(RegName) + - "\" for subtarget.")); + Fn.getContext().emitError(Twine("invalid register \"" + StringRef(RegName) + + "\" for subtarget.")); } switch (Reg) { @@ -4432,7 +4557,7 @@ Register SITargetLowering::getRegisterByName(const char *RegName, LLT VT, MachineBasicBlock * SITargetLowering::splitKillBlock(MachineInstr &MI, MachineBasicBlock *BB) const { - MachineBasicBlock *SplitBB = BB->splitAt(MI, false /*UpdateLiveIns*/); + MachineBasicBlock *SplitBB = BB->splitAt(MI, /*UpdateLiveIns=*/true); const SIInstrInfo *TII = getSubtarget()->getInstrInfo(); MI.setDesc(TII->getKillTerminatorFromPseudo(MI.getOpcode())); return SplitBB; @@ -4563,7 +4688,8 @@ emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI, const TargetRegisterClass *BoolRC = TRI->getBoolRC(); Register PhiExec = MRI.createVirtualRegister(BoolRC); Register NewExec = MRI.createVirtualRegister(BoolRC); - Register CurrentIdxReg = MRI.createVirtualRegister(&AMDGPU::SGPR_32RegClass); + Register CurrentIdxReg = + MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); Register CondReg = MRI.createVirtualRegister(BoolRC); BuildMI(LoopBB, I, DL, TII->get(TargetOpcode::PHI), PhiReg) @@ -4608,7 +4734,7 @@ emitLoadM0FromVGPRLoop(const SIInstrInfo *TII, MachineRegisterInfo &MRI, } else { // Move index from VCC into M0 if (Offset == 0) { - BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0) + BuildMI(LoopBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::M0) .addReg(CurrentIdxReg, RegState::Kill); } else { BuildMI(LoopBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0) @@ -4722,7 +4848,7 @@ static void setM0ToIndexFromSGPR(const SIInstrInfo *TII, if (Offset == 0) { // clang-format off - BuildMI(*MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0) + BuildMI(*MBB, I, DL, TII->get(AMDGPU::COPY), AMDGPU::M0) .add(*Idx); // clang-format on } else { @@ -4940,6 +5066,28 @@ static MachineBasicBlock *emitIndirectDst(MachineInstr &MI, return LoopBB; } +static uint32_t getIdentityValueForWaveReduction(unsigned Opc) { + switch (Opc) { + case AMDGPU::S_MIN_U32: + return std::numeric_limits<uint32_t>::max(); + case AMDGPU::S_MIN_I32: + return std::numeric_limits<int32_t>::max(); + case AMDGPU::S_MAX_U32: + return std::numeric_limits<uint32_t>::min(); + case AMDGPU::S_MAX_I32: + return std::numeric_limits<int32_t>::min(); + case AMDGPU::S_ADD_I32: + case AMDGPU::S_SUB_I32: + case AMDGPU::S_OR_B32: + case AMDGPU::S_XOR_B32: + return std::numeric_limits<uint32_t>::min(); + case AMDGPU::S_AND_B32: + return std::numeric_limits<uint32_t>::max(); + default: + llvm_unreachable("Unexpected opcode in getIdentityValueForWaveReduction"); + } +} + static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI, MachineBasicBlock &BB, const GCNSubtarget &ST, @@ -4955,13 +5103,78 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI, Register DstReg = MI.getOperand(0).getReg(); MachineBasicBlock *RetBB = nullptr; if (isSGPR) { - // These operations with a uniform value i.e. SGPR are idempotent. - // Reduced value will be same as given sgpr. - // clang-format off - BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg) - .addReg(SrcReg); - // clang-format on - RetBB = &BB; + switch (Opc) { + case AMDGPU::S_MIN_U32: + case AMDGPU::S_MIN_I32: + case AMDGPU::S_MAX_U32: + case AMDGPU::S_MAX_I32: + case AMDGPU::S_AND_B32: + case AMDGPU::S_OR_B32: { + // Idempotent operations. + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MOV_B32), DstReg).addReg(SrcReg); + RetBB = &BB; + break; + } + case AMDGPU::S_XOR_B32: + case AMDGPU::S_ADD_I32: + case AMDGPU::S_SUB_I32: { + const TargetRegisterClass *WaveMaskRegClass = TRI->getWaveMaskRegClass(); + const TargetRegisterClass *DstRegClass = MRI.getRegClass(DstReg); + Register ExecMask = MRI.createVirtualRegister(WaveMaskRegClass); + Register ActiveLanes = MRI.createVirtualRegister(DstRegClass); + + bool IsWave32 = ST.isWave32(); + unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; + MCRegister ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; + unsigned CountReg = + IsWave32 ? AMDGPU::S_BCNT1_I32_B32 : AMDGPU::S_BCNT1_I32_B64; + + auto Exec = + BuildMI(BB, MI, DL, TII->get(MovOpc), ExecMask).addReg(ExecReg); + + auto NewAccumulator = BuildMI(BB, MI, DL, TII->get(CountReg), ActiveLanes) + .addReg(Exec->getOperand(0).getReg()); + + switch (Opc) { + case AMDGPU::S_XOR_B32: { + // Performing an XOR operation on a uniform value + // depends on the parity of the number of active lanes. + // For even parity, the result will be 0, for odd + // parity the result will be the same as the input value. + Register ParityRegister = MRI.createVirtualRegister(DstRegClass); + + auto ParityReg = + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_AND_B32), ParityRegister) + .addReg(NewAccumulator->getOperand(0).getReg()) + .addImm(1); + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg) + .addReg(SrcReg) + .addReg(ParityReg->getOperand(0).getReg()); + break; + } + case AMDGPU::S_SUB_I32: { + Register NegatedVal = MRI.createVirtualRegister(DstRegClass); + + // Take the negation of the source operand. + auto InvertedValReg = + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), NegatedVal) + .addImm(-1) + .addReg(SrcReg); + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg) + .addReg(InvertedValReg->getOperand(0).getReg()) + .addReg(NewAccumulator->getOperand(0).getReg()); + break; + } + case AMDGPU::S_ADD_I32: { + BuildMI(BB, MI, DL, TII->get(AMDGPU::S_MUL_I32), DstReg) + .addReg(SrcReg) + .addReg(NewAccumulator->getOperand(0).getReg()); + break; + } + } + RetBB = &BB; + } + } } else { // TODO: Implement DPP Strategy and switch based on immediate strategy // operand. For now, for all the cases (default, Iterative and DPP we use @@ -4990,16 +5203,16 @@ static MachineBasicBlock *lowerWaveReduce(MachineInstr &MI, Register NewActiveBitsReg = MRI.createVirtualRegister(WaveMaskRegClass); Register FF1Reg = MRI.createVirtualRegister(DstRegClass); - Register LaneValueReg = MRI.createVirtualRegister(DstRegClass); + Register LaneValueReg = + MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); bool IsWave32 = ST.isWave32(); unsigned MovOpc = IsWave32 ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64; unsigned ExecReg = IsWave32 ? AMDGPU::EXEC_LO : AMDGPU::EXEC; - // Create initail values of induction variable from Exec, Accumulator and - // insert branch instr to newly created ComputeBlockk - uint32_t InitalValue = - (Opc == AMDGPU::S_MIN_U32) ? std::numeric_limits<uint32_t>::max() : 0; + // Create initial values of induction variable from Exec, Accumulator and + // insert branch instr to newly created ComputeBlock + uint32_t InitalValue = getIdentityValueForWaveReduction(Opc); auto TmpSReg = BuildMI(BB, I, DL, TII->get(MovOpc), LoopIterator).addReg(ExecReg); BuildMI(BB, I, DL, TII->get(AMDGPU::S_MOV_B32), InitalValReg) @@ -5071,8 +5284,22 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, switch (MI.getOpcode()) { case AMDGPU::WAVE_REDUCE_UMIN_PSEUDO_U32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MIN_U32); + case AMDGPU::WAVE_REDUCE_MIN_PSEUDO_I32: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MIN_I32); case AMDGPU::WAVE_REDUCE_UMAX_PSEUDO_U32: return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MAX_U32); + case AMDGPU::WAVE_REDUCE_MAX_PSEUDO_I32: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_MAX_I32); + case AMDGPU::WAVE_REDUCE_ADD_PSEUDO_I32: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_ADD_I32); + case AMDGPU::WAVE_REDUCE_SUB_PSEUDO_I32: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_SUB_I32); + case AMDGPU::WAVE_REDUCE_AND_PSEUDO_B32: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_AND_B32); + case AMDGPU::WAVE_REDUCE_OR_PSEUDO_B32: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_OR_B32); + case AMDGPU::WAVE_REDUCE_XOR_PSEUDO_B32: + return lowerWaveReduce(MI, *BB, *getSubtarget(), AMDGPU::S_XOR_B32); case AMDGPU::S_UADDO_PSEUDO: case AMDGPU::S_USUBO_PSEUDO: { const DebugLoc &DL = MI.getDebugLoc(); @@ -5162,7 +5389,7 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, MachineOperand &Src0 = MI.getOperand(1); MachineOperand &Src1 = MI.getOperand(2); - if (IsAdd && ST.hasLshlAddB64()) { + if (IsAdd && ST.hasLshlAddU64Inst()) { auto Add = BuildMI(*BB, MI, DL, TII->get(AMDGPU::V_LSHL_ADD_U64_e64), Dest.getReg()) .add(Src0) @@ -5249,18 +5476,18 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; if (Src0.isReg() && TRI->isVectorRegister(MRI, Src0.getReg())) { - Register RegOp0 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register RegOp0 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp0) .addReg(Src0.getReg()); Src0.setReg(RegOp0); } if (Src1.isReg() && TRI->isVectorRegister(MRI, Src1.getReg())) { - Register RegOp1 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register RegOp1 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp1) .addReg(Src1.getReg()); Src1.setReg(RegOp1); } - Register RegOp2 = MRI.createVirtualRegister(&AMDGPU::SReg_32RegClass); + Register RegOp2 = MRI.createVirtualRegister(&AMDGPU::SReg_32_XM0RegClass); if (TRI->isVectorRegister(MRI, Src2.getReg())) { BuildMI(*BB, MII, DL, TII->get(AMDGPU::V_READFIRSTLANE_B32), RegOp2) .addReg(Src2.getReg()); @@ -5316,12 +5543,22 @@ SITargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, return BB; } case AMDGPU::SI_INIT_M0: { + MachineOperand &M0Init = MI.getOperand(0); BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(), - TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0) - .add(MI.getOperand(0)); + TII->get(M0Init.isReg() ? AMDGPU::COPY : AMDGPU::S_MOV_B32), + AMDGPU::M0) + .add(M0Init); MI.eraseFromParent(); return BB; } + case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM: { + // Set SCC to true, in case the barrier instruction gets converted to a NOP. + BuildMI(*BB, MI.getIterator(), MI.getDebugLoc(), + TII->get(AMDGPU::S_CMP_EQ_U32)) + .addImm(0) + .addImm(0); + return BB; + } case AMDGPU::GET_GROUPSTATICSIZE: { assert(getTargetMachine().getTargetTriple().getOS() == Triple::AMDHSA || getTargetMachine().getTargetTriple().getOS() == Triple::AMDPAL); @@ -5751,35 +5988,6 @@ bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, return false; } -// Refer to comments added to the MIR variant of isFMAFasterThanFMulAndFAdd for -// specific details. -bool SITargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F, - Type *Ty) const { - switch (Ty->getScalarSizeInBits()) { - case 16: { - SIModeRegisterDefaults Mode = SIModeRegisterDefaults(F, *Subtarget); - return Subtarget->has16BitInsts() && - Mode.FP64FP16Denormals != DenormalMode::getPreserveSign(); - } - case 32: { - if (!Subtarget->hasMadMacF32Insts()) - return Subtarget->hasFastFMAF32(); - - SIModeRegisterDefaults Mode = SIModeRegisterDefaults(F, *Subtarget); - if (Mode.FP32Denormals != DenormalMode::getPreserveSign()) - return Subtarget->hasFastFMAF32() || Subtarget->hasDLInsts(); - - return Subtarget->hasFastFMAF32() && Subtarget->hasDLInsts(); - } - case 64: - return true; - default: - break; - } - - return false; -} - bool SITargetLowering::isFMADLegal(const MachineInstr &MI, LLT Ty) const { if (!Ty.isScalar()) return false; @@ -5839,10 +6047,12 @@ SDValue SITargetLowering::splitBinaryVectorOp(SDValue Op, SelectionDAG &DAG) const { unsigned Opc = Op.getOpcode(); EVT VT = Op.getValueType(); - assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4f32 || - VT == MVT::v8i16 || VT == MVT::v8f16 || VT == MVT::v16i16 || - VT == MVT::v16f16 || VT == MVT::v8f32 || VT == MVT::v16f32 || - VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16); + assert(VT == MVT::v4i16 || VT == MVT::v4f16 || VT == MVT::v4bf16 || + VT == MVT::v4f32 || VT == MVT::v8i16 || VT == MVT::v8f16 || + VT == MVT::v8bf16 || VT == MVT::v16i16 || VT == MVT::v16f16 || + VT == MVT::v16bf16 || VT == MVT::v8f32 || VT == MVT::v16f32 || + VT == MVT::v32f32 || VT == MVT::v32i16 || VT == MVT::v32f16 || + VT == MVT::v32bf16); auto [Lo0, Hi0] = DAG.SplitVectorOperand(Op.getNode(), 0); auto [Lo1, Hi1] = DAG.SplitVectorOperand(Op.getNode(), 1); @@ -5963,6 +6173,12 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::FMINNUM: case ISD::FMAXNUM: return lowerFMINNUM_FMAXNUM(Op, DAG); + case ISD::FMINIMUMNUM: + case ISD::FMAXIMUMNUM: + return lowerFMINIMUMNUM_FMAXIMUMNUM(Op, DAG); + case ISD::FMINIMUM: + case ISD::FMAXIMUM: + return lowerFMINIMUM_FMAXIMUM(Op, DAG); case ISD::FLDEXP: case ISD::STRICT_FLDEXP: return lowerFLDEXP(Op, DAG); @@ -5984,15 +6200,13 @@ SDValue SITargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { case ISD::FMUL: case ISD::FMINNUM_IEEE: case ISD::FMAXNUM_IEEE: - case ISD::FMINIMUM: - case ISD::FMAXIMUM: - case ISD::FMINIMUMNUM: - case ISD::FMAXIMUMNUM: case ISD::UADDSAT: case ISD::USUBSAT: case ISD::SADDSAT: case ISD::SSUBSAT: return splitBinaryVectorOp(Op, DAG); + case ISD::FCOPYSIGN: + return lowerFCOPYSIGN(Op, DAG); case ISD::MUL: return lowerMUL(Op, DAG); case ISD::SMULO: @@ -6053,7 +6267,7 @@ static SDValue adjustLoadValueTypeImpl(SDValue Result, EVT LoadVT, // Pad illegal v1i16/v3fi6 to v4i16 if ((LoadVT.getVectorNumElements() % 2) == 1) - Elts.push_back(DAG.getUNDEF(MVT::i16)); + Elts.push_back(DAG.getPOISON(MVT::i16)); Result = DAG.getBuildVector(IntLoadVT, DL, Elts); @@ -6145,7 +6359,7 @@ static SDValue lowerICMPIntrinsic(const SITargetLowering &TLI, SDNode *N, EVT VT = N->getValueType(0); unsigned CondCode = N->getConstantOperandVal(3); if (!ICmpInst::isIntPredicate(static_cast<ICmpInst::Predicate>(CondCode))) - return DAG.getUNDEF(VT); + return DAG.getPOISON(VT); ICmpInst::Predicate IcInput = static_cast<ICmpInst::Predicate>(CondCode); @@ -6180,7 +6394,7 @@ static SDValue lowerFCMPIntrinsic(const SITargetLowering &TLI, SDNode *N, unsigned CondCode = N->getConstantOperandVal(3); if (!FCmpInst::isFPPredicate(static_cast<FCmpInst::Predicate>(CondCode))) - return DAG.getUNDEF(VT); + return DAG.getPOISON(VT); SDValue Src0 = N->getOperand(1); SDValue Src1 = N->getOperand(2); @@ -6543,6 +6757,11 @@ void SITargetLowering::ReplaceNodeResults(SDNode *N, Results.push_back(LoadVal); return; } + case Intrinsic::amdgcn_dead: { + for (unsigned I = 0, E = N->getNumValues(); I < E; ++I) + Results.push_back(DAG.getPOISON(N->getValueType(I))); + return; + } } break; } @@ -6800,24 +7019,72 @@ SDValue SITargetLowering::getFPExtOrFPRound(SelectionDAG &DAG, SDValue Op, DAG.getTargetConstant(0, DL, MVT::i32)); } -SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { - assert(Op.getValueType() == MVT::f16 && - "Do not know how to custom lower FP_ROUND for non-f16 type"); +SDValue SITargetLowering::splitFP_ROUNDVectorOp(SDValue Op, + SelectionDAG &DAG) const { + EVT DstVT = Op.getValueType(); + unsigned NumElts = DstVT.getVectorNumElements(); + assert(NumElts > 2 && isPowerOf2_32(NumElts)); + + auto [Lo, Hi] = DAG.SplitVectorOperand(Op.getNode(), 0); + + SDLoc DL(Op); + unsigned Opc = Op.getOpcode(); + SDValue Flags = Op.getOperand(1); + EVT HalfDstVT = + EVT::getVectorVT(*DAG.getContext(), DstVT.getScalarType(), NumElts / 2); + SDValue OpLo = DAG.getNode(Opc, DL, HalfDstVT, Lo, Flags); + SDValue OpHi = DAG.getNode(Opc, DL, HalfDstVT, Hi, Flags); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, OpLo, OpHi); +} + +SDValue SITargetLowering::lowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const { SDValue Src = Op.getOperand(0); EVT SrcVT = Src.getValueType(); - if (SrcVT != MVT::f64) - return Op; + EVT DstVT = Op.getValueType(); - // TODO: Handle strictfp - if (Op.getOpcode() != ISD::FP_ROUND) + if (DstVT.isVector() && DstVT.getScalarType() == MVT::f16) { + assert(Subtarget->hasCvtPkF16F32Inst() && "support v_cvt_pk_f16_f32"); + if (SrcVT.getScalarType() != MVT::f32) + return SDValue(); + return SrcVT == MVT::v2f32 ? Op : splitFP_ROUNDVectorOp(Op, DAG); + } + + if (SrcVT.getScalarType() != MVT::f64) return Op; SDLoc DL(Op); + if (DstVT == MVT::f16) { + // TODO: Handle strictfp + if (Op.getOpcode() != ISD::FP_ROUND) + return Op; - SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src); - SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); - return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); + if (!Subtarget->has16BitInsts()) { + SDValue FpToFp16 = DAG.getNode(ISD::FP_TO_FP16, DL, MVT::i32, Src); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); + return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); + } + if (getTargetMachine().Options.UnsafeFPMath) { + SDValue Flags = Op.getOperand(1); + SDValue Src32 = DAG.getNode(ISD::FP_ROUND, DL, MVT::f32, Src, Flags); + return DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, Src32, Flags); + } + SDValue FpToFp16 = LowerF64ToF16Safe(Src, DL, DAG); + SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, FpToFp16); + return DAG.getNode(ISD::BITCAST, DL, MVT::f16, Trunc); + } + + assert(DstVT.getScalarType() == MVT::bf16 && + "custom lower FP_ROUND for f16 or bf16"); + assert(Subtarget->hasBF16ConversionInsts() && "f32 -> bf16 is legal"); + + // Round-inexact-to-odd f64 to f32, then do the final rounding using the + // hardware f32 -> bf16 instruction. + EVT F32VT = SrcVT.isVector() ? SrcVT.changeVectorElementType(MVT::f32) : + MVT::f32; + SDValue Rod = expandRoundInexactToOdd(F32VT, Src, DL, DAG); + return DAG.getNode(ISD::FP_ROUND, DL, DstVT, Rod, + DAG.getTargetConstant(0, DL, MVT::i32)); } SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op, @@ -6840,6 +7107,52 @@ SDValue SITargetLowering::lowerFMINNUM_FMAXNUM(SDValue Op, return Op; } +SDValue +SITargetLowering::lowerFMINIMUMNUM_FMAXIMUMNUM(SDValue Op, + SelectionDAG &DAG) const { + EVT VT = Op.getValueType(); + const MachineFunction &MF = DAG.getMachineFunction(); + const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); + bool IsIEEEMode = Info->getMode().IEEE; + + if (IsIEEEMode) + return expandFMINIMUMNUM_FMAXIMUMNUM(Op.getNode(), DAG); + + if (VT == MVT::v4f16 || VT == MVT::v8f16 || VT == MVT::v16f16 || + VT == MVT::v16bf16) + return splitBinaryVectorOp(Op, DAG); + return Op; +} + +SDValue SITargetLowering::lowerFMINIMUM_FMAXIMUM(SDValue Op, + SelectionDAG &DAG) const { + EVT VT = Op.getValueType(); + if (VT.isVector()) + return splitBinaryVectorOp(Op, DAG); + + assert(!Subtarget->hasIEEEMinimumMaximumInsts() && + !Subtarget->hasMinimum3Maximum3F16() && + Subtarget->hasMinimum3Maximum3PKF16() && VT == MVT::f16 && + "should not need to widen f16 minimum/maximum to v2f16"); + + // Widen f16 operation to v2f16 + + // fminimum f16:x, f16:y -> + // extract_vector_elt (fminimum (v2f16 (scalar_to_vector x)) + // (v2f16 (scalar_to_vector y))), 0 + SDLoc SL(Op); + SDValue WideSrc0 = + DAG.getNode(ISD::SCALAR_TO_VECTOR, SL, MVT::v2f16, Op.getOperand(0)); + SDValue WideSrc1 = + DAG.getNode(ISD::SCALAR_TO_VECTOR, SL, MVT::v2f16, Op.getOperand(1)); + + SDValue Widened = + DAG.getNode(Op.getOpcode(), SL, MVT::v2f16, WideSrc0, WideSrc1); + + return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, MVT::f16, Widened, + DAG.getConstant(0, SL, MVT::i32)); +} + SDValue SITargetLowering::lowerFLDEXP(SDValue Op, SelectionDAG &DAG) const { bool IsStrict = Op.getOpcode() == ISD::STRICT_FLDEXP; EVT VT = Op.getValueType(); @@ -6960,6 +7273,31 @@ SDValue SITargetLowering::promoteUniformOpToI32(SDValue Op, return DAG.getZExtOrTrunc(NewVal, DL, OpTy); } +SDValue SITargetLowering::lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const { + SDValue Mag = Op.getOperand(0); + EVT MagVT = Mag.getValueType(); + + if (MagVT.getVectorNumElements() > 2) + return splitBinaryVectorOp(Op, DAG); + + SDValue Sign = Op.getOperand(1); + EVT SignVT = Sign.getValueType(); + + if (MagVT == SignVT) + return Op; + + // fcopysign v2f16:mag, v2f32:sign -> + // fcopysign v2f16:mag, bitcast (trunc (bitcast sign to v2i32) to v2i16) + + SDLoc SL(Op); + SDValue SignAsInt32 = DAG.getNode(ISD::BITCAST, SL, MVT::v2i32, Sign); + SDValue SignAsInt16 = DAG.getNode(ISD::TRUNCATE, SL, MVT::v2i16, SignAsInt32); + + SDValue SignAsHalf16 = DAG.getNode(ISD::BITCAST, SL, MagVT, SignAsInt16); + + return DAG.getNode(ISD::FCOPYSIGN, SL, MagVT, Mag, SignAsHalf16); +} + // Custom lowering for vector multiplications and s_mul_u64. SDValue SITargetLowering::lowerMUL(SDValue Op, SelectionDAG &DAG) const { EVT VT = Op.getValueType(); @@ -7158,11 +7496,10 @@ SDValue SITargetLowering::lowerDEBUGTRAP(SDValue Op, SelectionDAG &DAG) const { if (!Subtarget->isTrapHandlerEnabled() || Subtarget->getTrapHandlerAbi() != GCNSubtarget::TrapHandlerAbi::AMDHSA) { - DiagnosticInfoUnsupported NoTrap(MF.getFunction(), - "debugtrap handler not supported", - Op.getDebugLoc(), DS_Warning); LLVMContext &Ctx = MF.getFunction().getContext(); - Ctx.diagnose(NoTrap); + Ctx.diagnose(DiagnosticInfoUnsupported(MF.getFunction(), + "debugtrap handler not supported", + Op.getDebugLoc(), DS_Warning)); return Chain; } @@ -7218,7 +7555,7 @@ SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL, if (UserSGPR == AMDGPU::NoRegister) { // We probably are in a function incorrectly marked with // amdgpu-no-queue-ptr. This is undefined. - return DAG.getUNDEF(MVT::i32); + return DAG.getPOISON(MVT::i32); } SDValue QueuePtr = @@ -7245,8 +7582,7 @@ SDValue SITargetLowering::getSegmentAperture(unsigned AS, const SDLoc &DL, /// not necessary. static bool isKnownNonNull(SDValue Val, SelectionDAG &DAG, const AMDGPUTargetMachine &TM, unsigned AddrSpace) { - if (isa<FrameIndexSDNode>(Val) || isa<GlobalAddressSDNode>(Val) || - isa<BasicBlockSDNode>(Val)) + if (isa<FrameIndexSDNode, GlobalAddressSDNode, BasicBlockSDNode>(Val)) return true; if (auto *ConstVal = dyn_cast<ConstantSDNode>(Val)) @@ -7340,12 +7676,8 @@ SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op, // global <-> flat are no-ops and never emitted. - const MachineFunction &MF = DAG.getMachineFunction(); - DiagnosticInfoUnsupported InvalidAddrSpaceCast( - MF.getFunction(), "invalid addrspacecast", SL.getDebugLoc()); - DAG.getContext()->diagnose(InvalidAddrSpaceCast); - - return DAG.getUNDEF(Op->getValueType(0)); + // Invalid casts are poison. + return DAG.getPOISON(Op->getValueType(0)); } // This lowers an INSERT_SUBVECTOR by extracting the individual elements from @@ -7687,7 +8019,7 @@ SDValue SITargetLowering::lowerVECTOR_SHUFFLE(SDValue Op, NewMaskIdx1 += NewSrcNumElts; Result1 = SubVec1; } else { - Result1 = DAG.getUNDEF(PackVT); + Result1 = DAG.getPOISON(PackVT); } SDValue Shuf = DAG.getVectorShuffle(PackVT, SL, Result0, Result1, @@ -7720,7 +8052,7 @@ SDValue SITargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op, SDValue SVal = Op.getOperand(0); EVT ResultVT = Op.getValueType(); EVT SValVT = SVal.getValueType(); - SDValue UndefVal = DAG.getUNDEF(SValVT); + SDValue UndefVal = DAG.getPOISON(SValVT); SDLoc SL(Op); SmallVector<SDValue, 8> VElts; @@ -7940,20 +8272,18 @@ SDValue SITargetLowering::lowerImplicitZextParam(SelectionDAG &DAG, SDValue Op, static SDValue emitNonHSAIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, EVT VT) { - DiagnosticInfoUnsupported BadIntrin(DAG.getMachineFunction().getFunction(), - "non-hsa intrinsic with hsa target", - DL.getDebugLoc()); - DAG.getContext()->diagnose(BadIntrin); - return DAG.getUNDEF(VT); + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( + DAG.getMachineFunction().getFunction(), + "non-hsa intrinsic with hsa target", DL.getDebugLoc())); + return DAG.getPOISON(VT); } static SDValue emitRemovedIntrinsicError(SelectionDAG &DAG, const SDLoc &DL, EVT VT) { - DiagnosticInfoUnsupported BadIntrin(DAG.getMachineFunction().getFunction(), - "intrinsic not supported on subtarget", - DL.getDebugLoc()); - DAG.getContext()->diagnose(BadIntrin); - return DAG.getUNDEF(VT); + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( + DAG.getMachineFunction().getFunction(), + "intrinsic not supported on subtarget", DL.getDebugLoc())); + return DAG.getPOISON(VT); } static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL, @@ -7978,7 +8308,7 @@ static SDValue getBuildDwordsVector(SelectionDAG &DAG, SDLoc DL, VecElts[i] = Elt; } for (unsigned i = Elts.size(); i < NumElts; ++i) - VecElts[i] = DAG.getUNDEF(MVT::f32); + VecElts[i] = DAG.getPOISON(MVT::f32); if (NumElts == 1) return VecElts[0]; @@ -7996,7 +8326,7 @@ static SDValue padEltsToUndef(SelectionDAG &DAG, const SDLoc &DL, EVT CastVT, else Elts.push_back(Src); - SDValue Undef = DAG.getUNDEF(SrcVT.getScalarType()); + SDValue Undef = DAG.getPOISON(SrcVT.getScalarType()); while (ExtraElts--) Elts.push_back(Undef); @@ -8247,7 +8577,7 @@ SDValue SITargetLowering::lowerImage(SDValue Op, // occupies full 32-bit. SDValue Bias = DAG.getBuildVector( MVT::v2f16, DL, - {Op.getOperand(ArgOffset + I), DAG.getUNDEF(MVT::f16)}); + {Op.getOperand(ArgOffset + I), DAG.getPOISON(MVT::f16)}); VAddrs.push_back(Bias); } else { assert((!IsA16 || Intr->NumBiasArgs == 0 || I != Intr->BiasIndex) && @@ -8377,7 +8707,7 @@ SDValue SITargetLowering::lowerImage(SDValue Op, // type if (DMaskLanes == 0 && !BaseOpcode->Store) { // This is a no-op load. This can be eliminated - SDValue Undef = DAG.getUNDEF(Op.getValueType()); + SDValue Undef = DAG.getPOISON(Op.getValueType()); if (isa<MemSDNode>(Op)) return DAG.getMergeValues({Undef, Op.getOperand(0)}, DL); return Undef; @@ -8436,11 +8766,15 @@ SDValue SITargetLowering::lowerImage(SDValue Op, : False); if (IsGFX10Plus) Ops.push_back(IsA16 ? True : False); - if (!Subtarget->hasGFX90AInsts()) { + + if (!Subtarget->hasGFX90AInsts()) Ops.push_back(TFE); // tfe - } else if (TFE->getAsZExtVal()) { - report_fatal_error("TFE is not supported on this GPU"); + else if (TFE->getAsZExtVal()) { + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( + DAG.getMachineFunction().getFunction(), + "TFE is not supported on this GPU", DL.getDebugLoc())); } + if (!IsGFX12Plus || BaseOpcode->Sampler || BaseOpcode->MSAA) Ops.push_back(LWE); // lwe if (!IsGFX10Plus) @@ -8471,9 +8805,23 @@ SDValue SITargetLowering::lowerImage(SDValue Op, if (Subtarget->hasGFX90AInsts()) { Opcode = AMDGPU::getMIMGOpcode(IntrOpcode, AMDGPU::MIMGEncGfx90a, NumVDataDwords, NumVAddrDwords); - if (Opcode == -1) - report_fatal_error( - "requested image instruction is not supported on this GPU"); + if (Opcode == -1) { + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( + DAG.getMachineFunction().getFunction(), + "requested image instruction is not supported on this GPU", + DL.getDebugLoc())); + + unsigned Idx = 0; + SmallVector<SDValue, 3> RetValues(OrigResultTypes.size()); + for (EVT VT : OrigResultTypes) { + if (VT == MVT::Other) + RetValues[Idx++] = Op.getOperand(0); // Chain + else + RetValues[Idx++] = DAG.getPOISON(VT); + } + + return DAG.getMergeValues(RetValues, DL); + } } if (Opcode == -1 && Subtarget->getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS) @@ -8579,7 +8927,7 @@ SDValue SITargetLowering::lowerSBuffer(EVT VT, SDLoc DL, SDValue Rsrc, LoadVT = MVT::getVectorVT(LoadVT.getScalarType(), 4); } - SDVTList VTList = DAG.getVTList({LoadVT, MVT::Glue}); + SDVTList VTList = DAG.getVTList({LoadVT, MVT::Other}); // Use the alignment to ensure that the required offsets will fit into the // immediate offsets. @@ -8619,6 +8967,11 @@ SDValue SITargetLowering::lowerWorkitemID(SelectionDAG &DAG, SDValue Op, if (MaxID == 0) return DAG.getConstant(0, SL, MVT::i32); + // It's undefined behavior if a function marked with the amdgpu-no-* + // attributes uses the corresponding intrinsic. + if (!Arg) + return DAG.getPOISON(Op->getValueType(0)); + SDValue Val = loadInputValue(DAG, &AMDGPU::VGPR_32RegClass, MVT::i32, SDLoc(DAG.getEntryNode()), Arg); @@ -8656,11 +9009,10 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, case Intrinsic::amdgcn_dispatch_ptr: case Intrinsic::amdgcn_queue_ptr: { if (!Subtarget->isAmdHsaOrMesa(MF.getFunction())) { - DiagnosticInfoUnsupported BadIntrin( + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( MF.getFunction(), "unsupported hsa intrinsic without hsa target", - DL.getDebugLoc()); - DAG.getContext()->diagnose(BadIntrin); - return DAG.getUNDEF(VT); + DL.getDebugLoc())); + return DAG.getPOISON(VT); } auto RegID = IntrinsicID == Intrinsic::amdgcn_dispatch_ptr @@ -8733,27 +9085,6 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), SI::KernelInputOffsets::NGROUPS_Z, Align(4), false); - case Intrinsic::r600_read_global_size_x: - if (Subtarget->isAmdHsaOS()) - return emitNonHSAIntrinsicError(DAG, DL, VT); - - return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), - SI::KernelInputOffsets::GLOBAL_SIZE_X, - Align(4), false); - case Intrinsic::r600_read_global_size_y: - if (Subtarget->isAmdHsaOS()) - return emitNonHSAIntrinsicError(DAG, DL, VT); - - return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), - SI::KernelInputOffsets::GLOBAL_SIZE_Y, - Align(4), false); - case Intrinsic::r600_read_global_size_z: - if (Subtarget->isAmdHsaOS()) - return emitNonHSAIntrinsicError(DAG, DL, VT); - - return lowerKernargMemParameter(DAG, VT, VT, DL, DAG.getEntryNode(), - SI::KernelInputOffsets::GLOBAL_SIZE_Z, - Align(4), false); case Intrinsic::r600_read_local_size_x: if (Subtarget->isAmdHsaOS()) return emitNonHSAIntrinsicError(DAG, DL, VT); @@ -9010,6 +9341,12 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, case Intrinsic::amdgcn_mov_dpp8: case Intrinsic::amdgcn_update_dpp: return lowerLaneOp(*this, Op.getNode(), DAG); + case Intrinsic::amdgcn_dead: { + SmallVector<SDValue, 8> Poisons; + for (const EVT ValTy : Op.getNode()->values()) + Poisons.push_back(DAG.getPOISON(ValTy)); + return DAG.getMergeValues(Poisons, SDLoc(Op)); + } default: if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr = AMDGPU::getImageDimIntrinsicInfo(IntrinsicID)) @@ -9109,16 +9446,27 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, IndexOperand &= ~(0xf << 24); if (CountDw < 1 || CountDw > 4) { - report_fatal_error( - "ds_ordered_count: dword count must be between 1 and 4"); + const Function &Fn = DAG.getMachineFunction().getFunction(); + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( + Fn, "ds_ordered_count: dword count must be between 1 and 4", + DL.getDebugLoc())); + CountDw = 1; } } - if (IndexOperand) - report_fatal_error("ds_ordered_count: bad index operand"); + if (IndexOperand) { + const Function &Fn = DAG.getMachineFunction().getFunction(); + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( + Fn, "ds_ordered_count: bad index operand", DL.getDebugLoc())); + } - if (WaveDone && !WaveRelease) - report_fatal_error("ds_ordered_count: wave_done requires wave_release"); + if (WaveDone && !WaveRelease) { + // TODO: Move this to IR verifier + const Function &Fn = DAG.getMachineFunction().getFunction(); + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( + Fn, "ds_ordered_count: wave_done requires wave_release", + DL.getDebugLoc())); + } unsigned Instruction = IntrID == Intrinsic::amdgcn_ds_ordered_add ? 0 : 1; unsigned ShaderType = @@ -9402,6 +9750,51 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, Op->getVTList(), Ops, VT, M->getMemOperand()); } + case Intrinsic::amdgcn_image_bvh_dual_intersect_ray: + case Intrinsic::amdgcn_image_bvh8_intersect_ray: { + MemSDNode *M = cast<MemSDNode>(Op); + SDValue NodePtr = M->getOperand(2); + SDValue RayExtent = M->getOperand(3); + SDValue InstanceMask = M->getOperand(4); + SDValue RayOrigin = M->getOperand(5); + SDValue RayDir = M->getOperand(6); + SDValue Offsets = M->getOperand(7); + SDValue TDescr = M->getOperand(8); + + assert(NodePtr.getValueType() == MVT::i64); + assert(RayDir.getValueType() == MVT::v3f32); + + if (!Subtarget->hasBVHDualAndBVH8Insts()) { + emitRemovedIntrinsicError(DAG, DL, Op.getValueType()); + return SDValue(); + } + + bool IsBVH8 = IntrID == Intrinsic::amdgcn_image_bvh8_intersect_ray; + const unsigned NumVDataDwords = 10; + const unsigned NumVAddrDwords = IsBVH8 ? 11 : 12; + int Opcode = AMDGPU::getMIMGOpcode( + IsBVH8 ? AMDGPU::IMAGE_BVH8_INTERSECT_RAY + : AMDGPU::IMAGE_BVH_DUAL_INTERSECT_RAY, + AMDGPU::MIMGEncGfx12, NumVDataDwords, NumVAddrDwords); + assert(Opcode != -1); + + SmallVector<SDValue, 7> Ops; + Ops.push_back(NodePtr); + Ops.push_back(DAG.getBuildVector( + MVT::v2i32, DL, + {DAG.getBitcast(MVT::i32, RayExtent), + DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, InstanceMask)})); + Ops.push_back(RayOrigin); + Ops.push_back(RayDir); + Ops.push_back(Offsets); + Ops.push_back(TDescr); + Ops.push_back(M->getChain()); + + auto *NewNode = DAG.getMachineNode(Opcode, DL, M->getVTList(), Ops); + MachineMemOperand *MemRef = M->getMemOperand(); + DAG.setNodeMemRefs(NewNode, {MemRef}); + return SDValue(NewNode, 0); + } case Intrinsic::amdgcn_image_bvh_intersect_ray: { MemSDNode *M = cast<MemSDNode>(Op); SDValue NodePtr = M->getOperand(2); @@ -9511,7 +9904,7 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, if (!UseNSA) { // Build a single vector containing all the operands so far prepared. if (NumVAddrDwords > 12) { - SDValue Undef = DAG.getUNDEF(MVT::i32); + SDValue Undef = DAG.getPOISON(MVT::i32); Ops.append(16 - Ops.size(), Undef); } assert(Ops.size() >= 8 && Ops.size() <= 12); @@ -9697,13 +10090,13 @@ SDValue SITargetLowering::handleD16VData(SDValue VData, SelectionDAG &DAG, // Handle v3i16 unsigned I = Elts.size() / 2; SDValue Pair = DAG.getBuildVector(MVT::v2i16, DL, - {Elts[I * 2], DAG.getUNDEF(MVT::i16)}); + {Elts[I * 2], DAG.getPOISON(MVT::i16)}); SDValue IntPair = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Pair); PackedElts.push_back(IntPair); } // Pad using UNDEF - PackedElts.resize(Elts.size(), DAG.getUNDEF(MVT::i32)); + PackedElts.resize(Elts.size(), DAG.getPOISON(MVT::i32)); // Build final vector EVT VecVT = @@ -9738,10 +10131,9 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, switch (IntrinsicID) { case Intrinsic::amdgcn_exp_compr: { if (!Subtarget->hasCompressedExport()) { - DiagnosticInfoUnsupported BadIntrin( + DAG.getContext()->diagnose(DiagnosticInfoUnsupported( DAG.getMachineFunction().getFunction(), - "intrinsic not supported on subtarget", DL.getDebugLoc()); - DAG.getContext()->diagnose(BadIntrin); + "intrinsic not supported on subtarget", DL.getDebugLoc())); } SDValue Src0 = Op.getOperand(4); SDValue Src1 = Op.getOperand(5); @@ -9750,7 +10142,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, return SDValue(); const ConstantSDNode *Done = cast<ConstantSDNode>(Op.getOperand(6)); - SDValue Undef = DAG.getUNDEF(MVT::f32); + SDValue Undef = DAG.getPOISON(MVT::f32); const SDValue Ops[] = { Op.getOperand(2), // tgt DAG.getNode(ISD::BITCAST, DL, MVT::f32, Src0), // src0 @@ -9963,7 +10355,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, case Intrinsic::amdgcn_raw_ptr_buffer_load_lds: case Intrinsic::amdgcn_struct_buffer_load_lds: case Intrinsic::amdgcn_struct_ptr_buffer_load_lds: { - assert(!AMDGPU::isGFX12Plus(*Subtarget)); + if (!Subtarget->hasVMemToLDSLoad()) + return SDValue(); unsigned Opc; bool HasVIndex = IntrinsicID == Intrinsic::amdgcn_struct_buffer_load_lds || @@ -10069,7 +10462,14 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, return SDValue(Load, 0); } + // Buffers are handled by LowerBufferFatPointers, and we're going to go + // for "trust me" that the remaining cases are global pointers until + // such time as we can put two mem operands on an intrinsic. + case Intrinsic::amdgcn_load_to_lds: case Intrinsic::amdgcn_global_load_lds: { + if (!Subtarget->hasVMemToLDSLoad()) + return SDValue(); + unsigned Opc; unsigned Size = Op->getConstantOperandVal(4); switch (Size) { @@ -10096,7 +10496,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, break; } - auto *M = cast<MemSDNode>(Op); SDValue M0Val = copyToM0(DAG, Chain, DL, Op.getOperand(3)); SmallVector<SDValue, 6> Ops; @@ -10136,6 +10535,7 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, Ops.push_back(M0Val.getValue(0)); // Chain Ops.push_back(M0Val.getValue(1)); // Glue + auto *M = cast<MemSDNode>(Op); MachineMemOperand *LoadMMO = M->getMemOperand(); MachinePointerInfo LoadPtrI = LoadMMO->getPointerInfo(); LoadPtrI.Offset = Op->getConstantOperandVal(5); @@ -10162,7 +10562,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other, Op->getOperand(2), Chain), 0); - case Intrinsic::amdgcn_s_barrier_init: case Intrinsic::amdgcn_s_barrier_signal_var: { // these two intrinsics have two operands: barrier pointer and member count SDValue Chain = Op->getOperand(0); @@ -10170,9 +10569,6 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, SDValue BarOp = Op->getOperand(2); SDValue CntOp = Op->getOperand(3); SDValue M0Val; - unsigned Opc = IntrinsicID == Intrinsic::amdgcn_s_barrier_init - ? AMDGPU::S_BARRIER_INIT_M0 - : AMDGPU::S_BARRIER_SIGNAL_M0; // extract the BarrierID from bits 4-9 of BarOp SDValue BarID; BarID = DAG.getNode(ISD::SRL, DL, MVT::i32, BarOp, @@ -10196,40 +10592,8 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0)); - auto *NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops); - return SDValue(NewMI, 0); - } - case Intrinsic::amdgcn_s_barrier_join: { - // these three intrinsics have one operand: barrier pointer - SDValue Chain = Op->getOperand(0); - SmallVector<SDValue, 2> Ops; - SDValue BarOp = Op->getOperand(2); - unsigned Opc; - - if (isa<ConstantSDNode>(BarOp)) { - uint64_t BarVal = cast<ConstantSDNode>(BarOp)->getZExtValue(); - Opc = AMDGPU::S_BARRIER_JOIN_IMM; - - // extract the BarrierID from bits 4-9 of the immediate - unsigned BarID = (BarVal >> 4) & 0x3F; - SDValue K = DAG.getTargetConstant(BarID, DL, MVT::i32); - Ops.push_back(K); - Ops.push_back(Chain); - } else { - Opc = AMDGPU::S_BARRIER_JOIN_M0; - - // extract the BarrierID from bits 4-9 of BarOp, copy to M0[5:0] - SDValue M0Val; - M0Val = DAG.getNode(ISD::SRL, DL, MVT::i32, BarOp, - DAG.getShiftAmountConstant(4, MVT::i32, DL)); - M0Val = - SDValue(DAG.getMachineNode(AMDGPU::S_AND_B32, DL, MVT::i32, M0Val, - DAG.getTargetConstant(0x3F, DL, MVT::i32)), - 0); - Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0)); - } - - auto *NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops); + auto *NewMI = DAG.getMachineNode(AMDGPU::S_BARRIER_SIGNAL_M0, DL, + Op->getVTList(), Ops); return SDValue(NewMI, 0); } case Intrinsic::amdgcn_s_prefetch_data: { @@ -10260,6 +10624,11 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op, } } +bool SITargetLowering::shouldPreservePtrArith(const Function &F, + EVT PtrVT) const { + return UseSelectionDAGPTRADD && PtrVT == MVT::i64; +} + // The raw.(t)buffer and struct.(t)buffer intrinsics have two offset args: // offset (the offset that is included in bounds checking and swizzling, to be // split between the instruction's voffset and immoffset fields) and soffset @@ -10626,7 +10995,7 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const { (AS == AMDGPUAS::GLOBAL_ADDRESS && Subtarget->getScalarizeGlobalBehavior() && Load->isSimple() && isMemOpHasNoClobberedMemOperand(Load))) { - if ((!Op->isDivergent() || AMDGPUInstrInfo::isUniformMMO(MMO)) && + if ((!Op->isDivergent() || AMDGPU::isUniformMMO(MMO)) && Alignment >= Align(4) && NumElements < 32) { if (MemVT.isPow2VectorType() || (Subtarget->hasScalarDwordx3Loads() && NumElements == 3)) @@ -11057,9 +11426,10 @@ SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const { const SDValue DisableDenormValue = getSPDenormModeValue( FP_DENORM_FLUSH_IN_FLUSH_OUT, DAG, Info, Subtarget); + SDVTList BindParamVTs = DAG.getVTList(MVT::Other, MVT::Glue); DisableDenorm = - DAG.getNode(AMDGPUISD::DENORM_MODE, SL, MVT::Other, Fma4.getValue(1), - DisableDenormValue, Fma4.getValue(2)) + DAG.getNode(AMDGPUISD::DENORM_MODE, SL, BindParamVTs, + Fma4.getValue(1), DisableDenormValue, Fma4.getValue(2)) .getNode(); } else { assert(HasDynamicDenormals == (bool)SavedDenormMode); @@ -11568,29 +11938,64 @@ SDValue SITargetLowering::performFCopySignCombine(SDNode *N, DAGCombinerInfo &DCI) const { SDValue MagnitudeOp = N->getOperand(0); SDValue SignOp = N->getOperand(1); + + // The generic combine for fcopysign + fp cast is too conservative with + // vectors, and also gets confused by the splitting we will perform here, so + // peek through FP casts. + if (SignOp.getOpcode() == ISD::FP_EXTEND || + SignOp.getOpcode() == ISD::FP_ROUND) + SignOp = SignOp.getOperand(0); + SelectionDAG &DAG = DCI.DAG; SDLoc DL(N); + EVT SignVT = SignOp.getValueType(); // f64 fcopysign is really an f32 copysign on the high bits, so replace the // lower half with a copy. // fcopysign f64:x, _:y -> x.lo32, (fcopysign (f32 x.hi32), _:y) - if (MagnitudeOp.getValueType() == MVT::f64) { - SDValue MagAsVector = - DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, MagnitudeOp); - SDValue MagLo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, - MagAsVector, DAG.getConstant(0, DL, MVT::i32)); - SDValue MagHi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, - MagAsVector, DAG.getConstant(1, DL, MVT::i32)); + EVT MagVT = MagnitudeOp.getValueType(); + + unsigned NumElts = MagVT.isVector() ? MagVT.getVectorNumElements() : 1; - SDValue HiOp = DAG.getNode(ISD::FCOPYSIGN, DL, MVT::f32, MagHi, SignOp); + if (MagVT.getScalarType() == MVT::f64) { + EVT F32VT = MagVT.isVector() + ? EVT::getVectorVT(*DAG.getContext(), MVT::f32, 2 * NumElts) + : MVT::v2f32; - SDValue Vector = - DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f32, MagLo, HiOp); + SDValue MagAsVector = DAG.getNode(ISD::BITCAST, DL, F32VT, MagnitudeOp); - return DAG.getNode(ISD::BITCAST, DL, MVT::f64, Vector); + SmallVector<SDValue, 8> NewElts; + for (unsigned I = 0; I != NumElts; ++I) { + SDValue MagLo = + DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, MagAsVector, + DAG.getConstant(2 * I, DL, MVT::i32)); + SDValue MagHi = + DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, MagAsVector, + DAG.getConstant(2 * I + 1, DL, MVT::i32)); + + SDValue SignOpElt = + MagVT.isVector() + ? DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SignVT.getScalarType(), + SignOp, DAG.getConstant(I, DL, MVT::i32)) + : SignOp; + + SDValue HiOp = + DAG.getNode(ISD::FCOPYSIGN, DL, MVT::f32, MagHi, SignOpElt); + + SDValue Vector = + DAG.getNode(ISD::BUILD_VECTOR, DL, MVT::v2f32, MagLo, HiOp); + + SDValue NewElt = DAG.getNode(ISD::BITCAST, DL, MVT::f64, Vector); + NewElts.push_back(NewElt); + } + + if (NewElts.size() == 1) + return NewElts[0]; + + return DAG.getNode(ISD::BUILD_VECTOR, DL, MagVT, NewElts); } - if (SignOp.getValueType() != MVT::f64) + if (SignVT.getScalarType() != MVT::f64) return SDValue(); // Reduce width of sign operand, we only need the highest bit. @@ -11598,13 +12003,31 @@ SDValue SITargetLowering::performFCopySignCombine(SDNode *N, // fcopysign f64:x, f64:y -> // fcopysign f64:x, (extract_vector_elt (bitcast f64:y to v2f32), 1) // TODO: In some cases it might make sense to go all the way to f16. - SDValue SignAsVector = DAG.getNode(ISD::BITCAST, DL, MVT::v2f32, SignOp); - SDValue SignAsF32 = - DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, SignAsVector, - DAG.getConstant(1, DL, MVT::i32)); + + EVT F32VT = MagVT.isVector() + ? EVT::getVectorVT(*DAG.getContext(), MVT::f32, 2 * NumElts) + : MVT::v2f32; + + SDValue SignAsVector = DAG.getNode(ISD::BITCAST, DL, F32VT, SignOp); + + SmallVector<SDValue, 8> F32Signs; + for (unsigned I = 0; I != NumElts; ++I) { + // Take sign from odd elements of cast vector + SDValue SignAsF32 = + DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32, SignAsVector, + DAG.getConstant(2 * I + 1, DL, MVT::i32)); + F32Signs.push_back(SignAsF32); + } + + SDValue NewSign = + NumElts == 1 + ? F32Signs.back() + : DAG.getNode(ISD::BUILD_VECTOR, DL, + EVT::getVectorVT(*DAG.getContext(), MVT::f32, NumElts), + F32Signs); return DAG.getNode(ISD::FCOPYSIGN, DL, N->getValueType(0), N->getOperand(0), - SignAsF32); + NewSign); } // (shl (add x, c1), c2) -> add (shl x, c2), (shl c1, c2) @@ -11691,7 +12114,6 @@ static unsigned getBasePtrIndex(const MemSDNode *N) { SDValue SITargetLowering::performMemSDNodeCombine(MemSDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; - SDLoc SL(N); unsigned PtrIdx = getBasePtrIndex(N); SDValue Ptr = N->getOperand(PtrIdx); @@ -11733,6 +12155,11 @@ SDValue SITargetLowering::splitBinaryBitConstantOp( if ((bitOpWithConstantIsReducible(Opc, ValLo) || bitOpWithConstantIsReducible(Opc, ValHi)) || (CRHS->hasOneUse() && !TII->isInlineConstant(CRHS->getAPIntValue()))) { + // We have 64-bit scalar and/or/xor, but do not have vector forms. + if (Subtarget->has64BitLiterals() && CRHS->hasOneUse() && + !CRHS->user_begin()->isDivergent()) + return SDValue(); + // If we need to materialize a 64-bit immediate, it will be split up later // anyway. Avoid creating the harder to understand 64-bit immediate // materialization. @@ -11749,12 +12176,32 @@ bool llvm::isBoolSGPR(SDValue V) { default: break; case ISD::SETCC: + case ISD::IS_FPCLASS: case AMDGPUISD::FP_CLASS: return true; case ISD::AND: case ISD::OR: case ISD::XOR: return isBoolSGPR(V.getOperand(0)) && isBoolSGPR(V.getOperand(1)); + case ISD::SADDO: + case ISD::UADDO: + case ISD::SSUBO: + case ISD::USUBO: + case ISD::SMULO: + case ISD::UMULO: + return V.getResNo() == 1; + case ISD::INTRINSIC_WO_CHAIN: { + unsigned IntrinsicID = V.getConstantOperandVal(0); + switch (IntrinsicID) { + case Intrinsic::amdgcn_is_shared: + case Intrinsic::amdgcn_is_private: + return true; + default: + return false; + } + + return false; + } } return false; } @@ -13008,6 +13455,8 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op, case ISD::FMAXNUM_IEEE: case ISD::FMINIMUM: case ISD::FMAXIMUM: + case ISD::FMINIMUMNUM: + case ISD::FMAXIMUMNUM: case AMDGPUISD::CLAMP: case AMDGPUISD::FMED3: case AMDGPUISD::FMAX3: @@ -13172,7 +13621,9 @@ bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF, case AMDGPU::G_FMINNUM_IEEE: case AMDGPU::G_FMAXNUM_IEEE: case AMDGPU::G_FMINIMUM: - case AMDGPU::G_FMAXIMUM: { + case AMDGPU::G_FMAXIMUM: + case AMDGPU::G_FMINIMUMNUM: + case AMDGPU::G_FMAXIMUMNUM: { if (Subtarget->supportsMinMaxDenormModes() || // FIXME: denormalsEnabledForType is broken for dynamic denormalsEnabledForType(MRI.getType(Reg), MF)) @@ -13214,6 +13665,7 @@ bool SITargetLowering::isCanonicalized(Register Reg, const MachineFunction &MF, case Intrinsic::amdgcn_frexp_mant: case Intrinsic::amdgcn_fdot2: case Intrinsic::amdgcn_trig_preop: + case Intrinsic::amdgcn_tanh: return true; default: break; @@ -13342,6 +13794,7 @@ static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) { switch (Opc) { case ISD::FMAXNUM: case ISD::FMAXNUM_IEEE: + case ISD::FMAXIMUMNUM: return AMDGPUISD::FMAX3; case ISD::FMAXIMUM: return AMDGPUISD::FMAXIMUM3; @@ -13351,6 +13804,7 @@ static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) { return AMDGPUISD::UMAX3; case ISD::FMINNUM: case ISD::FMINNUM_IEEE: + case ISD::FMINIMUMNUM: return AMDGPUISD::FMIN3; case ISD::FMINIMUM: return AMDGPUISD::FMINIMUM3; @@ -13428,10 +13882,34 @@ SDValue SITargetLowering::performFPMed3ImmCombine(SelectionDAG &DAG, if (K0->getValueAPF() > K1->getValueAPF()) return SDValue(); + // med3 with a nan input acts like + // v_min_f32(v_min_f32(S0.f32, S1.f32), S2.f32) + // + // So the result depends on whether the IEEE mode bit is enabled or not with a + // signaling nan input. + // ieee=1 + // s0 snan: yields s2 + // s1 snan: yields s2 + // s2 snan: qnan + + // s0 qnan: min(s1, s2) + // s1 qnan: min(s0, s2) + // s2 qnan: min(s0, s1) + + // ieee=0 + // s0 snan: min(s1, s2) + // s1 snan: min(s0, s2) + // s2 snan: qnan + + // s0 qnan: min(s1, s2) + // s1 qnan: min(s0, s2) + // s2 qnan: min(s0, s1) const MachineFunction &MF = DAG.getMachineFunction(); const SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>(); - // TODO: Check IEEE bit enabled? + // TODO: Check IEEE bit enabled. We can form fmed3 with IEEE=0 regardless of + // whether the input is a signaling nan if op0 is fmaximum or fmaximumnum. We + // can only form if op0 is fmaxnum_ieee if IEEE=1. EVT VT = Op0.getValueType(); if (Info->getMode().DX10Clamp) { // If dx10_clamp is enabled, NaNs clamp to 0.0. This is the same as the @@ -13472,13 +13950,16 @@ static bool supportsMin3Max3(const GCNSubtarget &Subtarget, unsigned Opc, case ISD::FMAXNUM: case ISD::FMINNUM_IEEE: case ISD::FMAXNUM_IEEE: + case ISD::FMINIMUMNUM: + case ISD::FMAXIMUMNUM: case AMDGPUISD::FMIN_LEGACY: case AMDGPUISD::FMAX_LEGACY: return (VT == MVT::f32) || (VT == MVT::f16 && Subtarget.hasMin3Max3_16()); case ISD::FMINIMUM: case ISD::FMAXIMUM: return (VT == MVT::f32 && Subtarget.hasMinimum3Maximum3F32()) || - (VT == MVT::f16 && Subtarget.hasMinimum3Maximum3F16()); + (VT == MVT::f16 && Subtarget.hasMinimum3Maximum3F16()) || + (VT == MVT::v2f16 && Subtarget.hasMinimum3Maximum3PKF16()); case ISD::SMAX: case ISD::SMIN: case ISD::UMAX: @@ -13546,9 +14027,14 @@ SDValue SITargetLowering::performMinMaxCombine(SDNode *N, return Med3; } - // fminnum(fmaxnum(x, K0), K1), K0 < K1 && !is_snan(x) -> fmed3(x, K0, K1) + // if !is_snan(x): + // fminnum(fmaxnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1) + // fminnum_ieee(fmaxnum_ieee(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1) + // fminnumnum(fmaxnumnum(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1) + // fmin_legacy(fmax_legacy(x, K0), K1), K0 < K1 -> fmed3(x, K0, K1) if (((Opc == ISD::FMINNUM && Op0.getOpcode() == ISD::FMAXNUM) || (Opc == ISD::FMINNUM_IEEE && Op0.getOpcode() == ISD::FMAXNUM_IEEE) || + (Opc == ISD::FMINIMUMNUM && Op0.getOpcode() == ISD::FMAXIMUMNUM) || (Opc == AMDGPUISD::FMIN_LEGACY && Op0.getOpcode() == AMDGPUISD::FMAX_LEGACY)) && (VT == MVT::f32 || VT == MVT::f64 || @@ -13559,6 +14045,17 @@ SDValue SITargetLowering::performMinMaxCombine(SDNode *N, return Res; } + // Prefer fminnum_ieee over fminimum. For gfx950, minimum/maximum are legal + // for some types, but at a higher cost since it's implemented with a 3 + // operand form. + const SDNodeFlags Flags = N->getFlags(); + if ((Opc == ISD::FMINIMUM || Opc == ISD::FMAXIMUM) && + !Subtarget->hasIEEEMinimumMaximumInsts() && Flags.hasNoNaNs()) { + unsigned NewOpc = + Opc == ISD::FMINIMUM ? ISD::FMINNUM_IEEE : ISD::FMAXNUM_IEEE; + return DAG.getNode(NewOpc, SDLoc(N), VT, Op0, Op1, Flags); + } + return SDValue(); } @@ -14623,6 +15120,49 @@ SDValue SITargetLowering::performAddCombine(SDNode *N, return SDValue(); } +SDValue SITargetLowering::performPtrAddCombine(SDNode *N, + DAGCombinerInfo &DCI) const { + SelectionDAG &DAG = DCI.DAG; + SDLoc DL(N); + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + if (N1.getOpcode() == ISD::ADD) { + // (ptradd x, (add y, z)) -> (ptradd (ptradd x, y), z) if z is a constant, + // y is not, and (add y, z) is used only once. + // (ptradd x, (add y, z)) -> (ptradd (ptradd x, z), y) if y is a constant, + // z is not, and (add y, z) is used only once. + // The goal is to move constant offsets to the outermost ptradd, to create + // more opportunities to fold offsets into memory instructions. + // Together with the generic combines in DAGCombiner.cpp, this also + // implements (ptradd (ptradd x, y), z) -> (ptradd (ptradd x, z), y)). + // + // This transform is here instead of in the general DAGCombiner as it can + // turn in-bounds pointer arithmetic out-of-bounds, which is problematic for + // AArch64's CPA. + SDValue X = N0; + SDValue Y = N1.getOperand(0); + SDValue Z = N1.getOperand(1); + if (N1.hasOneUse()) { + bool YIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Y); + bool ZIsConstant = DAG.isConstantIntBuildVectorOrConstantInt(Z); + if (ZIsConstant != YIsConstant) { + // If both additions in the original were NUW, the new ones are as well. + SDNodeFlags Flags = + (N->getFlags() & N1->getFlags()) & SDNodeFlags::NoUnsignedWrap; + if (YIsConstant) + std::swap(Y, Z); + + SDValue Inner = DAG.getMemBasePlusOffset(X, Y, DL, Flags); + DCI.AddToWorklist(Inner.getNode()); + return DAG.getMemBasePlusOffset(Inner, Z, DL, Flags); + } + } + } + + return SDValue(); +} + SDValue SITargetLowering::performSubCombine(SDNode *N, DAGCombinerInfo &DCI) const { SelectionDAG &DAG = DCI.DAG; @@ -14827,6 +15367,12 @@ SDValue SITargetLowering::performFMulCombine(SDNode *N, EVT ScalarVT = VT.getScalarType(); EVT IntVT = VT.changeElementType(MVT::i32); + if (!N->isDivergent() && getSubtarget()->hasSALUFloatInsts() && + (ScalarVT == MVT::f32 || ScalarVT == MVT::f16)) { + // Prefer to use s_mul_f16/f32 instead of v_ldexp_f16/f32. + return SDValue(); + } + SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); @@ -15155,6 +15701,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N, switch (N->getOpcode()) { case ISD::ADD: return performAddCombine(N, DCI); + case ISD::PTRADD: + return performPtrAddCombine(N, DCI); case ISD::SUB: return performSubCombine(N, DCI); case ISD::UADDO_CARRY: @@ -15176,6 +15724,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N, case ISD::FMINNUM_IEEE: case ISD::FMAXIMUM: case ISD::FMINIMUM: + case ISD::FMAXIMUMNUM: + case ISD::FMINIMUMNUM: case ISD::SMAX: case ISD::SMIN: case ISD::UMAX: @@ -15315,10 +15865,8 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node, unsigned NewDmask = 0; unsigned TFEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::tfe) - 1; unsigned LWEIdx = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::lwe) - 1; - bool UsesTFC = ((int(TFEIdx) >= 0 && Node->getConstantOperandVal(TFEIdx)) || - (int(LWEIdx) >= 0 && Node->getConstantOperandVal(LWEIdx))) - ? true - : false; + bool UsesTFC = (int(TFEIdx) >= 0 && Node->getConstantOperandVal(TFEIdx)) || + (int(LWEIdx) >= 0 && Node->getConstantOperandVal(LWEIdx)); unsigned TFCLane = 0; bool HasChain = Node->getNumValues() > 1; @@ -15408,9 +15956,9 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node, // Adjust the writemask in the node SmallVector<SDValue, 12> Ops; - Ops.insert(Ops.end(), Node->op_begin(), Node->op_begin() + DmaskIdx); + llvm::append_range(Ops, Node->ops().take_front(DmaskIdx)); Ops.push_back(DAG.getTargetConstant(NewDmask, SDLoc(Node), MVT::i32)); - Ops.insert(Ops.end(), Node->op_begin() + DmaskIdx + 1, Node->op_end()); + llvm::append_range(Ops, Node->ops().drop_front(DmaskIdx + 1)); MVT SVT = Node->getValueType(0).getVectorElementType().getSimpleVT(); @@ -15894,7 +16442,8 @@ SITargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI_, case 'v': switch (BitWidth) { case 16: - RC = &AMDGPU::VGPR_32RegClass; + RC = Subtarget->useRealTrue16Insts() ? &AMDGPU::VGPR_16RegClass + : &AMDGPU::VGPR_32RegClass; break; default: RC = TRI->getVGPRClassForBitWidth(BitWidth); @@ -16296,16 +16845,18 @@ void SITargetLowering::computeKnownBitsForFrameIndex( Known.Zero.setHighBits(getSubtarget()->getKnownHighZeroBitsForFrameIndex()); } -static void knownBitsForWorkitemID(const GCNSubtarget &ST, GISelKnownBits &KB, - KnownBits &Known, unsigned Dim) { +static void knownBitsForWorkitemID(const GCNSubtarget &ST, + GISelValueTracking &VT, KnownBits &Known, + unsigned Dim) { unsigned MaxValue = - ST.getMaxWorkitemID(KB.getMachineFunction().getFunction(), Dim); + ST.getMaxWorkitemID(VT.getMachineFunction().getFunction(), Dim); Known.Zero.setHighBits(llvm::countl_zero(MaxValue)); } void SITargetLowering::computeKnownBitsForTargetInstr( - GISelKnownBits &KB, Register R, KnownBits &Known, const APInt &DemandedElts, - const MachineRegisterInfo &MRI, unsigned Depth) const { + GISelValueTracking &VT, Register R, KnownBits &Known, + const APInt &DemandedElts, const MachineRegisterInfo &MRI, + unsigned Depth) const { const MachineInstr *MI = MRI.getVRegDef(R); switch (MI->getOpcode()) { case AMDGPU::G_INTRINSIC: @@ -16313,13 +16864,13 @@ void SITargetLowering::computeKnownBitsForTargetInstr( Intrinsic::ID IID = cast<GIntrinsic>(MI)->getIntrinsicID(); switch (IID) { case Intrinsic::amdgcn_workitem_id_x: - knownBitsForWorkitemID(*getSubtarget(), KB, Known, 0); + knownBitsForWorkitemID(*getSubtarget(), VT, Known, 0); break; case Intrinsic::amdgcn_workitem_id_y: - knownBitsForWorkitemID(*getSubtarget(), KB, Known, 1); + knownBitsForWorkitemID(*getSubtarget(), VT, Known, 1); break; case Intrinsic::amdgcn_workitem_id_z: - knownBitsForWorkitemID(*getSubtarget(), KB, Known, 2); + knownBitsForWorkitemID(*getSubtarget(), VT, Known, 2); break; case Intrinsic::amdgcn_mbcnt_lo: case Intrinsic::amdgcn_mbcnt_hi: { @@ -16329,7 +16880,7 @@ void SITargetLowering::computeKnownBitsForTargetInstr( ? getSubtarget()->getWavefrontSizeLog2() : 5); KnownBits Known2; - KB.computeKnownBitsImpl(MI->getOperand(3).getReg(), Known2, DemandedElts, + VT.computeKnownBitsImpl(MI->getOperand(3).getReg(), Known2, DemandedElts, Depth + 1); Known = KnownBits::add(Known, Known2); break; @@ -16356,17 +16907,17 @@ void SITargetLowering::computeKnownBitsForTargetInstr( auto [Dst, Src0, Src1, Src2] = MI->getFirst4Regs(); KnownBits Known2; - KB.computeKnownBitsImpl(Src2, Known2, DemandedElts, Depth + 1); + VT.computeKnownBitsImpl(Src2, Known2, DemandedElts, Depth + 1); if (Known2.isUnknown()) break; KnownBits Known1; - KB.computeKnownBitsImpl(Src1, Known1, DemandedElts, Depth + 1); + VT.computeKnownBitsImpl(Src1, Known1, DemandedElts, Depth + 1); if (Known1.isUnknown()) break; KnownBits Known0; - KB.computeKnownBitsImpl(Src0, Known0, DemandedElts, Depth + 1); + VT.computeKnownBitsImpl(Src0, Known0, DemandedElts, Depth + 1); if (Known0.isUnknown()) break; @@ -16379,15 +16930,16 @@ void SITargetLowering::computeKnownBitsForTargetInstr( } Align SITargetLowering::computeKnownAlignForTargetInstr( - GISelKnownBits &KB, Register R, const MachineRegisterInfo &MRI, + GISelValueTracking &VT, Register R, const MachineRegisterInfo &MRI, unsigned Depth) const { const MachineInstr *MI = MRI.getVRegDef(R); if (auto *GI = dyn_cast<GIntrinsic>(MI)) { // FIXME: Can this move to generic code? What about the case where the call // site specifies a lower alignment? Intrinsic::ID IID = GI->getIntrinsicID(); - LLVMContext &Ctx = KB.getMachineFunction().getFunction().getContext(); - AttributeList Attrs = Intrinsic::getAttributes(Ctx, IID); + LLVMContext &Ctx = VT.getMachineFunction().getFunction().getContext(); + AttributeList Attrs = + Intrinsic::getAttributes(Ctx, IID, Intrinsic::getType(Ctx, IID)); if (MaybeAlign RetAlign = Attrs.getRetAlignment()) return *RetAlign; } @@ -16568,6 +17120,7 @@ bool SITargetLowering::denormalsEnabledForType( } bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op, + const APInt &DemandedElts, const SelectionDAG &DAG, bool SNaN, unsigned Depth) const { @@ -16580,8 +17133,8 @@ bool SITargetLowering::isKnownNeverNaNForTargetNode(SDValue Op, return DAG.isKnownNeverNaN(Op.getOperand(0), SNaN, Depth + 1); } - return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DAG, SNaN, - Depth); + return AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(Op, DemandedElts, + DAG, SNaN, Depth); } // On older subtargets, global FP atomic instructions have a hardcoded FP mode @@ -16603,8 +17156,8 @@ static bool atomicIgnoresDenormalModeOrFPModeIsFTZ(const AtomicRMWInst *RMW) { static OptimizationRemark emitAtomicRMWLegalRemark(const AtomicRMWInst *RMW) { LLVMContext &Ctx = RMW->getContext(); - StringRef SS = Ctx.getSyncScopeName(RMW->getSyncScopeID()).value_or(""); - StringRef MemScope = SS.empty() ? StringRef("system") : SS; + StringRef MemScope = + Ctx.getSyncScopeName(RMW->getSyncScopeID()).value_or("system"); return OptimizationRemark(DEBUG_TYPE, "Passed", RMW) << "Hardware instruction generated for atomic " @@ -16820,39 +17373,39 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { // safe. The message phrasing also should be better. if (globalMemoryFPAtomicIsLegal(*Subtarget, RMW, HasSystemScope)) { if (AS == AMDGPUAS::FLAT_ADDRESS) { - // gfx940, gfx12 + // gfx942, gfx12 if (Subtarget->hasAtomicFlatPkAdd16Insts() && isV2F16OrV2BF16(Ty)) return ReportUnsafeHWInst(AtomicExpansionKind::None); } else if (AMDGPU::isExtendedGlobalAddrSpace(AS)) { - // gfx90a, gfx940, gfx12 + // gfx90a, gfx942, gfx12 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty)) return ReportUnsafeHWInst(AtomicExpansionKind::None); - // gfx940, gfx12 + // gfx942, gfx12 if (Subtarget->hasAtomicGlobalPkAddBF16Inst() && isV2BF16(Ty)) return ReportUnsafeHWInst(AtomicExpansionKind::None); } else if (AS == AMDGPUAS::BUFFER_FAT_POINTER) { - // gfx90a, gfx940, gfx12 + // gfx90a, gfx942, gfx12 if (Subtarget->hasAtomicBufferGlobalPkAddF16Insts() && isV2F16(Ty)) return ReportUnsafeHWInst(AtomicExpansionKind::None); - // While gfx90a/gfx940 supports v2bf16 for global/flat, it does not for + // While gfx90a/gfx942 supports v2bf16 for global/flat, it does not for // buffer. gfx12 does have the buffer version. if (Subtarget->hasAtomicBufferPkAddBF16Inst() && isV2BF16(Ty)) return ReportUnsafeHWInst(AtomicExpansionKind::None); } - // global and flat atomic fadd f64: gfx90a, gfx940. + // global and flat atomic fadd f64: gfx90a, gfx942. if (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && Ty->isDoubleTy()) return ReportUnsafeHWInst(AtomicExpansionKind::None); if (AS != AMDGPUAS::FLAT_ADDRESS) { if (Ty->isFloatTy()) { - // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx940, + // global/buffer atomic fadd f32 no-rtn: gfx908, gfx90a, gfx942, // gfx11+. if (RMW->use_empty() && Subtarget->hasAtomicFaddNoRtnInsts()) return ReportUnsafeHWInst(AtomicExpansionKind::None); - // global/buffer atomic fadd f32 rtn: gfx90a, gfx940, gfx11+. + // global/buffer atomic fadd f32 rtn: gfx90a, gfx942, gfx11+. if (!RMW->use_empty() && Subtarget->hasAtomicFaddRtnInsts()) return ReportUnsafeHWInst(AtomicExpansionKind::None); } else { @@ -16864,7 +17417,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { } } - // flat atomic fadd f32: gfx940, gfx11+. + // flat atomic fadd f32: gfx942, gfx11+. if (AS == AMDGPUAS::FLAT_ADDRESS && Ty->isFloatTy()) { if (Subtarget->hasFlatAtomicFaddF32Inst()) return ReportUnsafeHWInst(AtomicExpansionKind::None); @@ -16903,7 +17456,7 @@ SITargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { // float, double restored in gfx10. // double removed again in gfx11, so only f32 for gfx11/gfx12. // - // For gfx9, gfx90a and gfx940 support f64 for global (same as fadd), but + // For gfx9, gfx90a and gfx942 support f64 for global (same as fadd), but // no f32. if (AS == AMDGPUAS::FLAT_ADDRESS) { if (Subtarget->hasAtomicFMinFMaxF32FlatInsts() && Ty->isFloatTy()) @@ -17115,7 +17668,7 @@ SITargetLowering::getTargetMMOFlags(const Instruction &I) const { bool SITargetLowering::checkForPhysRegDependency( SDNode *Def, SDNode *User, unsigned Op, const TargetRegisterInfo *TRI, - const TargetInstrInfo *TII, unsigned &PhysReg, int &Cost) const { + const TargetInstrInfo *TII, MCRegister &PhysReg, int &Cost) const { if (User->getOpcode() != ISD::CopyToReg) return false; if (!Def->isMachineOpcode()) @@ -17138,33 +17691,6 @@ bool SITargetLowering::checkForPhysRegDependency( return false; } -/// Check if it is profitable to hoist instruction in then/else to if. -bool SITargetLowering::isProfitableToHoist(Instruction *I) const { - if (!I->hasOneUse()) - return true; - - Instruction *User = I->user_back(); - // TODO: Add more patterns that are not profitable to hoist and - // handle modifiers such as fabs and fneg - switch (I->getOpcode()) { - case Instruction::FMul: { - if (User->getOpcode() != Instruction::FSub && - User->getOpcode() != Instruction::FAdd) - return true; - - const TargetOptions &Options = getTargetMachine().Options; - - return ((!I->hasAllowContract() || !User->hasAllowContract()) && - Options.AllowFPOpFusion != FPOpFusion::Fast && - !Options.UnsafeFPMath) || - !isFMAFasterThanFMulAndFAdd(*I->getFunction(), User->getType()); - } - default: - return true; - } - return true; -} - void SITargetLowering::emitExpandAtomicAddrSpacePredicate( Instruction *AI) const { // Given: atomicrmw fadd ptr %addr, float %val ordering @@ -17237,9 +17763,11 @@ void SITargetLowering::emitExpandAtomicAddrSpacePredicate( // where we only insert a check for private and still use the flat instruction // for global and shared. - bool FullFlatEmulation = RMW && RMW->getOperation() == AtomicRMWInst::FAdd && - Subtarget->hasAtomicFaddInsts() && - RMW->getType()->isFloatTy(); + bool FullFlatEmulation = + RMW && RMW->getOperation() == AtomicRMWInst::FAdd && + ((Subtarget->hasAtomicFaddInsts() && RMW->getType()->isFloatTy()) || + (Subtarget->hasFlatBufferGlobalAtomicFaddF64Inst() && + RMW->getType()->isDoubleTy())); // If the return value isn't used, do not introduce a false use in the phi. bool ReturnValueIsUsed = !AI->use_empty(); @@ -17267,8 +17795,8 @@ void SITargetLowering::emitExpandAtomicAddrSpacePredicate( Value *LoadedShared = nullptr; if (FullFlatEmulation) { - CallInst *IsShared = Builder.CreateIntrinsic( - Intrinsic::amdgcn_is_shared, {}, {Addr}, nullptr, "is.shared"); + CallInst *IsShared = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_shared, + {Addr}, nullptr, "is.shared"); Builder.CreateCondBr(IsShared, SharedBB, CheckPrivateBB); Builder.SetInsertPoint(SharedBB); Value *CastToLocal = Builder.CreateAddrSpaceCast( @@ -17283,8 +17811,8 @@ void SITargetLowering::emitExpandAtomicAddrSpacePredicate( Builder.SetInsertPoint(CheckPrivateBB); } - CallInst *IsPrivate = Builder.CreateIntrinsic( - Intrinsic::amdgcn_is_private, {}, {Addr}, nullptr, "is.private"); + CallInst *IsPrivate = Builder.CreateIntrinsic(Intrinsic::amdgcn_is_private, + {Addr}, nullptr, "is.private"); Builder.CreateCondBr(IsPrivate, PrivateBB, GlobalBB); Builder.SetInsertPoint(PrivateBB); |