diff options
Diffstat (limited to 'llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp')
| -rw-r--r-- | llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 2143 |
1 files changed, 1479 insertions, 664 deletions
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index aae2fdaf5bec..07baf29ce701 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -12,12 +12,12 @@ #include "MCTargetDesc/AArch64AddressingModes.h" #include "Utils/AArch64SMEAttributes.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/Analysis/IVDescriptors.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/CodeGen/BasicTTIImpl.h" #include "llvm/CodeGen/CostTable.h" #include "llvm/CodeGen/TargetLowering.h" +#include "llvm/IR/DerivedTypes.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/IntrinsicsAArch64.h" @@ -185,7 +185,7 @@ public: TailFoldingOption TailFoldingOptionLoc; -cl::opt<TailFoldingOption, true, cl::parser<std::string>> SVETailFolding( +static cl::opt<TailFoldingOption, true, cl::parser<std::string>> SVETailFolding( "sve-tail-folding", cl::desc( "Control the use of vectorisation using tail-folding for SVE where the" @@ -262,29 +262,44 @@ bool AArch64TTIImpl::isMultiversionedFunction(const Function &F) const { return F.hasFnAttribute("fmv-features"); } +const FeatureBitset AArch64TTIImpl::InlineInverseFeatures = { + AArch64::FeatureExecuteOnly, +}; + bool AArch64TTIImpl::areInlineCompatible(const Function *Caller, const Function *Callee) const { - SMEAttrs CallerAttrs(*Caller), CalleeAttrs(*Callee); + SMECallAttrs CallAttrs(*Caller, *Callee); // When inlining, we should consider the body of the function, not the // interface. - if (CalleeAttrs.hasStreamingBody()) { - CalleeAttrs.set(SMEAttrs::SM_Compatible, false); - CalleeAttrs.set(SMEAttrs::SM_Enabled, true); + if (CallAttrs.callee().hasStreamingBody()) { + CallAttrs.callee().set(SMEAttrs::SM_Compatible, false); + CallAttrs.callee().set(SMEAttrs::SM_Enabled, true); } - if (CalleeAttrs.isNewZA() || CalleeAttrs.isNewZT0()) + if (CallAttrs.callee().isNewZA() || CallAttrs.callee().isNewZT0()) return false; - if (CallerAttrs.requiresLazySave(CalleeAttrs) || - CallerAttrs.requiresSMChange(CalleeAttrs) || - CallerAttrs.requiresPreservingZT0(CalleeAttrs) || - CallerAttrs.requiresPreservingAllZAState(CalleeAttrs)) { + if (CallAttrs.requiresLazySave() || CallAttrs.requiresSMChange() || + CallAttrs.requiresPreservingZT0() || + CallAttrs.requiresPreservingAllZAState()) { if (hasPossibleIncompatibleOps(Callee)) return false; } - return BaseT::areInlineCompatible(Caller, Callee); + const TargetMachine &TM = getTLI()->getTargetMachine(); + const FeatureBitset &CallerBits = + TM.getSubtargetImpl(*Caller)->getFeatureBits(); + const FeatureBitset &CalleeBits = + TM.getSubtargetImpl(*Callee)->getFeatureBits(); + // Adjust the feature bitsets by inverting some of the bits. This is needed + // for target features that represent restrictions rather than capabilities, + // for example a "+execute-only" callee can be inlined into a caller without + // "+execute-only", but not vice versa. + FeatureBitset EffectiveCallerBits = CallerBits ^ InlineInverseFeatures; + FeatureBitset EffectiveCalleeBits = CalleeBits ^ InlineInverseFeatures; + + return (EffectiveCallerBits & EffectiveCalleeBits) == EffectiveCalleeBits; } bool AArch64TTIImpl::areTypesABICompatible( @@ -333,12 +348,14 @@ AArch64TTIImpl::getInlineCallPenalty(const Function *F, const CallBase &Call, // streaming-mode change, and the call to G from F would also require a // streaming-mode change, then there is benefit to do the streaming-mode // change only once and avoid inlining of G into F. + SMEAttrs FAttrs(*F); - SMEAttrs CalleeAttrs(Call); - if (FAttrs.requiresSMChange(CalleeAttrs)) { + SMECallAttrs CallAttrs(Call); + + if (SMECallAttrs(FAttrs, CallAttrs.callee()).requiresSMChange()) { if (F == Call.getCaller()) // (1) return CallPenaltyChangeSM * DefaultCallPenalty; - if (FAttrs.requiresSMChange(SMEAttrs(*Call.getCaller()))) // (2) + if (SMECallAttrs(FAttrs, CallAttrs.caller()).requiresSMChange()) // (2) return InlineCallPenaltyChangeSM * DefaultCallPenalty; } @@ -355,7 +372,7 @@ bool AArch64TTIImpl::shouldMaximizeVectorBandwidth( /// Calculate the cost of materializing a 64-bit value. This helper /// method might only calculate a fraction of a larger immediate. Therefore it /// is valid to return a cost of ZERO. -InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) { +InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) const { // Check if the immediate can be encoded within an instruction. if (Val == 0 || AArch64_AM::isLogicalImmediate(Val, 64)) return 0; @@ -370,8 +387,9 @@ InstructionCost AArch64TTIImpl::getIntImmCost(int64_t Val) { } /// Calculate the cost of materializing the given constant. -InstructionCost AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty, - TTI::TargetCostKind CostKind) { +InstructionCost +AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty, + TTI::TargetCostKind CostKind) const { assert(Ty->isIntegerTy()); unsigned BitSize = Ty->getPrimitiveSizeInBits(); @@ -398,7 +416,7 @@ InstructionCost AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty, InstructionCost AArch64TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, const APInt &Imm, Type *Ty, TTI::TargetCostKind CostKind, - Instruction *Inst) { + Instruction *Inst) const { assert(Ty->isIntegerTy()); unsigned BitSize = Ty->getPrimitiveSizeInBits(); @@ -466,7 +484,7 @@ InstructionCost AArch64TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx, InstructionCost AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, const APInt &Imm, Type *Ty, - TTI::TargetCostKind CostKind) { + TTI::TargetCostKind CostKind) const { assert(Ty->isIntegerTy()); unsigned BitSize = Ty->getPrimitiveSizeInBits(); @@ -516,7 +534,7 @@ AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, } TargetTransformInfo::PopcntSupportKind -AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) { +AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) const { assert(isPowerOf2_32(TyWidth) && "Ty width must be power of 2"); if (TyWidth == 32 || TyWidth == 64) return TTI::PSK_FastHardware; @@ -561,7 +579,7 @@ static InstructionCost getHistogramCost(const IntrinsicCostAttributes &ICA) { InstructionCost AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, - TTI::TargetCostKind CostKind) { + TTI::TargetCostKind CostKind) const { // The code-generator is currently not able to handle scalable vectors // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting // it. This change will be removed when code-generation for these types is @@ -868,12 +886,11 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, const auto LegalisationCost = getTypeLegalizationCost(RetTy); if (OpInfoZ.isUniform()) { - // FIXME: The costs could be lower if the codegen is better. static const CostTblEntry FshlTbl[] = { - {Intrinsic::fshl, MVT::v4i32, 3}, // ushr + shl + orr - {Intrinsic::fshl, MVT::v2i64, 3}, {Intrinsic::fshl, MVT::v16i8, 4}, - {Intrinsic::fshl, MVT::v8i16, 4}, {Intrinsic::fshl, MVT::v2i32, 3}, - {Intrinsic::fshl, MVT::v8i8, 4}, {Intrinsic::fshl, MVT::v4i16, 4}}; + {Intrinsic::fshl, MVT::v4i32, 2}, // shl + usra + {Intrinsic::fshl, MVT::v2i64, 2}, {Intrinsic::fshl, MVT::v16i8, 2}, + {Intrinsic::fshl, MVT::v8i16, 2}, {Intrinsic::fshl, MVT::v2i32, 2}, + {Intrinsic::fshl, MVT::v8i8, 2}, {Intrinsic::fshl, MVT::v4i16, 2}}; // Costs for both fshl & fshr are the same, so just pass Intrinsic::fshl // to avoid having to duplicate the costs. const auto *Entry = @@ -940,6 +957,16 @@ AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, } break; } + case Intrinsic::experimental_cttz_elts: { + EVT ArgVT = getTLI()->getValueType(DL, ICA.getArgTypes()[0]); + if (!getTLI()->shouldExpandCttzElements(ArgVT)) { + // This will consist of a SVE brkb and a cntp instruction. These + // typically have the same latency and half the throughput as a vector + // add instruction. + return 4; + } + break; + } default: break; } @@ -984,6 +1011,660 @@ static std::optional<Instruction *> processPhiNode(InstCombiner &IC, return IC.replaceInstUsesWith(II, NPN); } +// A collection of properties common to SVE intrinsics that allow for combines +// to be written without needing to know the specific intrinsic. +struct SVEIntrinsicInfo { + // + // Helper routines for common intrinsic definitions. + // + + // e.g. llvm.aarch64.sve.add pg, op1, op2 + // with IID ==> llvm.aarch64.sve.add_u + static SVEIntrinsicInfo + defaultMergingOp(Intrinsic::ID IID = Intrinsic::not_intrinsic) { + return SVEIntrinsicInfo() + .setGoverningPredicateOperandIdx(0) + .setOperandIdxInactiveLanesTakenFrom(1) + .setMatchingUndefIntrinsic(IID); + } + + // e.g. llvm.aarch64.sve.neg inactive, pg, op + static SVEIntrinsicInfo defaultMergingUnaryOp() { + return SVEIntrinsicInfo() + .setGoverningPredicateOperandIdx(1) + .setOperandIdxInactiveLanesTakenFrom(0) + .setOperandIdxWithNoActiveLanes(0); + } + + // e.g. llvm.aarch64.sve.fcvtnt inactive, pg, op + static SVEIntrinsicInfo defaultMergingUnaryNarrowingTopOp() { + return SVEIntrinsicInfo() + .setGoverningPredicateOperandIdx(1) + .setOperandIdxInactiveLanesTakenFrom(0); + } + + // e.g. llvm.aarch64.sve.add_u pg, op1, op2 + static SVEIntrinsicInfo defaultUndefOp() { + return SVEIntrinsicInfo() + .setGoverningPredicateOperandIdx(0) + .setInactiveLanesAreNotDefined(); + } + + // e.g. llvm.aarch64.sve.prf pg, ptr (GPIndex = 0) + // llvm.aarch64.sve.st1 data, pg, ptr (GPIndex = 1) + static SVEIntrinsicInfo defaultVoidOp(unsigned GPIndex) { + return SVEIntrinsicInfo() + .setGoverningPredicateOperandIdx(GPIndex) + .setInactiveLanesAreUnused(); + } + + // e.g. llvm.aarch64.sve.cmpeq pg, op1, op2 + // llvm.aarch64.sve.ld1 pg, ptr + static SVEIntrinsicInfo defaultZeroingOp() { + return SVEIntrinsicInfo() + .setGoverningPredicateOperandIdx(0) + .setInactiveLanesAreUnused() + .setResultIsZeroInitialized(); + } + + // All properties relate to predication and thus having a general predicate + // is the minimum requirement to say there is intrinsic info to act on. + explicit operator bool() const { return hasGoverningPredicate(); } + + // + // Properties relating to the governing predicate. + // + + bool hasGoverningPredicate() const { + return GoverningPredicateIdx != std::numeric_limits<unsigned>::max(); + } + + unsigned getGoverningPredicateOperandIdx() const { + assert(hasGoverningPredicate() && "Propery not set!"); + return GoverningPredicateIdx; + } + + SVEIntrinsicInfo &setGoverningPredicateOperandIdx(unsigned Index) { + assert(!hasGoverningPredicate() && "Cannot set property twice!"); + GoverningPredicateIdx = Index; + return *this; + } + + // + // Properties relating to operations the intrinsic could be transformed into. + // NOTE: This does not mean such a transformation is always possible, but the + // knowledge makes it possible to reuse existing optimisations without needing + // to embed specific handling for each intrinsic. For example, instruction + // simplification can be used to optimise an intrinsic's active lanes. + // + + bool hasMatchingUndefIntrinsic() const { + return UndefIntrinsic != Intrinsic::not_intrinsic; + } + + Intrinsic::ID getMatchingUndefIntrinsic() const { + assert(hasMatchingUndefIntrinsic() && "Propery not set!"); + return UndefIntrinsic; + } + + SVEIntrinsicInfo &setMatchingUndefIntrinsic(Intrinsic::ID IID) { + assert(!hasMatchingUndefIntrinsic() && "Cannot set property twice!"); + UndefIntrinsic = IID; + return *this; + } + + bool hasMatchingIROpode() const { return IROpcode != 0; } + + unsigned getMatchingIROpode() const { + assert(hasMatchingIROpode() && "Propery not set!"); + return IROpcode; + } + + SVEIntrinsicInfo &setMatchingIROpcode(unsigned Opcode) { + assert(!hasMatchingIROpode() && "Cannot set property twice!"); + IROpcode = Opcode; + return *this; + } + + // + // Properties relating to the result of inactive lanes. + // + + bool inactiveLanesTakenFromOperand() const { + return ResultLanes == InactiveLanesTakenFromOperand; + } + + unsigned getOperandIdxInactiveLanesTakenFrom() const { + assert(inactiveLanesTakenFromOperand() && "Propery not set!"); + return OperandIdxForInactiveLanes; + } + + SVEIntrinsicInfo &setOperandIdxInactiveLanesTakenFrom(unsigned Index) { + assert(ResultLanes == Uninitialized && "Cannot set property twice!"); + ResultLanes = InactiveLanesTakenFromOperand; + OperandIdxForInactiveLanes = Index; + return *this; + } + + bool inactiveLanesAreNotDefined() const { + return ResultLanes == InactiveLanesAreNotDefined; + } + + SVEIntrinsicInfo &setInactiveLanesAreNotDefined() { + assert(ResultLanes == Uninitialized && "Cannot set property twice!"); + ResultLanes = InactiveLanesAreNotDefined; + return *this; + } + + bool inactiveLanesAreUnused() const { + return ResultLanes == InactiveLanesAreUnused; + } + + SVEIntrinsicInfo &setInactiveLanesAreUnused() { + assert(ResultLanes == Uninitialized && "Cannot set property twice!"); + ResultLanes = InactiveLanesAreUnused; + return *this; + } + + // NOTE: Whilst not limited to only inactive lanes, the common use case is: + // inactiveLanesAreZeroed = + // resultIsZeroInitialized() && inactiveLanesAreUnused() + bool resultIsZeroInitialized() const { return ResultIsZeroInitialized; } + + SVEIntrinsicInfo &setResultIsZeroInitialized() { + ResultIsZeroInitialized = true; + return *this; + } + + // + // The first operand of unary merging operations is typically only used to + // set the result for inactive lanes. Knowing this allows us to deadcode the + // operand when we can prove there are no inactive lanes. + // + + bool hasOperandWithNoActiveLanes() const { + return OperandIdxWithNoActiveLanes != std::numeric_limits<unsigned>::max(); + } + + unsigned getOperandIdxWithNoActiveLanes() const { + assert(hasOperandWithNoActiveLanes() && "Propery not set!"); + return OperandIdxWithNoActiveLanes; + } + + SVEIntrinsicInfo &setOperandIdxWithNoActiveLanes(unsigned Index) { + assert(!hasOperandWithNoActiveLanes() && "Cannot set property twice!"); + OperandIdxWithNoActiveLanes = Index; + return *this; + } + +private: + unsigned GoverningPredicateIdx = std::numeric_limits<unsigned>::max(); + + Intrinsic::ID UndefIntrinsic = Intrinsic::not_intrinsic; + unsigned IROpcode = 0; + + enum PredicationStyle { + Uninitialized, + InactiveLanesTakenFromOperand, + InactiveLanesAreNotDefined, + InactiveLanesAreUnused + } ResultLanes = Uninitialized; + + bool ResultIsZeroInitialized = false; + unsigned OperandIdxForInactiveLanes = std::numeric_limits<unsigned>::max(); + unsigned OperandIdxWithNoActiveLanes = std::numeric_limits<unsigned>::max(); +}; + +static SVEIntrinsicInfo constructSVEIntrinsicInfo(IntrinsicInst &II) { + // Some SVE intrinsics do not use scalable vector types, but since they are + // not relevant from an SVEIntrinsicInfo perspective, they are also ignored. + if (!isa<ScalableVectorType>(II.getType()) && + all_of(II.args(), [&](const Value *V) { + return !isa<ScalableVectorType>(V->getType()); + })) + return SVEIntrinsicInfo(); + + Intrinsic::ID IID = II.getIntrinsicID(); + switch (IID) { + default: + break; + case Intrinsic::aarch64_sve_fcvt_bf16f32_v2: + case Intrinsic::aarch64_sve_fcvt_f16f32: + case Intrinsic::aarch64_sve_fcvt_f16f64: + case Intrinsic::aarch64_sve_fcvt_f32f16: + case Intrinsic::aarch64_sve_fcvt_f32f64: + case Intrinsic::aarch64_sve_fcvt_f64f16: + case Intrinsic::aarch64_sve_fcvt_f64f32: + case Intrinsic::aarch64_sve_fcvtlt_f32f16: + case Intrinsic::aarch64_sve_fcvtlt_f64f32: + case Intrinsic::aarch64_sve_fcvtx_f32f64: + case Intrinsic::aarch64_sve_fcvtzs: + case Intrinsic::aarch64_sve_fcvtzs_i32f16: + case Intrinsic::aarch64_sve_fcvtzs_i32f64: + case Intrinsic::aarch64_sve_fcvtzs_i64f16: + case Intrinsic::aarch64_sve_fcvtzs_i64f32: + case Intrinsic::aarch64_sve_fcvtzu: + case Intrinsic::aarch64_sve_fcvtzu_i32f16: + case Intrinsic::aarch64_sve_fcvtzu_i32f64: + case Intrinsic::aarch64_sve_fcvtzu_i64f16: + case Intrinsic::aarch64_sve_fcvtzu_i64f32: + case Intrinsic::aarch64_sve_scvtf: + case Intrinsic::aarch64_sve_scvtf_f16i32: + case Intrinsic::aarch64_sve_scvtf_f16i64: + case Intrinsic::aarch64_sve_scvtf_f32i64: + case Intrinsic::aarch64_sve_scvtf_f64i32: + case Intrinsic::aarch64_sve_ucvtf: + case Intrinsic::aarch64_sve_ucvtf_f16i32: + case Intrinsic::aarch64_sve_ucvtf_f16i64: + case Intrinsic::aarch64_sve_ucvtf_f32i64: + case Intrinsic::aarch64_sve_ucvtf_f64i32: + return SVEIntrinsicInfo::defaultMergingUnaryOp(); + + case Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2: + case Intrinsic::aarch64_sve_fcvtnt_f16f32: + case Intrinsic::aarch64_sve_fcvtnt_f32f64: + case Intrinsic::aarch64_sve_fcvtxnt_f32f64: + return SVEIntrinsicInfo::defaultMergingUnaryNarrowingTopOp(); + + case Intrinsic::aarch64_sve_fabd: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fabd_u); + case Intrinsic::aarch64_sve_fadd: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fadd_u) + .setMatchingIROpcode(Instruction::FAdd); + case Intrinsic::aarch64_sve_fdiv: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fdiv_u) + .setMatchingIROpcode(Instruction::FDiv); + case Intrinsic::aarch64_sve_fmax: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmax_u); + case Intrinsic::aarch64_sve_fmaxnm: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmaxnm_u); + case Intrinsic::aarch64_sve_fmin: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmin_u); + case Intrinsic::aarch64_sve_fminnm: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fminnm_u); + case Intrinsic::aarch64_sve_fmla: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmla_u); + case Intrinsic::aarch64_sve_fmls: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmls_u); + case Intrinsic::aarch64_sve_fmul: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmul_u) + .setMatchingIROpcode(Instruction::FMul); + case Intrinsic::aarch64_sve_fmulx: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fmulx_u); + case Intrinsic::aarch64_sve_fnmla: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fnmla_u); + case Intrinsic::aarch64_sve_fnmls: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fnmls_u); + case Intrinsic::aarch64_sve_fsub: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_fsub_u) + .setMatchingIROpcode(Instruction::FSub); + case Intrinsic::aarch64_sve_add: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_add_u) + .setMatchingIROpcode(Instruction::Add); + case Intrinsic::aarch64_sve_mla: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_mla_u); + case Intrinsic::aarch64_sve_mls: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_mls_u); + case Intrinsic::aarch64_sve_mul: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_mul_u) + .setMatchingIROpcode(Instruction::Mul); + case Intrinsic::aarch64_sve_sabd: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sabd_u); + case Intrinsic::aarch64_sve_sdiv: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sdiv_u) + .setMatchingIROpcode(Instruction::SDiv); + case Intrinsic::aarch64_sve_smax: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_smax_u); + case Intrinsic::aarch64_sve_smin: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_smin_u); + case Intrinsic::aarch64_sve_smulh: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_smulh_u); + case Intrinsic::aarch64_sve_sub: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sub_u) + .setMatchingIROpcode(Instruction::Sub); + case Intrinsic::aarch64_sve_uabd: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_uabd_u); + case Intrinsic::aarch64_sve_udiv: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_udiv_u) + .setMatchingIROpcode(Instruction::UDiv); + case Intrinsic::aarch64_sve_umax: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_umax_u); + case Intrinsic::aarch64_sve_umin: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_umin_u); + case Intrinsic::aarch64_sve_umulh: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_umulh_u); + case Intrinsic::aarch64_sve_asr: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_asr_u) + .setMatchingIROpcode(Instruction::AShr); + case Intrinsic::aarch64_sve_lsl: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_lsl_u) + .setMatchingIROpcode(Instruction::Shl); + case Intrinsic::aarch64_sve_lsr: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_lsr_u) + .setMatchingIROpcode(Instruction::LShr); + case Intrinsic::aarch64_sve_and: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_and_u) + .setMatchingIROpcode(Instruction::And); + case Intrinsic::aarch64_sve_bic: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_bic_u); + case Intrinsic::aarch64_sve_eor: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_eor_u) + .setMatchingIROpcode(Instruction::Xor); + case Intrinsic::aarch64_sve_orr: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_orr_u) + .setMatchingIROpcode(Instruction::Or); + case Intrinsic::aarch64_sve_sqsub: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_sqsub_u); + case Intrinsic::aarch64_sve_uqsub: + return SVEIntrinsicInfo::defaultMergingOp(Intrinsic::aarch64_sve_uqsub_u); + + case Intrinsic::aarch64_sve_add_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::Add); + case Intrinsic::aarch64_sve_and_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::And); + case Intrinsic::aarch64_sve_asr_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::AShr); + case Intrinsic::aarch64_sve_eor_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::Xor); + case Intrinsic::aarch64_sve_fadd_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::FAdd); + case Intrinsic::aarch64_sve_fdiv_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::FDiv); + case Intrinsic::aarch64_sve_fmul_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::FMul); + case Intrinsic::aarch64_sve_fsub_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::FSub); + case Intrinsic::aarch64_sve_lsl_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::Shl); + case Intrinsic::aarch64_sve_lsr_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::LShr); + case Intrinsic::aarch64_sve_mul_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::Mul); + case Intrinsic::aarch64_sve_orr_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::Or); + case Intrinsic::aarch64_sve_sdiv_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::SDiv); + case Intrinsic::aarch64_sve_sub_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::Sub); + case Intrinsic::aarch64_sve_udiv_u: + return SVEIntrinsicInfo::defaultUndefOp().setMatchingIROpcode( + Instruction::UDiv); + + case Intrinsic::aarch64_sve_addqv: + case Intrinsic::aarch64_sve_and_z: + case Intrinsic::aarch64_sve_bic_z: + case Intrinsic::aarch64_sve_brka_z: + case Intrinsic::aarch64_sve_brkb_z: + case Intrinsic::aarch64_sve_brkn_z: + case Intrinsic::aarch64_sve_brkpa_z: + case Intrinsic::aarch64_sve_brkpb_z: + case Intrinsic::aarch64_sve_cntp: + case Intrinsic::aarch64_sve_compact: + case Intrinsic::aarch64_sve_eor_z: + case Intrinsic::aarch64_sve_eorv: + case Intrinsic::aarch64_sve_eorqv: + case Intrinsic::aarch64_sve_nand_z: + case Intrinsic::aarch64_sve_nor_z: + case Intrinsic::aarch64_sve_orn_z: + case Intrinsic::aarch64_sve_orr_z: + case Intrinsic::aarch64_sve_orv: + case Intrinsic::aarch64_sve_orqv: + case Intrinsic::aarch64_sve_pnext: + case Intrinsic::aarch64_sve_rdffr_z: + case Intrinsic::aarch64_sve_saddv: + case Intrinsic::aarch64_sve_uaddv: + case Intrinsic::aarch64_sve_umaxv: + case Intrinsic::aarch64_sve_umaxqv: + case Intrinsic::aarch64_sve_cmpeq: + case Intrinsic::aarch64_sve_cmpeq_wide: + case Intrinsic::aarch64_sve_cmpge: + case Intrinsic::aarch64_sve_cmpge_wide: + case Intrinsic::aarch64_sve_cmpgt: + case Intrinsic::aarch64_sve_cmpgt_wide: + case Intrinsic::aarch64_sve_cmphi: + case Intrinsic::aarch64_sve_cmphi_wide: + case Intrinsic::aarch64_sve_cmphs: + case Intrinsic::aarch64_sve_cmphs_wide: + case Intrinsic::aarch64_sve_cmple_wide: + case Intrinsic::aarch64_sve_cmplo_wide: + case Intrinsic::aarch64_sve_cmpls_wide: + case Intrinsic::aarch64_sve_cmplt_wide: + case Intrinsic::aarch64_sve_cmpne: + case Intrinsic::aarch64_sve_cmpne_wide: + case Intrinsic::aarch64_sve_facge: + case Intrinsic::aarch64_sve_facgt: + case Intrinsic::aarch64_sve_fcmpeq: + case Intrinsic::aarch64_sve_fcmpge: + case Intrinsic::aarch64_sve_fcmpgt: + case Intrinsic::aarch64_sve_fcmpne: + case Intrinsic::aarch64_sve_fcmpuo: + case Intrinsic::aarch64_sve_ld1: + case Intrinsic::aarch64_sve_ld1_gather: + case Intrinsic::aarch64_sve_ld1_gather_index: + case Intrinsic::aarch64_sve_ld1_gather_scalar_offset: + case Intrinsic::aarch64_sve_ld1_gather_sxtw: + case Intrinsic::aarch64_sve_ld1_gather_sxtw_index: + case Intrinsic::aarch64_sve_ld1_gather_uxtw: + case Intrinsic::aarch64_sve_ld1_gather_uxtw_index: + case Intrinsic::aarch64_sve_ld1q_gather_index: + case Intrinsic::aarch64_sve_ld1q_gather_scalar_offset: + case Intrinsic::aarch64_sve_ld1q_gather_vector_offset: + case Intrinsic::aarch64_sve_ld1ro: + case Intrinsic::aarch64_sve_ld1rq: + case Intrinsic::aarch64_sve_ld1udq: + case Intrinsic::aarch64_sve_ld1uwq: + case Intrinsic::aarch64_sve_ld2_sret: + case Intrinsic::aarch64_sve_ld2q_sret: + case Intrinsic::aarch64_sve_ld3_sret: + case Intrinsic::aarch64_sve_ld3q_sret: + case Intrinsic::aarch64_sve_ld4_sret: + case Intrinsic::aarch64_sve_ld4q_sret: + case Intrinsic::aarch64_sve_ldff1: + case Intrinsic::aarch64_sve_ldff1_gather: + case Intrinsic::aarch64_sve_ldff1_gather_index: + case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset: + case Intrinsic::aarch64_sve_ldff1_gather_sxtw: + case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index: + case Intrinsic::aarch64_sve_ldff1_gather_uxtw: + case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index: + case Intrinsic::aarch64_sve_ldnf1: + case Intrinsic::aarch64_sve_ldnt1: + case Intrinsic::aarch64_sve_ldnt1_gather: + case Intrinsic::aarch64_sve_ldnt1_gather_index: + case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset: + case Intrinsic::aarch64_sve_ldnt1_gather_uxtw: + return SVEIntrinsicInfo::defaultZeroingOp(); + + case Intrinsic::aarch64_sve_prf: + case Intrinsic::aarch64_sve_prfb_gather_index: + case Intrinsic::aarch64_sve_prfb_gather_scalar_offset: + case Intrinsic::aarch64_sve_prfb_gather_sxtw_index: + case Intrinsic::aarch64_sve_prfb_gather_uxtw_index: + case Intrinsic::aarch64_sve_prfd_gather_index: + case Intrinsic::aarch64_sve_prfd_gather_scalar_offset: + case Intrinsic::aarch64_sve_prfd_gather_sxtw_index: + case Intrinsic::aarch64_sve_prfd_gather_uxtw_index: + case Intrinsic::aarch64_sve_prfh_gather_index: + case Intrinsic::aarch64_sve_prfh_gather_scalar_offset: + case Intrinsic::aarch64_sve_prfh_gather_sxtw_index: + case Intrinsic::aarch64_sve_prfh_gather_uxtw_index: + case Intrinsic::aarch64_sve_prfw_gather_index: + case Intrinsic::aarch64_sve_prfw_gather_scalar_offset: + case Intrinsic::aarch64_sve_prfw_gather_sxtw_index: + case Intrinsic::aarch64_sve_prfw_gather_uxtw_index: + return SVEIntrinsicInfo::defaultVoidOp(0); + + case Intrinsic::aarch64_sve_st1_scatter: + case Intrinsic::aarch64_sve_st1_scatter_scalar_offset: + case Intrinsic::aarch64_sve_st1_scatter_sxtw: + case Intrinsic::aarch64_sve_st1_scatter_sxtw_index: + case Intrinsic::aarch64_sve_st1_scatter_uxtw: + case Intrinsic::aarch64_sve_st1_scatter_uxtw_index: + case Intrinsic::aarch64_sve_st1dq: + case Intrinsic::aarch64_sve_st1q_scatter_index: + case Intrinsic::aarch64_sve_st1q_scatter_scalar_offset: + case Intrinsic::aarch64_sve_st1q_scatter_vector_offset: + case Intrinsic::aarch64_sve_st1wq: + case Intrinsic::aarch64_sve_stnt1: + case Intrinsic::aarch64_sve_stnt1_scatter: + case Intrinsic::aarch64_sve_stnt1_scatter_index: + case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset: + case Intrinsic::aarch64_sve_stnt1_scatter_uxtw: + return SVEIntrinsicInfo::defaultVoidOp(1); + case Intrinsic::aarch64_sve_st2: + case Intrinsic::aarch64_sve_st2q: + return SVEIntrinsicInfo::defaultVoidOp(2); + case Intrinsic::aarch64_sve_st3: + case Intrinsic::aarch64_sve_st3q: + return SVEIntrinsicInfo::defaultVoidOp(3); + case Intrinsic::aarch64_sve_st4: + case Intrinsic::aarch64_sve_st4q: + return SVEIntrinsicInfo::defaultVoidOp(4); + } + + return SVEIntrinsicInfo(); +} + +static bool isAllActivePredicate(Value *Pred) { + // Look through convert.from.svbool(convert.to.svbool(...) chain. + Value *UncastedPred; + if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>( + m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>( + m_Value(UncastedPred))))) + // If the predicate has the same or less lanes than the uncasted + // predicate then we know the casting has no effect. + if (cast<ScalableVectorType>(Pred->getType())->getMinNumElements() <= + cast<ScalableVectorType>(UncastedPred->getType())->getMinNumElements()) + Pred = UncastedPred; + auto *C = dyn_cast<Constant>(Pred); + return (C && C->isAllOnesValue()); +} + +// Simplify `V` by only considering the operations that affect active lanes. +// This function should only return existing Values or newly created Constants. +static Value *stripInactiveLanes(Value *V, const Value *Pg) { + auto *Dup = dyn_cast<IntrinsicInst>(V); + if (Dup && Dup->getIntrinsicID() == Intrinsic::aarch64_sve_dup && + Dup->getOperand(1) == Pg && isa<Constant>(Dup->getOperand(2))) + return ConstantVector::getSplat( + cast<VectorType>(V->getType())->getElementCount(), + cast<Constant>(Dup->getOperand(2))); + + return V; +} + +static std::optional<Instruction *> +simplifySVEIntrinsicBinOp(InstCombiner &IC, IntrinsicInst &II, + const SVEIntrinsicInfo &IInfo) { + const unsigned Opc = IInfo.getMatchingIROpode(); + assert(Instruction::isBinaryOp(Opc) && "Expected a binary operation!"); + + Value *Pg = II.getOperand(0); + Value *Op1 = II.getOperand(1); + Value *Op2 = II.getOperand(2); + const DataLayout &DL = II.getDataLayout(); + + // Canonicalise constants to the RHS. + if (Instruction::isCommutative(Opc) && IInfo.inactiveLanesAreNotDefined() && + isa<Constant>(Op1) && !isa<Constant>(Op2)) { + IC.replaceOperand(II, 1, Op2); + IC.replaceOperand(II, 2, Op1); + return &II; + } + + // Only active lanes matter when simplifying the operation. + Op1 = stripInactiveLanes(Op1, Pg); + Op2 = stripInactiveLanes(Op2, Pg); + + Value *SimpleII; + if (auto FII = dyn_cast<FPMathOperator>(&II)) + SimpleII = simplifyBinOp(Opc, Op1, Op2, FII->getFastMathFlags(), DL); + else + SimpleII = simplifyBinOp(Opc, Op1, Op2, DL); + + // An SVE intrinsic's result is always defined. However, this is not the case + // for its equivalent IR instruction (e.g. when shifting by an amount more + // than the data's bitwidth). Simplifications to an undefined result must be + // ignored to preserve the intrinsic's expected behaviour. + if (!SimpleII || isa<UndefValue>(SimpleII)) + return std::nullopt; + + if (IInfo.inactiveLanesAreNotDefined()) + return IC.replaceInstUsesWith(II, SimpleII); + + Value *Inactive = II.getOperand(IInfo.getOperandIdxInactiveLanesTakenFrom()); + + // The intrinsic does nothing (e.g. sve.mul(pg, A, 1.0)). + if (SimpleII == Inactive) + return IC.replaceInstUsesWith(II, SimpleII); + + // Inactive lanes must be preserved. + SimpleII = IC.Builder.CreateSelect(Pg, SimpleII, Inactive); + return IC.replaceInstUsesWith(II, SimpleII); +} + +// Use SVE intrinsic info to eliminate redundant operands and/or canonicalise +// to operations with less strict inactive lane requirements. +static std::optional<Instruction *> +simplifySVEIntrinsic(InstCombiner &IC, IntrinsicInst &II, + const SVEIntrinsicInfo &IInfo) { + if (!IInfo.hasGoverningPredicate()) + return std::nullopt; + + auto *OpPredicate = II.getOperand(IInfo.getGoverningPredicateOperandIdx()); + + // If there are no active lanes. + if (match(OpPredicate, m_ZeroInt())) { + if (IInfo.inactiveLanesTakenFromOperand()) + return IC.replaceInstUsesWith( + II, II.getOperand(IInfo.getOperandIdxInactiveLanesTakenFrom())); + + if (IInfo.inactiveLanesAreUnused()) { + if (IInfo.resultIsZeroInitialized()) + IC.replaceInstUsesWith(II, Constant::getNullValue(II.getType())); + + return IC.eraseInstFromFunction(II); + } + } + + // If there are no inactive lanes. + if (isAllActivePredicate(OpPredicate)) { + if (IInfo.hasOperandWithNoActiveLanes()) { + unsigned OpIdx = IInfo.getOperandIdxWithNoActiveLanes(); + if (!isa<UndefValue>(II.getOperand(OpIdx))) + return IC.replaceOperand(II, OpIdx, UndefValue::get(II.getType())); + } + + if (IInfo.hasMatchingUndefIntrinsic()) { + auto *NewDecl = Intrinsic::getOrInsertDeclaration( + II.getModule(), IInfo.getMatchingUndefIntrinsic(), {II.getType()}); + II.setCalledFunction(NewDecl); + return &II; + } + } + + // Operation specific simplifications. + if (IInfo.hasMatchingIROpode() && + Instruction::isBinaryOp(IInfo.getMatchingIROpode())) + return simplifySVEIntrinsicBinOp(IC, II, IInfo); + + return std::nullopt; +} + // (from_svbool (binop (to_svbool pred) (svbool_t _) (svbool_t _)))) // => (binop (pred) (from_svbool _) (from_svbool _)) // @@ -1095,85 +1776,6 @@ instCombineConvertFromSVBool(InstCombiner &IC, IntrinsicInst &II) { return IC.replaceInstUsesWith(II, EarliestReplacement); } -static bool isAllActivePredicate(Value *Pred) { - // Look through convert.from.svbool(convert.to.svbool(...) chain. - Value *UncastedPred; - if (match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_convert_from_svbool>( - m_Intrinsic<Intrinsic::aarch64_sve_convert_to_svbool>( - m_Value(UncastedPred))))) - // If the predicate has the same or less lanes than the uncasted - // predicate then we know the casting has no effect. - if (cast<ScalableVectorType>(Pred->getType())->getMinNumElements() <= - cast<ScalableVectorType>(UncastedPred->getType())->getMinNumElements()) - Pred = UncastedPred; - - return match(Pred, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>( - m_ConstantInt<AArch64SVEPredPattern::all>())); -} - -// Simplify unary operation where predicate has all inactive lanes by replacing -// instruction with its operand -static std::optional<Instruction *> -instCombineSVENoActiveReplace(InstCombiner &IC, IntrinsicInst &II, - bool hasInactiveVector) { - int PredOperand = hasInactiveVector ? 1 : 0; - int ReplaceOperand = hasInactiveVector ? 0 : 1; - if (match(II.getOperand(PredOperand), m_ZeroInt())) { - IC.replaceInstUsesWith(II, II.getOperand(ReplaceOperand)); - return IC.eraseInstFromFunction(II); - } - return std::nullopt; -} - -// Simplify unary operation where predicate has all inactive lanes or -// replace unused first operand with undef when all lanes are active -static std::optional<Instruction *> -instCombineSVEAllOrNoActiveUnary(InstCombiner &IC, IntrinsicInst &II) { - if (isAllActivePredicate(II.getOperand(1)) && - !isa<llvm::UndefValue>(II.getOperand(0)) && - !isa<llvm::PoisonValue>(II.getOperand(0))) { - Value *Undef = llvm::UndefValue::get(II.getType()); - return IC.replaceOperand(II, 0, Undef); - } - return instCombineSVENoActiveReplace(IC, II, true); -} - -// Erase unary operation where predicate has all inactive lanes -static std::optional<Instruction *> -instCombineSVENoActiveUnaryErase(InstCombiner &IC, IntrinsicInst &II, - int PredPos) { - if (match(II.getOperand(PredPos), m_ZeroInt())) { - return IC.eraseInstFromFunction(II); - } - return std::nullopt; -} - -// Simplify operation where predicate has all inactive lanes by replacing -// instruction with zeroed object -static std::optional<Instruction *> -instCombineSVENoActiveZero(InstCombiner &IC, IntrinsicInst &II) { - if (match(II.getOperand(0), m_ZeroInt())) { - Constant *Node; - Type *RetTy = II.getType(); - if (RetTy->isStructTy()) { - auto StructT = cast<StructType>(RetTy); - auto VecT = StructT->getElementType(0); - SmallVector<llvm::Constant *, 4> ZerVec; - for (unsigned i = 0; i < StructT->getNumElements(); i++) { - ZerVec.push_back(VecT->isFPOrFPVectorTy() ? ConstantFP::get(VecT, 0.0) - : ConstantInt::get(VecT, 0)); - } - Node = ConstantStruct::get(StructT, ZerVec); - } else - Node = RetTy->isFPOrFPVectorTy() ? ConstantFP::get(RetTy, 0.0) - : ConstantInt::get(II.getType(), 0); - - IC.replaceInstUsesWith(II, Node); - return IC.eraseInstFromFunction(II); - } - return std::nullopt; -} - static std::optional<Instruction *> instCombineSVESel(InstCombiner &IC, IntrinsicInst &II) { // svsel(ptrue, x, y) => x @@ -1224,18 +1826,7 @@ static std::optional<Instruction *> instCombineSVECmpNE(InstCombiner &IC, IntrinsicInst &II) { LLVMContext &Ctx = II.getContext(); - // Replace by zero constant when all lanes are inactive - if (auto II_NA = instCombineSVENoActiveZero(IC, II)) - return II_NA; - - // Check that the predicate is all active - auto *Pg = dyn_cast<IntrinsicInst>(II.getArgOperand(0)); - if (!Pg || Pg->getIntrinsicID() != Intrinsic::aarch64_sve_ptrue) - return std::nullopt; - - const auto PTruePattern = - cast<ConstantInt>(Pg->getOperand(0))->getZExtValue(); - if (PTruePattern != AArch64SVEPredPattern::all) + if (!isAllActivePredicate(II.getArgOperand(0))) return std::nullopt; // Check that we have a compare of zero.. @@ -1450,7 +2041,7 @@ static std::optional<Instruction *> instCombineRDFFR(InstCombiner &IC, auto *PTrue = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_ptrue, {II.getType()}, {AllPat}); auto *RDFFR = - IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {}, {PTrue}); + IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_rdffr_z, {PTrue}); RDFFR->takeName(&II); return IC.replaceInstUsesWith(II, RDFFR); } @@ -1460,10 +2051,10 @@ instCombineSVECntElts(InstCombiner &IC, IntrinsicInst &II, unsigned NumElts) { const auto Pattern = cast<ConstantInt>(II.getArgOperand(0))->getZExtValue(); if (Pattern == AArch64SVEPredPattern::all) { - Constant *StepVal = ConstantInt::get(II.getType(), NumElts); - auto *VScale = IC.Builder.CreateVScale(StepVal); - VScale->takeName(&II); - return IC.replaceInstUsesWith(II, VScale); + Value *Cnt = IC.Builder.CreateElementCount( + II.getType(), ElementCount::getScalable(NumElts)); + Cnt->takeName(&II); + return IC.replaceInstUsesWith(II, Cnt); } unsigned MinNumElts = getNumElementsFromSVEPredPattern(Pattern); @@ -1592,10 +2183,6 @@ instCombineSVELD1(InstCombiner &IC, IntrinsicInst &II, const DataLayout &DL) { Value *PtrOp = II.getOperand(1); Type *VecTy = II.getType(); - // Replace by zero constant when all lanes are inactive - if (auto II_NA = instCombineSVENoActiveZero(IC, II)) - return II_NA; - if (isAllActivePredicate(Pred)) { LoadInst *Load = IC.Builder.CreateLoad(VecTy, PtrOp); Load->copyMetadata(II); @@ -1649,48 +2236,15 @@ instCombineSVEVectorBinOp(InstCombiner &IC, IntrinsicInst &II) { auto *OpPredicate = II.getOperand(0); auto BinOpCode = intrinsicIDToBinOpCode(II.getIntrinsicID()); if (BinOpCode == Instruction::BinaryOpsEnd || - !match(OpPredicate, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>( - m_ConstantInt<AArch64SVEPredPattern::all>()))) + !isAllActivePredicate(OpPredicate)) return std::nullopt; auto BinOp = IC.Builder.CreateBinOpFMF( BinOpCode, II.getOperand(1), II.getOperand(2), II.getFastMathFlags()); return IC.replaceInstUsesWith(II, BinOp); } -// Canonicalise operations that take an all active predicate (e.g. sve.add -> -// sve.add_u). -static std::optional<Instruction *> instCombineSVEAllActive(IntrinsicInst &II, - Intrinsic::ID IID) { - auto *OpPredicate = II.getOperand(0); - if (!match(OpPredicate, m_Intrinsic<Intrinsic::aarch64_sve_ptrue>( - m_ConstantInt<AArch64SVEPredPattern::all>()))) - return std::nullopt; - - auto *Mod = II.getModule(); - auto *NewDecl = Intrinsic::getOrInsertDeclaration(Mod, IID, {II.getType()}); - II.setCalledFunction(NewDecl); - - return &II; -} - -// Simplify operations where predicate has all inactive lanes or try to replace -// with _u form when all lanes are active -static std::optional<Instruction *> -instCombineSVEAllOrNoActive(InstCombiner &IC, IntrinsicInst &II, - Intrinsic::ID IID) { - if (match(II.getOperand(0), m_ZeroInt())) { - // llvm_ir, pred(0), op1, op2 - Spec says to return op1 when all lanes are - // inactive for sv[func]_m - return IC.replaceInstUsesWith(II, II.getOperand(1)); - } - return instCombineSVEAllActive(II, IID); -} - static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC, IntrinsicInst &II) { - if (auto II_U = - instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_add_u)) - return II_U; if (auto MLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul, Intrinsic::aarch64_sve_mla>( IC, II, true)) @@ -1704,9 +2258,6 @@ static std::optional<Instruction *> instCombineSVEVectorAdd(InstCombiner &IC, static std::optional<Instruction *> instCombineSVEVectorFAdd(InstCombiner &IC, IntrinsicInst &II) { - if (auto II_U = - instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fadd_u)) - return II_U; if (auto FMLA = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, Intrinsic::aarch64_sve_fmla>(IC, II, @@ -1747,9 +2298,6 @@ instCombineSVEVectorFAddU(InstCombiner &IC, IntrinsicInst &II) { static std::optional<Instruction *> instCombineSVEVectorFSub(InstCombiner &IC, IntrinsicInst &II) { - if (auto II_U = - instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fsub_u)) - return II_U; if (auto FMLS = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_fmul, Intrinsic::aarch64_sve_fmls>(IC, II, @@ -1790,9 +2338,6 @@ instCombineSVEVectorFSubU(InstCombiner &IC, IntrinsicInst &II) { static std::optional<Instruction *> instCombineSVEVectorSub(InstCombiner &IC, IntrinsicInst &II) { - if (auto II_U = - instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sub_u)) - return II_U; if (auto MLS = instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul, Intrinsic::aarch64_sve_mls>( IC, II, true)) @@ -1800,51 +2345,6 @@ static std::optional<Instruction *> instCombineSVEVectorSub(InstCombiner &IC, return std::nullopt; } -static std::optional<Instruction *> instCombineSVEVectorMul(InstCombiner &IC, - IntrinsicInst &II, - Intrinsic::ID IID) { - auto *OpPredicate = II.getOperand(0); - auto *OpMultiplicand = II.getOperand(1); - auto *OpMultiplier = II.getOperand(2); - - // Return true if a given instruction is a unit splat value, false otherwise. - auto IsUnitSplat = [](auto *I) { - auto *SplatValue = getSplatValue(I); - if (!SplatValue) - return false; - return match(SplatValue, m_FPOne()) || match(SplatValue, m_One()); - }; - - // Return true if a given instruction is an aarch64_sve_dup intrinsic call - // with a unit splat value, false otherwise. - auto IsUnitDup = [](auto *I) { - auto *IntrI = dyn_cast<IntrinsicInst>(I); - if (!IntrI || IntrI->getIntrinsicID() != Intrinsic::aarch64_sve_dup) - return false; - - auto *SplatValue = IntrI->getOperand(2); - return match(SplatValue, m_FPOne()) || match(SplatValue, m_One()); - }; - - if (IsUnitSplat(OpMultiplier)) { - // [f]mul pg %n, (dupx 1) => %n - OpMultiplicand->takeName(&II); - return IC.replaceInstUsesWith(II, OpMultiplicand); - } else if (IsUnitDup(OpMultiplier)) { - // [f]mul pg %n, (dup pg 1) => %n - auto *DupInst = cast<IntrinsicInst>(OpMultiplier); - auto *DupPg = DupInst->getOperand(1); - // TODO: this is naive. The optimization is still valid if DupPg - // 'encompasses' OpPredicate, not only if they're the same predicate. - if (OpPredicate == DupPg) { - OpMultiplicand->takeName(&II); - return IC.replaceInstUsesWith(II, OpMultiplicand); - } - } - - return instCombineSVEVectorBinOp(IC, II); -} - static std::optional<Instruction *> instCombineSVEUnpack(InstCombiner &IC, IntrinsicInst &II) { Value *UnpackArg = II.getArgOperand(0); @@ -1907,9 +2407,9 @@ static std::optional<Instruction *> instCombineSVEUzp1(InstCombiner &IC, if (TyA == B->getType() && RetTy == ScalableVectorType::getDoubleElementsVectorType(TyA)) { auto *SubVec = IC.Builder.CreateInsertVector( - RetTy, PoisonValue::get(RetTy), A, IC.Builder.getInt64(0)); - auto *ConcatVec = IC.Builder.CreateInsertVector( - RetTy, SubVec, B, IC.Builder.getInt64(TyA->getMinNumElements())); + RetTy, PoisonValue::get(RetTy), A, uint64_t(0)); + auto *ConcatVec = IC.Builder.CreateInsertVector(RetTy, SubVec, B, + TyA->getMinNumElements()); ConcatVec->takeName(&II); return IC.replaceInstUsesWith(II, ConcatVec); } @@ -1941,10 +2441,6 @@ instCombineLD1GatherIndex(InstCombiner &IC, IntrinsicInst &II) { Type *Ty = II.getType(); Value *PassThru = ConstantAggregateZero::get(Ty); - // Replace by zero constant when all lanes are inactive - if (auto II_NA = instCombineSVENoActiveZero(IC, II)) - return II_NA; - // Contiguous gather => masked load. // (sve.ld1.gather.index Mask BasePtr (sve.index IndexBase 1)) // => (masked.load (gep BasePtr IndexBase) Align Mask zeroinitializer) @@ -2107,9 +2603,9 @@ static std::optional<Instruction *> instCombineSVEDupqLane(InstCombiner &IC, auto *WideShuffleMaskTy = ScalableVectorType::get(IC.Builder.getInt32Ty(), PatternElementCount); - auto ZeroIdx = ConstantInt::get(IC.Builder.getInt64Ty(), APInt(64, 0)); auto InsertSubvector = IC.Builder.CreateInsertVector( - II.getType(), PoisonValue::get(II.getType()), InsertEltChain, ZeroIdx); + II.getType(), PoisonValue::get(II.getType()), InsertEltChain, + uint64_t(0)); auto WideBitcast = IC.Builder.CreateBitOrPointerCast(InsertSubvector, WideScalableTy); auto WideShuffleMask = ConstantAggregateZero::get(WideShuffleMaskTy); @@ -2200,175 +2696,56 @@ static std::optional<Instruction *> instCombineDMB(InstCombiner &IC, return std::nullopt; } +static std::optional<Instruction *> instCombinePTrue(InstCombiner &IC, + IntrinsicInst &II) { + if (match(II.getOperand(0), m_ConstantInt<AArch64SVEPredPattern::all>())) + return IC.replaceInstUsesWith(II, Constant::getAllOnesValue(II.getType())); + return std::nullopt; +} + +static std::optional<Instruction *> instCombineSVEUxt(InstCombiner &IC, + IntrinsicInst &II, + unsigned NumBits) { + Value *Passthru = II.getOperand(0); + Value *Pg = II.getOperand(1); + Value *Op = II.getOperand(2); + + // Convert UXT[BHW] to AND. + if (isa<UndefValue>(Passthru) || isAllActivePredicate(Pg)) { + auto *Ty = cast<VectorType>(II.getType()); + auto MaskValue = APInt::getLowBitsSet(Ty->getScalarSizeInBits(), NumBits); + auto *Mask = ConstantInt::get(Ty, MaskValue); + auto *And = IC.Builder.CreateIntrinsic(Intrinsic::aarch64_sve_and_u, {Ty}, + {Pg, Op, Mask}); + return IC.replaceInstUsesWith(II, And); + } + + return std::nullopt; +} + +static std::optional<Instruction *> +instCombineInStreamingMode(InstCombiner &IC, IntrinsicInst &II) { + SMEAttrs FnSMEAttrs(*II.getFunction()); + bool IsStreaming = FnSMEAttrs.hasStreamingInterfaceOrBody(); + if (IsStreaming || !FnSMEAttrs.hasStreamingCompatibleInterface()) + return IC.replaceInstUsesWith( + II, ConstantInt::getBool(II.getType(), IsStreaming)); + return std::nullopt; +} + std::optional<Instruction *> AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const { + const SVEIntrinsicInfo &IInfo = constructSVEIntrinsicInfo(II); + if (std::optional<Instruction *> I = simplifySVEIntrinsic(IC, II, IInfo)) + return I; + Intrinsic::ID IID = II.getIntrinsicID(); switch (IID) { default: break; case Intrinsic::aarch64_dmb: return instCombineDMB(IC, II); - case Intrinsic::aarch64_sve_fcvt_bf16f32_v2: - case Intrinsic::aarch64_sve_fcvt_f16f32: - case Intrinsic::aarch64_sve_fcvt_f16f64: - case Intrinsic::aarch64_sve_fcvt_f32f16: - case Intrinsic::aarch64_sve_fcvt_f32f64: - case Intrinsic::aarch64_sve_fcvt_f64f16: - case Intrinsic::aarch64_sve_fcvt_f64f32: - case Intrinsic::aarch64_sve_fcvtlt_f32f16: - case Intrinsic::aarch64_sve_fcvtlt_f64f32: - case Intrinsic::aarch64_sve_fcvtx_f32f64: - case Intrinsic::aarch64_sve_fcvtzs: - case Intrinsic::aarch64_sve_fcvtzs_i32f16: - case Intrinsic::aarch64_sve_fcvtzs_i32f64: - case Intrinsic::aarch64_sve_fcvtzs_i64f16: - case Intrinsic::aarch64_sve_fcvtzs_i64f32: - case Intrinsic::aarch64_sve_fcvtzu: - case Intrinsic::aarch64_sve_fcvtzu_i32f16: - case Intrinsic::aarch64_sve_fcvtzu_i32f64: - case Intrinsic::aarch64_sve_fcvtzu_i64f16: - case Intrinsic::aarch64_sve_fcvtzu_i64f32: - case Intrinsic::aarch64_sve_scvtf: - case Intrinsic::aarch64_sve_scvtf_f16i32: - case Intrinsic::aarch64_sve_scvtf_f16i64: - case Intrinsic::aarch64_sve_scvtf_f32i64: - case Intrinsic::aarch64_sve_scvtf_f64i32: - case Intrinsic::aarch64_sve_ucvtf: - case Intrinsic::aarch64_sve_ucvtf_f16i32: - case Intrinsic::aarch64_sve_ucvtf_f16i64: - case Intrinsic::aarch64_sve_ucvtf_f32i64: - case Intrinsic::aarch64_sve_ucvtf_f64i32: - return instCombineSVEAllOrNoActiveUnary(IC, II); - case Intrinsic::aarch64_sve_fcvtnt_bf16f32_v2: - case Intrinsic::aarch64_sve_fcvtnt_f16f32: - case Intrinsic::aarch64_sve_fcvtnt_f32f64: - case Intrinsic::aarch64_sve_fcvtxnt_f32f64: - return instCombineSVENoActiveReplace(IC, II, true); - case Intrinsic::aarch64_sve_st1_scatter: - case Intrinsic::aarch64_sve_st1_scatter_scalar_offset: - case Intrinsic::aarch64_sve_st1_scatter_sxtw: - case Intrinsic::aarch64_sve_st1_scatter_sxtw_index: - case Intrinsic::aarch64_sve_st1_scatter_uxtw: - case Intrinsic::aarch64_sve_st1_scatter_uxtw_index: - case Intrinsic::aarch64_sve_st1dq: - case Intrinsic::aarch64_sve_st1q_scatter_index: - case Intrinsic::aarch64_sve_st1q_scatter_scalar_offset: - case Intrinsic::aarch64_sve_st1q_scatter_vector_offset: - case Intrinsic::aarch64_sve_st1wq: - case Intrinsic::aarch64_sve_stnt1: - case Intrinsic::aarch64_sve_stnt1_scatter: - case Intrinsic::aarch64_sve_stnt1_scatter_index: - case Intrinsic::aarch64_sve_stnt1_scatter_scalar_offset: - case Intrinsic::aarch64_sve_stnt1_scatter_uxtw: - return instCombineSVENoActiveUnaryErase(IC, II, 1); - case Intrinsic::aarch64_sve_st2: - case Intrinsic::aarch64_sve_st2q: - return instCombineSVENoActiveUnaryErase(IC, II, 2); - case Intrinsic::aarch64_sve_st3: - case Intrinsic::aarch64_sve_st3q: - return instCombineSVENoActiveUnaryErase(IC, II, 3); - case Intrinsic::aarch64_sve_st4: - case Intrinsic::aarch64_sve_st4q: - return instCombineSVENoActiveUnaryErase(IC, II, 4); - case Intrinsic::aarch64_sve_addqv: - case Intrinsic::aarch64_sve_and_z: - case Intrinsic::aarch64_sve_bic_z: - case Intrinsic::aarch64_sve_brka_z: - case Intrinsic::aarch64_sve_brkb_z: - case Intrinsic::aarch64_sve_brkn_z: - case Intrinsic::aarch64_sve_brkpa_z: - case Intrinsic::aarch64_sve_brkpb_z: - case Intrinsic::aarch64_sve_cntp: - case Intrinsic::aarch64_sve_compact: - case Intrinsic::aarch64_sve_eor_z: - case Intrinsic::aarch64_sve_eorv: - case Intrinsic::aarch64_sve_eorqv: - case Intrinsic::aarch64_sve_nand_z: - case Intrinsic::aarch64_sve_nor_z: - case Intrinsic::aarch64_sve_orn_z: - case Intrinsic::aarch64_sve_orr_z: - case Intrinsic::aarch64_sve_orv: - case Intrinsic::aarch64_sve_orqv: - case Intrinsic::aarch64_sve_pnext: - case Intrinsic::aarch64_sve_rdffr_z: - case Intrinsic::aarch64_sve_saddv: - case Intrinsic::aarch64_sve_uaddv: - case Intrinsic::aarch64_sve_umaxv: - case Intrinsic::aarch64_sve_umaxqv: - case Intrinsic::aarch64_sve_cmpeq: - case Intrinsic::aarch64_sve_cmpeq_wide: - case Intrinsic::aarch64_sve_cmpge: - case Intrinsic::aarch64_sve_cmpge_wide: - case Intrinsic::aarch64_sve_cmpgt: - case Intrinsic::aarch64_sve_cmpgt_wide: - case Intrinsic::aarch64_sve_cmphi: - case Intrinsic::aarch64_sve_cmphi_wide: - case Intrinsic::aarch64_sve_cmphs: - case Intrinsic::aarch64_sve_cmphs_wide: - case Intrinsic::aarch64_sve_cmple_wide: - case Intrinsic::aarch64_sve_cmplo_wide: - case Intrinsic::aarch64_sve_cmpls_wide: - case Intrinsic::aarch64_sve_cmplt_wide: - case Intrinsic::aarch64_sve_facge: - case Intrinsic::aarch64_sve_facgt: - case Intrinsic::aarch64_sve_fcmpeq: - case Intrinsic::aarch64_sve_fcmpge: - case Intrinsic::aarch64_sve_fcmpgt: - case Intrinsic::aarch64_sve_fcmpne: - case Intrinsic::aarch64_sve_fcmpuo: - case Intrinsic::aarch64_sve_ld1_gather: - case Intrinsic::aarch64_sve_ld1_gather_scalar_offset: - case Intrinsic::aarch64_sve_ld1_gather_sxtw: - case Intrinsic::aarch64_sve_ld1_gather_sxtw_index: - case Intrinsic::aarch64_sve_ld1_gather_uxtw: - case Intrinsic::aarch64_sve_ld1_gather_uxtw_index: - case Intrinsic::aarch64_sve_ld1q_gather_index: - case Intrinsic::aarch64_sve_ld1q_gather_scalar_offset: - case Intrinsic::aarch64_sve_ld1q_gather_vector_offset: - case Intrinsic::aarch64_sve_ld1ro: - case Intrinsic::aarch64_sve_ld1rq: - case Intrinsic::aarch64_sve_ld1udq: - case Intrinsic::aarch64_sve_ld1uwq: - case Intrinsic::aarch64_sve_ld2_sret: - case Intrinsic::aarch64_sve_ld2q_sret: - case Intrinsic::aarch64_sve_ld3_sret: - case Intrinsic::aarch64_sve_ld3q_sret: - case Intrinsic::aarch64_sve_ld4_sret: - case Intrinsic::aarch64_sve_ld4q_sret: - case Intrinsic::aarch64_sve_ldff1: - case Intrinsic::aarch64_sve_ldff1_gather: - case Intrinsic::aarch64_sve_ldff1_gather_index: - case Intrinsic::aarch64_sve_ldff1_gather_scalar_offset: - case Intrinsic::aarch64_sve_ldff1_gather_sxtw: - case Intrinsic::aarch64_sve_ldff1_gather_sxtw_index: - case Intrinsic::aarch64_sve_ldff1_gather_uxtw: - case Intrinsic::aarch64_sve_ldff1_gather_uxtw_index: - case Intrinsic::aarch64_sve_ldnf1: - case Intrinsic::aarch64_sve_ldnt1: - case Intrinsic::aarch64_sve_ldnt1_gather: - case Intrinsic::aarch64_sve_ldnt1_gather_index: - case Intrinsic::aarch64_sve_ldnt1_gather_scalar_offset: - case Intrinsic::aarch64_sve_ldnt1_gather_uxtw: - return instCombineSVENoActiveZero(IC, II); - case Intrinsic::aarch64_sve_prf: - case Intrinsic::aarch64_sve_prfb_gather_index: - case Intrinsic::aarch64_sve_prfb_gather_scalar_offset: - case Intrinsic::aarch64_sve_prfb_gather_sxtw_index: - case Intrinsic::aarch64_sve_prfb_gather_uxtw_index: - case Intrinsic::aarch64_sve_prfd_gather_index: - case Intrinsic::aarch64_sve_prfd_gather_scalar_offset: - case Intrinsic::aarch64_sve_prfd_gather_sxtw_index: - case Intrinsic::aarch64_sve_prfd_gather_uxtw_index: - case Intrinsic::aarch64_sve_prfh_gather_index: - case Intrinsic::aarch64_sve_prfh_gather_scalar_offset: - case Intrinsic::aarch64_sve_prfh_gather_sxtw_index: - case Intrinsic::aarch64_sve_prfh_gather_uxtw_index: - case Intrinsic::aarch64_sve_prfw_gather_index: - case Intrinsic::aarch64_sve_prfw_gather_scalar_offset: - case Intrinsic::aarch64_sve_prfw_gather_sxtw_index: - case Intrinsic::aarch64_sve_prfw_gather_uxtw_index: - return instCombineSVENoActiveUnaryErase(IC, II, 0); case Intrinsic::aarch64_neon_fmaxnm: case Intrinsic::aarch64_neon_fminnm: return instCombineMaxMinNM(IC, II); @@ -2401,39 +2778,12 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC, case Intrinsic::aarch64_sve_ptest_first: case Intrinsic::aarch64_sve_ptest_last: return instCombineSVEPTest(IC, II); - case Intrinsic::aarch64_sve_fabd: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fabd_u); case Intrinsic::aarch64_sve_fadd: return instCombineSVEVectorFAdd(IC, II); case Intrinsic::aarch64_sve_fadd_u: return instCombineSVEVectorFAddU(IC, II); - case Intrinsic::aarch64_sve_fdiv: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fdiv_u); - case Intrinsic::aarch64_sve_fmax: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmax_u); - case Intrinsic::aarch64_sve_fmaxnm: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmaxnm_u); - case Intrinsic::aarch64_sve_fmin: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmin_u); - case Intrinsic::aarch64_sve_fminnm: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fminnm_u); - case Intrinsic::aarch64_sve_fmla: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmla_u); - case Intrinsic::aarch64_sve_fmls: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmls_u); - case Intrinsic::aarch64_sve_fmul: - if (auto II_U = - instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmul_u)) - return II_U; - return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u); case Intrinsic::aarch64_sve_fmul_u: - return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_fmul_u); - case Intrinsic::aarch64_sve_fmulx: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fmulx_u); - case Intrinsic::aarch64_sve_fnmla: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmla_u); - case Intrinsic::aarch64_sve_fnmls: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_fnmls_u); + return instCombineSVEVectorBinOp(IC, II); case Intrinsic::aarch64_sve_fsub: return instCombineSVEVectorFSub(IC, II); case Intrinsic::aarch64_sve_fsub_u: @@ -2444,57 +2794,12 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC, return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u, Intrinsic::aarch64_sve_mla_u>( IC, II, true); - case Intrinsic::aarch64_sve_mla: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mla_u); - case Intrinsic::aarch64_sve_mls: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mls_u); - case Intrinsic::aarch64_sve_mul: - if (auto II_U = - instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_mul_u)) - return II_U; - return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u); - case Intrinsic::aarch64_sve_mul_u: - return instCombineSVEVectorMul(IC, II, Intrinsic::aarch64_sve_mul_u); - case Intrinsic::aarch64_sve_sabd: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sabd_u); - case Intrinsic::aarch64_sve_smax: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smax_u); - case Intrinsic::aarch64_sve_smin: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smin_u); - case Intrinsic::aarch64_sve_smulh: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_smulh_u); case Intrinsic::aarch64_sve_sub: return instCombineSVEVectorSub(IC, II); case Intrinsic::aarch64_sve_sub_u: return instCombineSVEVectorFuseMulAddSub<Intrinsic::aarch64_sve_mul_u, Intrinsic::aarch64_sve_mls_u>( IC, II, true); - case Intrinsic::aarch64_sve_uabd: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uabd_u); - case Intrinsic::aarch64_sve_umax: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umax_u); - case Intrinsic::aarch64_sve_umin: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umin_u); - case Intrinsic::aarch64_sve_umulh: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_umulh_u); - case Intrinsic::aarch64_sve_asr: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_asr_u); - case Intrinsic::aarch64_sve_lsl: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsl_u); - case Intrinsic::aarch64_sve_lsr: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_lsr_u); - case Intrinsic::aarch64_sve_and: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_and_u); - case Intrinsic::aarch64_sve_bic: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_bic_u); - case Intrinsic::aarch64_sve_eor: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_eor_u); - case Intrinsic::aarch64_sve_orr: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_orr_u); - case Intrinsic::aarch64_sve_sqsub: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_sqsub_u); - case Intrinsic::aarch64_sve_uqsub: - return instCombineSVEAllOrNoActive(IC, II, Intrinsic::aarch64_sve_uqsub_u); case Intrinsic::aarch64_sve_tbl: return instCombineSVETBL(IC, II); case Intrinsic::aarch64_sve_uunpkhi: @@ -2525,6 +2830,16 @@ AArch64TTIImpl::instCombineIntrinsic(InstCombiner &IC, return instCombineSVEDupqLane(IC, II); case Intrinsic::aarch64_sve_insr: return instCombineSVEInsr(IC, II); + case Intrinsic::aarch64_sve_ptrue: + return instCombinePTrue(IC, II); + case Intrinsic::aarch64_sve_uxtb: + return instCombineSVEUxt(IC, II, 8); + case Intrinsic::aarch64_sve_uxth: + return instCombineSVEUxt(IC, II, 16); + case Intrinsic::aarch64_sve_uxtw: + return instCombineSVEUxt(IC, II, 32); + case Intrinsic::aarch64_sme_in_streaming_mode: + return instCombineInStreamingMode(IC, II); } return std::nullopt; @@ -2587,7 +2902,7 @@ AArch64TTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const { bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode, ArrayRef<const Value *> Args, - Type *SrcOverrideTy) { + Type *SrcOverrideTy) const { // A helper that returns a vector type from the given type. The number of // elements in type Ty determines the vector width. auto toVectorTy = [&](Type *ArgTy) { @@ -2684,7 +2999,7 @@ bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode, // trunc i16 (lshr (add %x, %y), 1) -> i8 // bool AArch64TTIImpl::isExtPartOfAvgExpr(const Instruction *ExtUser, Type *Dst, - Type *Src) { + Type *Src) const { // The source should be a legal vector type. if (!Src->isVectorTy() || !TLI->isTypeLegal(TLI->getValueType(DL, Src)) || (Src->isScalableTy() && !ST->hasSVE2())) @@ -2729,7 +3044,7 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, TTI::CastContextHint CCH, TTI::TargetCostKind CostKind, - const Instruction *I) { + const Instruction *I) const { int ISD = TLI->InstructionOpcodeToISD(Opcode); assert(ISD && "Invalid opcode"); // If the cast is observable, and it is used by a widening instruction (e.g., @@ -2785,6 +3100,17 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, BF16Tbl, ISD, DstTy.getSimpleVT(), SrcTy.getSimpleVT())) return AdjustCost(Entry->Cost); + // Symbolic constants for the SVE sitofp/uitofp entries in the table below + // The cost of unpacking twice is artificially increased for now in order + // to avoid regressions against NEON, which will use tbl instructions directly + // instead of multiple layers of [s|u]unpk[lo|hi]. + // We use the unpacks in cases where the destination type is illegal and + // requires splitting of the input, even if the input type itself is legal. + const unsigned int SVE_EXT_COST = 1; + const unsigned int SVE_FCVT_COST = 1; + const unsigned int SVE_UNPACK_ONCE = 4; + const unsigned int SVE_UNPACK_TWICE = 16; + static const TypeConversionCostTblEntry ConversionTbl[] = { {ISD::TRUNCATE, MVT::v2i8, MVT::v2i64, 1}, // xtn {ISD::TRUNCATE, MVT::v2i16, MVT::v2i64, 1}, // xtn @@ -2910,13 +3236,59 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i32, 1}, {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i64, 1}, + // SVE: to nxv2f16 + {ISD::SINT_TO_FP, MVT::nxv2f16, MVT::nxv2i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv2f16, MVT::nxv2i16, SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv2f16, MVT::nxv2i32, SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv2f16, MVT::nxv2i64, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f16, MVT::nxv2i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f16, MVT::nxv2i16, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f16, MVT::nxv2i32, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f16, MVT::nxv2i64, SVE_FCVT_COST}, + + // SVE: to nxv4f16 + {ISD::SINT_TO_FP, MVT::nxv4f16, MVT::nxv4i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv4f16, MVT::nxv4i16, SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv4f16, MVT::nxv4i32, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv4f16, MVT::nxv4i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv4f16, MVT::nxv4i16, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv4f16, MVT::nxv4i32, SVE_FCVT_COST}, + + // SVE: to nxv8f16 + {ISD::SINT_TO_FP, MVT::nxv8f16, MVT::nxv8i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv8f16, MVT::nxv8i16, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv8f16, MVT::nxv8i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv8f16, MVT::nxv8i16, SVE_FCVT_COST}, + + // SVE: to nxv16f16 + {ISD::SINT_TO_FP, MVT::nxv16f16, MVT::nxv16i8, + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv16f16, MVT::nxv16i8, + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + // Complex: to v2f32 {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i8, 3}, {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i16, 3}, - {ISD::SINT_TO_FP, MVT::v2f32, MVT::v2i64, 2}, {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i8, 3}, {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i16, 3}, - {ISD::UINT_TO_FP, MVT::v2f32, MVT::v2i64, 2}, + + // SVE: to nxv2f32 + {ISD::SINT_TO_FP, MVT::nxv2f32, MVT::nxv2i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv2f32, MVT::nxv2i16, SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv2f32, MVT::nxv2i32, SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv2f32, MVT::nxv2i64, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f32, MVT::nxv2i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f32, MVT::nxv2i16, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f32, MVT::nxv2i32, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f32, MVT::nxv2i64, SVE_FCVT_COST}, // Complex: to v4f32 {ISD::SINT_TO_FP, MVT::v4f32, MVT::v4i8, 4}, @@ -2924,12 +3296,38 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i8, 3}, {ISD::UINT_TO_FP, MVT::v4f32, MVT::v4i16, 2}, + // SVE: to nxv4f32 + {ISD::SINT_TO_FP, MVT::nxv4f32, MVT::nxv4i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv4f32, MVT::nxv4i16, SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv4f32, MVT::nxv4i32, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv4f32, MVT::nxv4i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv4f32, MVT::nxv4i16, SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv4f32, MVT::nxv4i32, SVE_FCVT_COST}, + // Complex: to v8f32 {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i8, 10}, {ISD::SINT_TO_FP, MVT::v8f32, MVT::v8i16, 4}, {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i8, 10}, {ISD::UINT_TO_FP, MVT::v8f32, MVT::v8i16, 4}, + // SVE: to nxv8f32 + {ISD::SINT_TO_FP, MVT::nxv8f32, MVT::nxv8i8, + SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv8f32, MVT::nxv8i16, + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv8f32, MVT::nxv8i8, + SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv8f32, MVT::nxv8i16, + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + + // SVE: to nxv16f32 + {ISD::SINT_TO_FP, MVT::nxv16f32, MVT::nxv16i8, + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv16f32, MVT::nxv16i8, + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST}, + // Complex: to v16f32 {ISD::SINT_TO_FP, MVT::v16f32, MVT::v16i8, 21}, {ISD::UINT_TO_FP, MVT::v16f32, MVT::v16i8, 21}, @@ -2942,10 +3340,46 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i16, 4}, {ISD::UINT_TO_FP, MVT::v2f64, MVT::v2i32, 2}, + // SVE: to nxv2f64 + {ISD::SINT_TO_FP, MVT::nxv2f64, MVT::nxv2i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv2f64, MVT::nxv2i16, SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv2f64, MVT::nxv2i32, SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv2f64, MVT::nxv2i64, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f64, MVT::nxv2i8, + SVE_EXT_COST + SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f64, MVT::nxv2i16, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f64, MVT::nxv2i32, SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv2f64, MVT::nxv2i64, SVE_FCVT_COST}, + // Complex: to v4f64 {ISD::SINT_TO_FP, MVT::v4f64, MVT::v4i32, 4}, {ISD::UINT_TO_FP, MVT::v4f64, MVT::v4i32, 4}, + // SVE: to nxv4f64 + {ISD::SINT_TO_FP, MVT::nxv4f64, MVT::nxv4i8, + SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv4f64, MVT::nxv4i16, + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv4f64, MVT::nxv4i32, + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv4f64, MVT::nxv4i8, + SVE_EXT_COST + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv4f64, MVT::nxv4i16, + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv4f64, MVT::nxv4i32, + SVE_UNPACK_ONCE + 2 * SVE_FCVT_COST}, + + // SVE: to nxv8f64 + {ISD::SINT_TO_FP, MVT::nxv8f64, MVT::nxv8i8, + SVE_EXT_COST + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST}, + {ISD::SINT_TO_FP, MVT::nxv8f64, MVT::nxv8i16, + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv8f64, MVT::nxv8i8, + SVE_EXT_COST + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST}, + {ISD::UINT_TO_FP, MVT::nxv8f64, MVT::nxv8i16, + SVE_UNPACK_TWICE + 4 * SVE_FCVT_COST}, + // LowerVectorFP_TO_INT {ISD::FP_TO_SINT, MVT::v2i32, MVT::v2f32, 1}, {ISD::FP_TO_SINT, MVT::v4i32, MVT::v4f32, 1}, @@ -2991,20 +3425,24 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, {ISD::FP_TO_SINT, MVT::nxv2i32, MVT::nxv2f64, 1}, {ISD::FP_TO_SINT, MVT::nxv2i16, MVT::nxv2f64, 1}, {ISD::FP_TO_SINT, MVT::nxv2i8, MVT::nxv2f64, 1}, + {ISD::FP_TO_SINT, MVT::nxv2i1, MVT::nxv2f64, 1}, {ISD::FP_TO_UINT, MVT::nxv2i64, MVT::nxv2f64, 1}, {ISD::FP_TO_UINT, MVT::nxv2i32, MVT::nxv2f64, 1}, {ISD::FP_TO_UINT, MVT::nxv2i16, MVT::nxv2f64, 1}, {ISD::FP_TO_UINT, MVT::nxv2i8, MVT::nxv2f64, 1}, + {ISD::FP_TO_UINT, MVT::nxv2i1, MVT::nxv2f64, 1}, // Complex, from nxv4f32. {ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f32, 4}, {ISD::FP_TO_SINT, MVT::nxv4i32, MVT::nxv4f32, 1}, {ISD::FP_TO_SINT, MVT::nxv4i16, MVT::nxv4f32, 1}, {ISD::FP_TO_SINT, MVT::nxv4i8, MVT::nxv4f32, 1}, + {ISD::FP_TO_SINT, MVT::nxv4i1, MVT::nxv4f32, 1}, {ISD::FP_TO_UINT, MVT::nxv4i64, MVT::nxv4f32, 4}, {ISD::FP_TO_UINT, MVT::nxv4i32, MVT::nxv4f32, 1}, {ISD::FP_TO_UINT, MVT::nxv4i16, MVT::nxv4f32, 1}, {ISD::FP_TO_UINT, MVT::nxv4i8, MVT::nxv4f32, 1}, + {ISD::FP_TO_UINT, MVT::nxv4i1, MVT::nxv4f32, 1}, // Complex, from nxv8f64. Illegal -> illegal conversions not required. {ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f64, 7}, @@ -3031,10 +3469,12 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, {ISD::FP_TO_SINT, MVT::nxv8i32, MVT::nxv8f16, 4}, {ISD::FP_TO_SINT, MVT::nxv8i16, MVT::nxv8f16, 1}, {ISD::FP_TO_SINT, MVT::nxv8i8, MVT::nxv8f16, 1}, + {ISD::FP_TO_SINT, MVT::nxv8i1, MVT::nxv8f16, 1}, {ISD::FP_TO_UINT, MVT::nxv8i64, MVT::nxv8f16, 10}, {ISD::FP_TO_UINT, MVT::nxv8i32, MVT::nxv8f16, 4}, {ISD::FP_TO_UINT, MVT::nxv8i16, MVT::nxv8f16, 1}, {ISD::FP_TO_UINT, MVT::nxv8i8, MVT::nxv8f16, 1}, + {ISD::FP_TO_UINT, MVT::nxv8i1, MVT::nxv8f16, 1}, // Complex, from nxv4f16. {ISD::FP_TO_SINT, MVT::nxv4i64, MVT::nxv4f16, 4}, @@ -3167,6 +3607,20 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, FP16Tbl, ISD, DstTy.getSimpleVT(), SrcTy.getSimpleVT())) return AdjustCost(Entry->Cost); + // INT_TO_FP of i64->f32 will scalarize, which is required to avoid + // double-rounding issues. + if ((ISD == ISD::SINT_TO_FP || ISD == ISD::UINT_TO_FP) && + DstTy.getScalarType() == MVT::f32 && SrcTy.getScalarSizeInBits() > 32 && + isa<FixedVectorType>(Dst) && isa<FixedVectorType>(Src)) + return AdjustCost( + cast<FixedVectorType>(Dst)->getNumElements() * + getCastInstrCost(Opcode, Dst->getScalarType(), Src->getScalarType(), + CCH, CostKind) + + BaseT::getScalarizationOverhead(cast<FixedVectorType>(Src), false, true, + CostKind) + + BaseT::getScalarizationOverhead(cast<FixedVectorType>(Dst), true, false, + CostKind)); + if ((ISD == ISD::ZERO_EXTEND || ISD == ISD::SIGN_EXTEND) && CCH == TTI::CastContextHint::Masked && ST->isSVEorStreamingSVEAvailable() && @@ -3198,10 +3652,10 @@ InstructionCost AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I)); } -InstructionCost AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, - Type *Dst, - VectorType *VecTy, - unsigned Index) { +InstructionCost +AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst, + VectorType *VecTy, unsigned Index, + TTI::TargetCostKind CostKind) const { // Make sure we were given a valid extend opcode. assert((Opcode == Instruction::SExt || Opcode == Instruction::ZExt) && @@ -3216,7 +3670,6 @@ InstructionCost AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, // Get the cost for the extract. We compute the cost (if any) for the extend // below. - TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; InstructionCost Cost = getVectorInstrCost(Instruction::ExtractElement, VecTy, CostKind, Index, nullptr, nullptr); @@ -3261,7 +3714,7 @@ InstructionCost AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode, TTI::TargetCostKind CostKind, - const Instruction *I) { + const Instruction *I) const { if (CostKind != TTI::TCK_RecipThroughput) return Opcode == Instruction::PHI ? 0 : 1; assert(CostKind == TTI::TCK_RecipThroughput && "unexpected CostKind"); @@ -3270,9 +3723,9 @@ InstructionCost AArch64TTIImpl::getCFInstrCost(unsigned Opcode, } InstructionCost AArch64TTIImpl::getVectorInstrCostHelper( - unsigned Opcode, Type *Val, unsigned Index, bool HasRealUse, - const Instruction *I, Value *Scalar, - ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) { + unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, + bool HasRealUse, const Instruction *I, Value *Scalar, + ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const { assert(Val->isVectorTy() && "This must be a vector type"); if (Index != -1U) { @@ -3304,12 +3757,16 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper( // and its second operand is a load, then we will generate a LD1, which // are expensive instructions. if (I && dyn_cast<LoadInst>(I->getOperand(1))) - return ST->getVectorInsertExtractBaseCost() + 1; + return CostKind == TTI::TCK_CodeSize + ? 0 + : ST->getVectorInsertExtractBaseCost() + 1; // i1 inserts and extract will include an extra cset or cmp of the vector // value. Increase the cost by 1 to account. if (Val->getScalarSizeInBits() == 1) - return ST->getVectorInsertExtractBaseCost() + 1; + return CostKind == TTI::TCK_CodeSize + ? 2 + : ST->getVectorInsertExtractBaseCost() + 1; // FIXME: // If the extract-element and insert-element instructions could be @@ -3380,7 +3837,7 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper( return false; for (auto &[S, U, L] : ScalarUserAndIdx) { for (auto *U : S->users()) { - if (UserToExtractIdx.find(U) != UserToExtractIdx.end()) { + if (UserToExtractIdx.contains(U)) { auto *FMul = cast<BinaryOperator>(U); auto *Op0 = FMul->getOperand(0); auto *Op1 = FMul->getOperand(1); @@ -3433,51 +3890,54 @@ InstructionCost AArch64TTIImpl::getVectorInstrCostHelper( return 0; // All other insert/extracts cost this much. - return ST->getVectorInsertExtractBaseCost(); + return CostKind == TTI::TCK_CodeSize ? 1 + : ST->getVectorInsertExtractBaseCost(); } InstructionCost AArch64TTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, - unsigned Index, Value *Op0, - Value *Op1) { + unsigned Index, + const Value *Op0, + const Value *Op1) const { bool HasRealUse = Opcode == Instruction::InsertElement && Op0 && !isa<UndefValue>(Op0); - return getVectorInstrCostHelper(Opcode, Val, Index, HasRealUse); + return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, HasRealUse); } InstructionCost AArch64TTIImpl::getVectorInstrCost( unsigned Opcode, Type *Val, TTI::TargetCostKind CostKind, unsigned Index, Value *Scalar, - ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) { - return getVectorInstrCostHelper(Opcode, Val, Index, false, nullptr, Scalar, - ScalarUserAndIdx); + ArrayRef<std::tuple<Value *, User *, int>> ScalarUserAndIdx) const { + return getVectorInstrCostHelper(Opcode, Val, CostKind, Index, false, nullptr, + Scalar, ScalarUserAndIdx); } InstructionCost AArch64TTIImpl::getVectorInstrCost(const Instruction &I, Type *Val, TTI::TargetCostKind CostKind, - unsigned Index) { - return getVectorInstrCostHelper(I.getOpcode(), Val, Index, + unsigned Index) const { + return getVectorInstrCostHelper(I.getOpcode(), Val, CostKind, Index, true /* HasRealUse */, &I); } InstructionCost AArch64TTIImpl::getScalarizationOverhead( VectorType *Ty, const APInt &DemandedElts, bool Insert, bool Extract, - TTI::TargetCostKind CostKind, ArrayRef<Value *> VL) { + TTI::TargetCostKind CostKind, bool ForPoisonSrc, + ArrayRef<Value *> VL) const { if (isa<ScalableVectorType>(Ty)) return InstructionCost::getInvalid(); if (Ty->getElementType()->isFloatingPointTy()) return BaseT::getScalarizationOverhead(Ty, DemandedElts, Insert, Extract, CostKind); - return DemandedElts.popcount() * (Insert + Extract) * - ST->getVectorInsertExtractBaseCost(); + unsigned VecInstCost = + CostKind == TTI::TCK_CodeSize ? 1 : ST->getVectorInsertExtractBaseCost(); + return DemandedElts.popcount() * (Insert + Extract) * VecInstCost; } InstructionCost AArch64TTIImpl::getArithmeticInstrCost( unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info, TTI::OperandValueInfo Op2Info, - ArrayRef<const Value *> Args, - const Instruction *CxtI) { + ArrayRef<const Value *> Args, const Instruction *CxtI) const { // The code-generator is currently not able to handle scalable vectors // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting @@ -3500,39 +3960,157 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost( default: return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info); + case ISD::SREM: case ISD::SDIV: - if (Op2Info.isConstant() && Op2Info.isUniform() && Op2Info.isPowerOf2()) { - // On AArch64, scalar signed division by constants power-of-two are - // normally expanded to the sequence ADD + CMP + SELECT + SRA. - // The OperandValue properties many not be same as that of previous - // operation; conservatively assume OP_None. - InstructionCost Cost = getArithmeticInstrCost( - Instruction::Add, Ty, CostKind, - Op1Info.getNoProps(), Op2Info.getNoProps()); - Cost += getArithmeticInstrCost(Instruction::Sub, Ty, CostKind, - Op1Info.getNoProps(), Op2Info.getNoProps()); - Cost += getArithmeticInstrCost( - Instruction::Select, Ty, CostKind, - Op1Info.getNoProps(), Op2Info.getNoProps()); - Cost += getArithmeticInstrCost(Instruction::AShr, Ty, CostKind, - Op1Info.getNoProps(), Op2Info.getNoProps()); - return Cost; + /* + Notes for sdiv/srem specific costs: + 1. This only considers the cases where the divisor is constant, uniform and + (pow-of-2/non-pow-of-2). Other cases are not important since they either + result in some form of (ldr + adrp), corresponding to constant vectors, or + scalarization of the division operation. + 2. Constant divisors, either negative in whole or partially, don't result in + significantly different codegen as compared to positive constant divisors. + So, we don't consider negative divisors separately. + 3. If the codegen is significantly different with SVE, it has been indicated + using comments at appropriate places. + + sdiv specific cases: + ----------------------------------------------------------------------- + codegen | pow-of-2 | Type + ----------------------------------------------------------------------- + add + cmp + csel + asr | Y | i64 + add + cmp + csel + asr | Y | i32 + ----------------------------------------------------------------------- + + srem specific cases: + ----------------------------------------------------------------------- + codegen | pow-of-2 | Type + ----------------------------------------------------------------------- + negs + and + and + csneg | Y | i64 + negs + and + and + csneg | Y | i32 + ----------------------------------------------------------------------- + + other sdiv/srem cases: + ------------------------------------------------------------------------- + common codegen | + srem | + sdiv | pow-of-2 | Type + ------------------------------------------------------------------------- + smulh + asr + add + add | - | - | N | i64 + smull + lsr + add + add | - | - | N | i32 + usra | and + sub | sshr | Y | <2 x i64> + 2 * (scalar code) | - | - | N | <2 x i64> + usra | bic + sub | sshr + neg | Y | <4 x i32> + smull2 + smull + uzp2 | mls | - | N | <4 x i32> + + sshr + usra | | | | + ------------------------------------------------------------------------- + */ + if (Op2Info.isConstant() && Op2Info.isUniform()) { + InstructionCost AddCost = + getArithmeticInstrCost(Instruction::Add, Ty, CostKind, + Op1Info.getNoProps(), Op2Info.getNoProps()); + InstructionCost AsrCost = + getArithmeticInstrCost(Instruction::AShr, Ty, CostKind, + Op1Info.getNoProps(), Op2Info.getNoProps()); + InstructionCost MulCost = + getArithmeticInstrCost(Instruction::Mul, Ty, CostKind, + Op1Info.getNoProps(), Op2Info.getNoProps()); + // add/cmp/csel/csneg should have similar cost while asr/negs/and should + // have similar cost. + auto VT = TLI->getValueType(DL, Ty); + if (VT.isScalarInteger() && VT.getSizeInBits() <= 64) { + if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) { + // Neg can be folded into the asr instruction. + return ISD == ISD::SDIV ? (3 * AddCost + AsrCost) + : (3 * AsrCost + AddCost); + } else { + return MulCost + AsrCost + 2 * AddCost; + } + } else if (VT.isVector()) { + InstructionCost UsraCost = 2 * AsrCost; + if (Op2Info.isPowerOf2() || Op2Info.isNegatedPowerOf2()) { + // Division with scalable types corresponds to native 'asrd' + // instruction when SVE is available. + // e.g. %1 = sdiv <vscale x 4 x i32> %a, splat (i32 8) + + // One more for the negation in SDIV + InstructionCost Cost = + (Op2Info.isNegatedPowerOf2() && ISD == ISD::SDIV) ? AsrCost : 0; + if (Ty->isScalableTy() && ST->hasSVE()) + Cost += 2 * AsrCost; + else { + Cost += + UsraCost + + (ISD == ISD::SDIV + ? (LT.second.getScalarType() == MVT::i64 ? 1 : 2) * AsrCost + : 2 * AddCost); + } + return Cost; + } else if (LT.second == MVT::v2i64) { + return VT.getVectorNumElements() * + getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind, + Op1Info.getNoProps(), + Op2Info.getNoProps()); + } else { + // When SVE is available, we get: + // smulh + lsr + add/sub + asr + add/sub. + if (Ty->isScalableTy() && ST->hasSVE()) + return MulCost /*smulh cost*/ + 2 * AddCost + 2 * AsrCost; + return 2 * MulCost + AddCost /*uzp2 cost*/ + AsrCost + UsraCost; + } + } + } + if (Op2Info.isConstant() && !Op2Info.isUniform() && + LT.second.isFixedLengthVector()) { + // FIXME: When the constant vector is non-uniform, this may result in + // loading the vector from constant pool or in some cases, may also result + // in scalarization. For now, we are approximating this with the + // scalarization cost. + auto ExtractCost = 2 * getVectorInstrCost(Instruction::ExtractElement, Ty, + CostKind, -1, nullptr, nullptr); + auto InsertCost = getVectorInstrCost(Instruction::InsertElement, Ty, + CostKind, -1, nullptr, nullptr); + unsigned NElts = cast<FixedVectorType>(Ty)->getNumElements(); + return ExtractCost + InsertCost + + NElts * getArithmeticInstrCost(Opcode, Ty->getScalarType(), + CostKind, Op1Info.getNoProps(), + Op2Info.getNoProps()); } [[fallthrough]]; - case ISD::UDIV: { + case ISD::UDIV: + case ISD::UREM: { auto VT = TLI->getValueType(DL, Ty); - if (Op2Info.isConstant() && Op2Info.isUniform()) { - if (TLI->isOperationLegalOrCustom(ISD::MULHU, VT)) { - // Vector signed division by constant are expanded to the - // sequence MULHS + ADD/SUB + SRA + SRL + ADD, and unsigned division - // to MULHS + SUB + SRL + ADD + SRL. - InstructionCost MulCost = getArithmeticInstrCost( - Instruction::Mul, Ty, CostKind, Op1Info.getNoProps(), Op2Info.getNoProps()); - InstructionCost AddCost = getArithmeticInstrCost( - Instruction::Add, Ty, CostKind, Op1Info.getNoProps(), Op2Info.getNoProps()); - InstructionCost ShrCost = getArithmeticInstrCost( - Instruction::AShr, Ty, CostKind, Op1Info.getNoProps(), Op2Info.getNoProps()); - return MulCost * 2 + AddCost * 2 + ShrCost * 2 + 1; + if (Op2Info.isConstant()) { + // If the operand is a power of 2 we can use the shift or and cost. + if (ISD == ISD::UDIV && Op2Info.isPowerOf2()) + return getArithmeticInstrCost(Instruction::LShr, Ty, CostKind, + Op1Info.getNoProps(), + Op2Info.getNoProps()); + if (ISD == ISD::UREM && Op2Info.isPowerOf2()) + return getArithmeticInstrCost(Instruction::And, Ty, CostKind, + Op1Info.getNoProps(), + Op2Info.getNoProps()); + + if (ISD == ISD::UDIV || ISD == ISD::UREM) { + // Divides by a constant are expanded to MULHU + SUB + SRL + ADD + SRL. + // The MULHU will be expanded to UMULL for the types not listed below, + // and will become a pair of UMULL+MULL2 for 128bit vectors. + bool HasMULH = VT == MVT::i64 || LT.second == MVT::nxv2i64 || + LT.second == MVT::nxv4i32 || LT.second == MVT::nxv8i16 || + LT.second == MVT::nxv16i8; + bool Is128bit = LT.second.is128BitVector(); + + InstructionCost MulCost = + getArithmeticInstrCost(Instruction::Mul, Ty, CostKind, + Op1Info.getNoProps(), Op2Info.getNoProps()); + InstructionCost AddCost = + getArithmeticInstrCost(Instruction::Add, Ty, CostKind, + Op1Info.getNoProps(), Op2Info.getNoProps()); + InstructionCost ShrCost = + getArithmeticInstrCost(Instruction::AShr, Ty, CostKind, + Op1Info.getNoProps(), Op2Info.getNoProps()); + InstructionCost DivCost = MulCost * (Is128bit ? 2 : 1) + // UMULL/UMULH + (HasMULH ? 0 : ShrCost) + // UMULL shift + AddCost * 2 + ShrCost; + return DivCost + (ISD == ISD::UREM ? MulCost + AddCost : 0); } } @@ -3544,14 +4122,12 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost( InstructionCost Cost = BaseT::getArithmeticInstrCost( Opcode, Ty, CostKind, Op1Info, Op2Info); - if (Ty->isVectorTy()) { + if (Ty->isVectorTy() && (ISD == ISD::SDIV || ISD == ISD::UDIV)) { if (TLI->isOperationLegalOrCustom(ISD, LT.second) && ST->hasSVE()) { // SDIV/UDIV operations are lowered using SVE, then we can have less // costs. - if (isa<FixedVectorType>(Ty) && cast<FixedVectorType>(Ty) - ->getPrimitiveSizeInBits() - .getFixedValue() < 128) { - EVT VT = TLI->getValueType(DL, Ty); + if (VT.isSimple() && isa<FixedVectorType>(Ty) && + Ty->getPrimitiveSizeInBits().getFixedValue() < 128) { static const CostTblEntry DivTbl[]{ {ISD::SDIV, MVT::v2i8, 5}, {ISD::SDIV, MVT::v4i8, 8}, {ISD::SDIV, MVT::v8i8, 8}, {ISD::SDIV, MVT::v2i16, 5}, @@ -3586,10 +4162,10 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost( } // On AArch64, without SVE, vector divisions are expanded // into scalar divisions of each pair of elements. - Cost += getArithmeticInstrCost(Instruction::ExtractElement, Ty, - CostKind, Op1Info, Op2Info); - Cost += getArithmeticInstrCost(Instruction::InsertElement, Ty, CostKind, - Op1Info, Op2Info); + Cost += getVectorInstrCost(Instruction::ExtractElement, Ty, CostKind, + -1, nullptr, nullptr); + Cost += getVectorInstrCost(Instruction::InsertElement, Ty, CostKind, -1, + nullptr, nullptr); } // TODO: if one of the arguments is scalar, then it's not necessary to @@ -3675,9 +4251,9 @@ InstructionCost AArch64TTIImpl::getArithmeticInstrCost( } } -InstructionCost AArch64TTIImpl::getAddressComputationCost(Type *Ty, - ScalarEvolution *SE, - const SCEV *Ptr) { +InstructionCost +AArch64TTIImpl::getAddressComputationCost(Type *Ty, ScalarEvolution *SE, + const SCEV *Ptr) const { // Address computations in vectorized code with non-consecutive addresses will // likely result in more instructions compared to scalar code where the // computation can more often be merged into the index mode. The resulting @@ -3697,15 +4273,10 @@ InstructionCost AArch64TTIImpl::getAddressComputationCost(Type *Ty, InstructionCost AArch64TTIImpl::getCmpSelInstrCost( unsigned Opcode, Type *ValTy, Type *CondTy, CmpInst::Predicate VecPred, TTI::TargetCostKind CostKind, TTI::OperandValueInfo Op1Info, - TTI::OperandValueInfo Op2Info, const Instruction *I) { - // TODO: Handle other cost kinds. - if (CostKind != TTI::TCK_RecipThroughput) - return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, - Op1Info, Op2Info, I); - + TTI::OperandValueInfo Op2Info, const Instruction *I) const { int ISD = TLI->InstructionOpcodeToISD(Opcode); // We don't lower some vector selects well that are wider than the register - // width. + // width. TODO: Improve this with different cost kinds. if (isa<FixedVectorType>(ValTy) && ISD == ISD::SELECT) { // We would need this many instructions to hide the scalarization happening. const int AmortizationCost = 20; @@ -3762,21 +4333,52 @@ InstructionCost AArch64TTIImpl::getCmpSelInstrCost( } if (isa<FixedVectorType>(ValTy) && ISD == ISD::SETCC) { - auto LT = getTypeLegalizationCost(ValTy); - // Cost v4f16 FCmp without FP16 support via converting to v4f32 and back. - if (LT.second == MVT::v4f16 && !ST->hasFullFP16()) - return LT.first * 4; // fcvtl + fcvtl + fcmp + xtn + Type *ValScalarTy = ValTy->getScalarType(); + if ((ValScalarTy->isHalfTy() && !ST->hasFullFP16()) || + ValScalarTy->isBFloatTy()) { + auto *ValVTy = cast<FixedVectorType>(ValTy); + + // Without dedicated instructions we promote [b]f16 compares to f32. + auto *PromotedTy = + VectorType::get(Type::getFloatTy(ValTy->getContext()), ValVTy); + + InstructionCost Cost = 0; + // Promote operands to float vectors. + Cost += 2 * getCastInstrCost(Instruction::FPExt, PromotedTy, ValTy, + TTI::CastContextHint::None, CostKind); + // Compare float vectors. + Cost += getCmpSelInstrCost(Opcode, PromotedTy, CondTy, VecPred, CostKind, + Op1Info, Op2Info); + // During codegen we'll truncate the vector result from i32 to i16. + Cost += + getCastInstrCost(Instruction::Trunc, VectorType::getInteger(ValVTy), + VectorType::getInteger(PromotedTy), + TTI::CastContextHint::None, CostKind); + return Cost; + } } - // Treat the icmp in icmp(and, 0) as free, as we can make use of ands. - // FIXME: This can apply to more conditions and add/sub if it can be shown to - // be profitable. - if (ValTy->isIntegerTy() && ISD == ISD::SETCC && I && - ICmpInst::isEquality(VecPred) && + // Treat the icmp in icmp(and, 0) or icmp(and, -1/1) when it can be folded to + // icmp(and, 0) as free, as we can make use of ands, but only if the + // comparison is not unsigned. FIXME: Enable for non-throughput cost kinds + // providing it will not cause performance regressions. + if (CostKind == TTI::TCK_RecipThroughput && ValTy->isIntegerTy() && + ISD == ISD::SETCC && I && !CmpInst::isUnsigned(VecPred) && TLI->isTypeLegal(TLI->getValueType(DL, ValTy)) && - match(I->getOperand(1), m_Zero()) && - match(I->getOperand(0), m_And(m_Value(), m_Value()))) - return 0; + match(I->getOperand(0), m_And(m_Value(), m_Value()))) { + if (match(I->getOperand(1), m_Zero())) + return 0; + + // x >= 1 / x < 1 -> x > 0 / x <= 0 + if (match(I->getOperand(1), m_One()) && + (VecPred == CmpInst::ICMP_SLT || VecPred == CmpInst::ICMP_SGE)) + return 0; + + // x <= -1 / x > -1 -> x > 0 / x <= 0 + if (match(I->getOperand(1), m_AllOnes()) && + (VecPred == CmpInst::ICMP_SLE || VecPred == CmpInst::ICMP_SGT)) + return 0; + } // The base case handles scalable vectors fine for now, since it treats the // cost as 1 * legalization cost. @@ -3810,7 +4412,7 @@ bool AArch64TTIImpl::prefersVectorizedAddressing() const { InstructionCost AArch64TTIImpl::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, - TTI::TargetCostKind CostKind) { + TTI::TargetCostKind CostKind) const { if (useNeonVector(Src)) return BaseT::getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind); @@ -3857,7 +4459,7 @@ static unsigned getSVEGatherScatterOverhead(unsigned Opcode, InstructionCost AArch64TTIImpl::getGatherScatterOpCost( unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, - Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) { + Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { if (useNeonVector(DataTy) || !isLegalMaskedGatherScatter(DataTy)) return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, Alignment, CostKind, I); @@ -3893,11 +4495,11 @@ bool AArch64TTIImpl::useNeonVector(const Type *Ty) const { } InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty, - MaybeAlign Alignment, + Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, TTI::OperandValueInfo OpInfo, - const Instruction *I) { + const Instruction *I) const { EVT VT = TLI->getValueType(DL, Ty, true); // Type legalization can't handle structs if (VT == MVT::Other) @@ -3928,7 +4530,7 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty, return 1; if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store && - LT.second.is128BitVector() && (!Alignment || *Alignment < Align(16))) { + LT.second.is128BitVector() && Alignment < Align(16)) { // Unaligned stores are extremely inefficient. We don't split all // unaligned 128-bit stores because the negative impact that has shown in // practice on inlined block copy code. @@ -3955,8 +4557,7 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty, EVT EltVT = VT.getVectorElementType(); unsigned EltSize = EltVT.getScalarSizeInBits(); if (!isPowerOf2_32(EltSize) || EltSize < 8 || EltSize > 64 || - VT.getVectorNumElements() >= (128 / EltSize) || !Alignment || - *Alignment != Align(1)) + VT.getVectorNumElements() >= (128 / EltSize) || Alignment != Align(1)) return LT.first; // FIXME: v3i8 lowering currently is very inefficient, due to automatic // widening to v4i8, which produces suboptimal results. @@ -3992,13 +4593,20 @@ InstructionCost AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty, InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, - bool UseMaskForCond, bool UseMaskForGaps) { + bool UseMaskForCond, bool UseMaskForGaps) const { assert(Factor >= 2 && "Invalid interleave factor"); auto *VecVTy = cast<VectorType>(VecTy); if (VecTy->isScalableTy() && !ST->hasSVE()) return InstructionCost::getInvalid(); + // Scalable VFs will emit vector.[de]interleave intrinsics, and currently we + // only have lowering for power-of-2 factors. + // TODO: Add lowering for vector.[de]interleave3 intrinsics and support in + // InterleavedAccessPass for ld3/st3 + if (VecTy->isScalableTy() && !isPowerOf2_32(Factor)) + return InstructionCost::getInvalid(); + // Vectorization for masked interleaved accesses is only enabled for scalable // VF. if (!VecTy->isScalableTy() && (UseMaskForCond || UseMaskForGaps)) @@ -4025,7 +4633,7 @@ InstructionCost AArch64TTIImpl::getInterleavedMemoryOpCost( } InstructionCost -AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) { +AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) const { InstructionCost Cost = 0; TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; for (auto *I : Tys) { @@ -4039,7 +4647,7 @@ AArch64TTIImpl::getCostOfKeepingLiveOverCall(ArrayRef<Type *> Tys) { return Cost; } -unsigned AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF) { +unsigned AArch64TTIImpl::getMaxInterleaveFactor(ElementCount VF) const { return ST->getMaxInterleaveFactor(); } @@ -4095,39 +4703,100 @@ getFalkorUnrollingPreferences(Loop *L, ScalarEvolution &SE, } } +// This function returns true if the loop: +// 1. Has a valid cost, and +// 2. Has a cost within the supplied budget. +// Otherwise it returns false. +static bool isLoopSizeWithinBudget(Loop *L, const AArch64TTIImpl &TTI, + InstructionCost Budget, + unsigned *FinalSize) { + // Estimate the size of the loop. + InstructionCost LoopCost = 0; + + for (auto *BB : L->getBlocks()) { + for (auto &I : *BB) { + SmallVector<const Value *, 4> Operands(I.operand_values()); + InstructionCost Cost = + TTI.getInstructionCost(&I, Operands, TTI::TCK_CodeSize); + // This can happen with intrinsics that don't currently have a cost model + // or for some operations that require SVE. + if (!Cost.isValid()) + return false; + + LoopCost += Cost; + if (LoopCost > Budget) + return false; + } + } + + if (FinalSize) + *FinalSize = LoopCost.getValue(); + return true; +} + +static bool shouldUnrollMultiExitLoop(Loop *L, ScalarEvolution &SE, + const AArch64TTIImpl &TTI) { + // Only consider loops with unknown trip counts for which we can determine + // a symbolic expression. Multi-exit loops with small known trip counts will + // likely be unrolled anyway. + const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L); + if (isa<SCEVConstant>(BTC) || isa<SCEVCouldNotCompute>(BTC)) + return false; + + // It might not be worth unrolling loops with low max trip counts. Restrict + // this to max trip counts > 32 for now. + unsigned MaxTC = SE.getSmallConstantMaxTripCount(L); + if (MaxTC > 0 && MaxTC <= 32) + return false; + + // Make sure the loop size is <= 5. + if (!isLoopSizeWithinBudget(L, TTI, 5, nullptr)) + return false; + + // Small search loops with multiple exits can be highly beneficial to unroll. + // We only care about loops with exactly two exiting blocks, although each + // block could jump to the same exit block. + ArrayRef<BasicBlock *> Blocks = L->getBlocks(); + if (Blocks.size() != 2) + return false; + + if (any_of(Blocks, [](BasicBlock *BB) { + return !isa<BranchInst>(BB->getTerminator()); + })) + return false; + + return true; +} + /// For Apple CPUs, we want to runtime-unroll loops to make better use if the /// OOO engine's wide instruction window and various predictors. static void getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE, TargetTransformInfo::UnrollingPreferences &UP, - AArch64TTIImpl &TTI) { + const AArch64TTIImpl &TTI) { // Limit loops with structure that is highly likely to benefit from runtime - // unrolling; that is we exclude outer loops, loops with multiple exits and - // many blocks (i.e. likely with complex control flow). Note that the - // heuristics here may be overly conservative and we err on the side of - // avoiding runtime unrolling rather than unroll excessively. They are all - // subject to further refinement. - if (!L->isInnermost() || !L->getExitBlock() || L->getNumBlocks() > 8) + // unrolling; that is we exclude outer loops and loops with many blocks (i.e. + // likely with complex control flow). Note that the heuristics here may be + // overly conservative and we err on the side of avoiding runtime unrolling + // rather than unroll excessively. They are all subject to further refinement. + if (!L->isInnermost() || L->getNumBlocks() > 8) + return; + + // Loops with multiple exits are handled by common code. + if (!L->getExitBlock()) return; - const SCEV *BTC = SE.getBackedgeTakenCount(L); + const SCEV *BTC = SE.getSymbolicMaxBackedgeTakenCount(L); if (isa<SCEVConstant>(BTC) || isa<SCEVCouldNotCompute>(BTC) || (SE.getSmallConstantMaxTripCount(L) > 0 && SE.getSmallConstantMaxTripCount(L) <= 32)) return; + if (findStringMetadataForLoop(L, "llvm.loop.isvectorized")) return; - int64_t Size = 0; - for (auto *BB : L->getBlocks()) { - for (auto &I : *BB) { - if (!isa<IntrinsicInst>(&I) && isa<CallBase>(&I)) - return; - SmallVector<const Value *, 4> Operands(I.operand_values()); - Size += - *TTI.getInstructionCost(&I, Operands, TTI::TCK_CodeSize).getValue(); - } - } + if (SE.getSymbolicMaxBackedgeTakenCount(L) != SE.getBackedgeTakenCount(L)) + return; // Limit to loops with trip counts that are cheap to expand. UP.SCEVExpansionBudget = 1; @@ -4136,7 +4805,9 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE, // dependencies, to expose more parallel memory access streams. BasicBlock *Header = L->getHeader(); if (Header == L->getLoopLatch()) { - if (Size > 8) + // Estimate the size of the loop. + unsigned Size; + if (!isLoopSizeWithinBudget(L, TTI, 8, &Size)) return; SmallPtrSet<Value *, 8> LoadedValues; @@ -4190,7 +4861,7 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE, auto *Latch = L->getLoopLatch(); SmallVector<BasicBlock *> Preds(predecessors(Latch)); if (!Term || !Term->isConditional() || Preds.size() == 1 || - none_of(Preds, [Header](BasicBlock *Pred) { return Header == Pred; }) || + !llvm::is_contained(Preds, Header) || none_of(Preds, [L](BasicBlock *Pred) { return L->contains(Pred); })) return; @@ -4216,9 +4887,9 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE, } } -void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, - TTI::UnrollingPreferences &UP, - OptimizationRemarkEmitter *ORE) { +void AArch64TTIImpl::getUnrollingPreferences( + Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP, + OptimizationRemarkEmitter *ORE) const { // Enable partial unrolling and runtime unrolling. BaseT::getUnrollingPreferences(L, SE, UP, ORE); @@ -4233,6 +4904,24 @@ void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, // Disable partial & runtime unrolling on -Os. UP.PartialOptSizeThreshold = 0; + // No need to unroll auto-vectorized loops + if (findStringMetadataForLoop(L, "llvm.loop.isvectorized")) + return; + + // Scan the loop: don't unroll loops with calls as this could prevent + // inlining. + for (auto *BB : L->getBlocks()) { + for (auto &I : *BB) { + if (isa<CallBase>(I)) { + if (isa<CallInst>(I) || isa<InvokeInst>(I)) + if (const Function *F = cast<CallBase>(I).getCalledFunction()) + if (!isLoweredToCall(F)) + continue; + return; + } + } + } + // Apply subtarget-specific unrolling preferences. switch (ST->getProcFamily()) { case AArch64Subtarget::AppleA14: @@ -4249,30 +4938,24 @@ void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, break; } - // Scan the loop: don't unroll loops with calls as this could prevent - // inlining. Don't unroll vector loops either, as they don't benefit much from - // unrolling. - for (auto *BB : L->getBlocks()) { - for (auto &I : *BB) { - // Don't unroll vectorised loop. - if (I.getType()->isVectorTy()) - return; - - if (isa<CallInst>(I) || isa<InvokeInst>(I)) { - if (const Function *F = cast<CallBase>(I).getCalledFunction()) { - if (!isLoweredToCall(F)) - continue; - } - return; - } - } + // If this is a small, multi-exit loop similar to something like std::find, + // then there is typically a performance improvement achieved by unrolling. + if (!L->getExitBlock() && shouldUnrollMultiExitLoop(L, SE, *this)) { + UP.RuntimeUnrollMultiExit = true; + UP.Runtime = true; + // Limit unroll count. + UP.DefaultUnrollRuntimeCount = 4; + // Allow slightly more costly trip-count expansion to catch search loops + // with pointer inductions. + UP.SCEVExpansionBudget = 5; + return; } // Enable runtime unrolling for in-order models // If mcpu is omitted, getProcFamily() returns AArch64Subtarget::Others, so by // checking for that case, we can ensure that the default behaviour is // unchanged - if (ST->getProcFamily() != AArch64Subtarget::Others && + if (ST->getProcFamily() != AArch64Subtarget::Generic && !ST->getSchedModel().isOutOfOrder()) { UP.Runtime = true; UP.Partial = true; @@ -4285,12 +4968,13 @@ void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE, } void AArch64TTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE, - TTI::PeelingPreferences &PP) { + TTI::PeelingPreferences &PP) const { BaseT::getPeelingPreferences(L, SE, PP); } Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, - Type *ExpectedType) { + Type *ExpectedType, + bool CanCreate) const { switch (Inst->getIntrinsicID()) { default: return nullptr; @@ -4299,7 +4983,7 @@ Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, case Intrinsic::aarch64_neon_st4: { // Create a struct type StructType *ST = dyn_cast<StructType>(ExpectedType); - if (!ST) + if (!CanCreate || !ST) return nullptr; unsigned NumElts = Inst->arg_size() - 1; if (ST->getNumElements() != NumElts) @@ -4326,7 +5010,7 @@ Value *AArch64TTIImpl::getOrCreateResultFromMemIntrinsic(IntrinsicInst *Inst, } bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, - MemIntrinsicInfo &Info) { + MemIntrinsicInfo &Info) const { switch (Inst->getIntrinsicID()) { default: break; @@ -4371,7 +5055,7 @@ bool AArch64TTIImpl::getTgtMemIntrinsic(IntrinsicInst *Inst, /// sext instructions that sign extended the same initial value. A getelementptr /// is considered as "complex" if it has more than 2 operands. bool AArch64TTIImpl::shouldConsiderAddressTypePromotion( - const Instruction &I, bool &AllowPromotionWithoutCommonHeader) { + const Instruction &I, bool &AllowPromotionWithoutCommonHeader) const { bool Considerable = false; AllowPromotionWithoutCommonHeader = false; if (!isa<SExtInst>(&I)) @@ -4419,8 +5103,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction( case RecurKind::FMin: case RecurKind::FMax: case RecurKind::FMulAdd: - case RecurKind::IAnyOf: - case RecurKind::FAnyOf: + case RecurKind::AnyOf: return true; default: return false; @@ -4430,7 +5113,7 @@ bool AArch64TTIImpl::isLegalToVectorizeReduction( InstructionCost AArch64TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, FastMathFlags FMF, - TTI::TargetCostKind CostKind) { + TTI::TargetCostKind CostKind) const { // The code-generator is currently not able to handle scalable vectors // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting // it. This change will be removed when code-generation for these types is @@ -4455,7 +5138,7 @@ AArch64TTIImpl::getMinMaxReductionCost(Intrinsic::ID IID, VectorType *Ty, } InstructionCost AArch64TTIImpl::getArithmeticReductionCostSVE( - unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) { + unsigned Opcode, VectorType *ValTy, TTI::TargetCostKind CostKind) const { std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(ValTy); InstructionCost LegalizationCost = 0; if (LT.first > 1) { @@ -4482,7 +5165,7 @@ InstructionCost AArch64TTIImpl::getArithmeticReductionCostSVE( InstructionCost AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy, std::optional<FastMathFlags> FMF, - TTI::TargetCostKind CostKind) { + TTI::TargetCostKind CostKind) const { // The code-generator is currently not able to handle scalable vectors // of <vscale x 1 x eltty> yet, so return an invalid cost to avoid selecting // it. This change will be removed when code-generation for these types is @@ -4531,6 +5214,7 @@ AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy, {ISD::ADD, MVT::v16i8, 2}, {ISD::ADD, MVT::v4i16, 2}, {ISD::ADD, MVT::v8i16, 2}, + {ISD::ADD, MVT::v2i32, 2}, {ISD::ADD, MVT::v4i32, 2}, {ISD::ADD, MVT::v2i64, 2}, {ISD::OR, MVT::v8i8, 15}, @@ -4609,7 +5293,57 @@ AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode, VectorType *ValTy, return BaseT::getArithmeticReductionCost(Opcode, ValTy, FMF, CostKind); } -InstructionCost AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index) { +InstructionCost AArch64TTIImpl::getExtendedReductionCost( + unsigned Opcode, bool IsUnsigned, Type *ResTy, VectorType *VecTy, + std::optional<FastMathFlags> FMF, TTI::TargetCostKind CostKind) const { + EVT VecVT = TLI->getValueType(DL, VecTy); + EVT ResVT = TLI->getValueType(DL, ResTy); + + if (Opcode == Instruction::Add && VecVT.isSimple() && ResVT.isSimple() && + VecVT.getSizeInBits() >= 64) { + std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VecTy); + + // The legal cases are: + // UADDLV 8/16/32->32 + // UADDLP 32->64 + unsigned RevVTSize = ResVT.getSizeInBits(); + if (((LT.second == MVT::v8i8 || LT.second == MVT::v16i8) && + RevVTSize <= 32) || + ((LT.second == MVT::v4i16 || LT.second == MVT::v8i16) && + RevVTSize <= 32) || + ((LT.second == MVT::v2i32 || LT.second == MVT::v4i32) && + RevVTSize <= 64)) + return (LT.first - 1) * 2 + 2; + } + + return BaseT::getExtendedReductionCost(Opcode, IsUnsigned, ResTy, VecTy, FMF, + CostKind); +} + +InstructionCost +AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy, + VectorType *VecTy, + TTI::TargetCostKind CostKind) const { + EVT VecVT = TLI->getValueType(DL, VecTy); + EVT ResVT = TLI->getValueType(DL, ResTy); + + if (ST->hasDotProd() && VecVT.isSimple() && ResVT.isSimple()) { + std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VecTy); + + // The legal cases with dotprod are + // UDOT 8->32 + // Which requires an additional uaddv to sum the i32 values. + if ((LT.second == MVT::v8i8 || LT.second == MVT::v16i8) && + ResVT == MVT::i32) + return LT.first + 2; + } + + return BaseT::getMulAccReductionCost(IsUnsigned, ResTy, VecTy, CostKind); +} + +InstructionCost +AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index, + TTI::TargetCostKind CostKind) const { static const CostTblEntry ShuffleTbl[] = { { TTI::SK_Splice, MVT::nxv16i8, 1 }, { TTI::SK_Splice, MVT::nxv8i16, 1 }, @@ -4635,7 +5369,6 @@ InstructionCost AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index) { std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp); Type *LegalVTy = EVT(LT.second).getTypeForEVT(Tp->getContext()); - TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; EVT PromotedVT = LT.second.getScalarType() == MVT::i1 ? TLI->getPromotedVTForPredicate(EVT(LT.second)) : LT.second; @@ -4668,27 +5401,54 @@ InstructionCost AArch64TTIImpl::getSpliceCost(VectorType *Tp, int Index) { InstructionCost AArch64TTIImpl::getPartialReductionCost( unsigned Opcode, Type *InputTypeA, Type *InputTypeB, Type *AccumType, ElementCount VF, TTI::PartialReductionExtendKind OpAExtend, - TTI::PartialReductionExtendKind OpBExtend, - std::optional<unsigned> BinOp) const { + TTI::PartialReductionExtendKind OpBExtend, std::optional<unsigned> BinOp, + TTI::TargetCostKind CostKind) const { InstructionCost Invalid = InstructionCost::getInvalid(); InstructionCost Cost(TTI::TCC_Basic); - if (Opcode != Instruction::Add) + if (CostKind != TTI::TCK_RecipThroughput) + return Invalid; + + // Sub opcodes currently only occur in chained cases. + // Independent partial reduction subtractions are still costed as an add + if ((Opcode != Instruction::Add && Opcode != Instruction::Sub) || + OpAExtend == TTI::PR_None) return Invalid; - if (InputTypeA != InputTypeB) + // We only support multiply binary operations for now, and for muls we + // require the types being extended to be the same. + // NOTE: For muls AArch64 supports lowering mixed extensions to a usdot but + // only if the i8mm or sve/streaming features are available. + if (BinOp && (*BinOp != Instruction::Mul || InputTypeA != InputTypeB || + OpBExtend == TTI::PR_None || + (OpAExtend != OpBExtend && !ST->hasMatMulInt8() && + !ST->isSVEorStreamingSVEAvailable()))) return Invalid; + assert((BinOp || (OpBExtend == TTI::PR_None && !InputTypeB)) && + "Unexpected values for OpBExtend or InputTypeB"); EVT InputEVT = EVT::getEVT(InputTypeA); EVT AccumEVT = EVT::getEVT(AccumType); - if (VF.isScalable() && !ST->isSVEorStreamingSVEAvailable()) - return Invalid; - if (VF.isFixed() && (!ST->isNeonAvailable() || !ST->hasDotProd())) + unsigned VFMinValue = VF.getKnownMinValue(); + + if (VF.isScalable()) { + if (!ST->isSVEorStreamingSVEAvailable()) + return Invalid; + + // Don't accept a partial reduction if the scaled accumulator is vscale x 1, + // since we can't lower that type. + unsigned Scale = + AccumEVT.getScalarSizeInBits() / InputEVT.getScalarSizeInBits(); + if (VFMinValue == Scale) + return Invalid; + } + if (VF.isFixed() && + (!ST->isNeonAvailable() || !ST->hasDotProd() || AccumEVT == MVT::i64)) return Invalid; if (InputEVT == MVT::i8) { - switch (VF.getKnownMinValue()) { + switch (VFMinValue) { default: return Invalid; case 8: @@ -4707,36 +5467,33 @@ InstructionCost AArch64TTIImpl::getPartialReductionCost( } else if (InputEVT == MVT::i16) { // FIXME: Allow i32 accumulator but increase cost, as we would extend // it to i64. - if (VF.getKnownMinValue() != 8 || AccumEVT != MVT::i64) + if (VFMinValue != 8 || AccumEVT != MVT::i64) return Invalid; } else return Invalid; - // AArch64 supports lowering mixed extensions to a usdot but only if the - // i8mm or sve/streaming features are available. - if (OpAExtend == TTI::PR_None || OpBExtend == TTI::PR_None || - (OpAExtend != OpBExtend && !ST->hasMatMulInt8() && - !ST->isSVEorStreamingSVEAvailable())) - return Invalid; - - if (!BinOp || *BinOp != Instruction::Mul) - return Invalid; - return Cost; } -InstructionCost AArch64TTIImpl::getShuffleCost( - TTI::ShuffleKind Kind, VectorType *Tp, ArrayRef<int> Mask, - TTI::TargetCostKind CostKind, int Index, VectorType *SubTp, - ArrayRef<const Value *> Args, const Instruction *CxtI) { - std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(Tp); +InstructionCost +AArch64TTIImpl::getShuffleCost(TTI::ShuffleKind Kind, VectorType *DstTy, + VectorType *SrcTy, ArrayRef<int> Mask, + TTI::TargetCostKind CostKind, int Index, + VectorType *SubTp, ArrayRef<const Value *> Args, + const Instruction *CxtI) const { + assert((Mask.empty() || DstTy->isScalableTy() || + Mask.size() == DstTy->getElementCount().getKnownMinValue()) && + "Expected the Mask to match the return size if given"); + assert(SrcTy->getScalarType() == DstTy->getScalarType() && + "Expected the same scalar types"); + std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(SrcTy); // If we have a Mask, and the LT is being legalized somehow, split the Mask // into smaller vectors and sum the cost of each shuffle. - if (!Mask.empty() && isa<FixedVectorType>(Tp) && LT.second.isVector() && - Tp->getScalarSizeInBits() == LT.second.getScalarSizeInBits() && + if (!Mask.empty() && isa<FixedVectorType>(SrcTy) && LT.second.isVector() && + LT.second.getScalarSizeInBits() * Mask.size() > 128 && + SrcTy->getScalarSizeInBits() == LT.second.getScalarSizeInBits() && Mask.size() > LT.second.getVectorNumElements() && !Index && !SubTp) { - // Check for LD3/LD4 instructions, which are represented in llvm IR as // deinterleaving-shuffle(load). The shuffle cost could potentially be free, // but we model it with a cost of LT.first so that LD3/LD4 have a higher @@ -4752,22 +5509,24 @@ InstructionCost AArch64TTIImpl::getShuffleCost( // cost than just the store. if (CxtI && CxtI->hasOneUse() && isa<StoreInst>(*CxtI->user_begin()) && (ShuffleVectorInst::isInterleaveMask( - Mask, 4, Tp->getElementCount().getKnownMinValue() * 2) || + Mask, 4, SrcTy->getElementCount().getKnownMinValue() * 2) || ShuffleVectorInst::isInterleaveMask( - Mask, 3, Tp->getElementCount().getKnownMinValue() * 2))) + Mask, 3, SrcTy->getElementCount().getKnownMinValue() * 2))) return LT.first; unsigned TpNumElts = Mask.size(); unsigned LTNumElts = LT.second.getVectorNumElements(); unsigned NumVecs = (TpNumElts + LTNumElts - 1) / LTNumElts; - VectorType *NTp = - VectorType::get(Tp->getScalarType(), LT.second.getVectorElementCount()); + VectorType *NTp = VectorType::get(SrcTy->getScalarType(), + LT.second.getVectorElementCount()); InstructionCost Cost; + std::map<std::tuple<unsigned, unsigned, SmallVector<int>>, InstructionCost> + PreviousCosts; for (unsigned N = 0; N < NumVecs; N++) { SmallVector<int> NMask; // Split the existing mask into chunks of size LTNumElts. Track the source // sub-vectors to ensure the result has at most 2 inputs. - unsigned Source1, Source2; + unsigned Source1 = -1U, Source2 = -1U; unsigned NumSources = 0; for (unsigned E = 0; E < LTNumElts; E++) { int MaskElt = (N * LTNumElts + E < TpNumElts) ? Mask[N * LTNumElts + E] @@ -4799,20 +5558,32 @@ InstructionCost AArch64TTIImpl::getShuffleCost( else NMask.push_back(MaskElt % LTNumElts); } + // Check if we have already generated this sub-shuffle, which means we + // will have already generated the output. For example a <16 x i32> splat + // will be the same sub-splat 4 times, which only needs to be generated + // once and reused. + auto Result = + PreviousCosts.insert({std::make_tuple(Source1, Source2, NMask), 0}); + // Check if it was already in the map (already costed). + if (!Result.second) + continue; // If the sub-mask has at most 2 input sub-vectors then re-cost it using // getShuffleCost. If not then cost it using the worst case as the number // of element moves into a new vector. - if (NumSources <= 2) - Cost += getShuffleCost(NumSources <= 1 ? TTI::SK_PermuteSingleSrc + InstructionCost NCost = + NumSources <= 2 + ? getShuffleCost(NumSources <= 1 ? TTI::SK_PermuteSingleSrc : TTI::SK_PermuteTwoSrc, - NTp, NMask, CostKind, 0, nullptr, Args, CxtI); - else - Cost += LTNumElts; + NTp, NTp, NMask, CostKind, 0, nullptr, Args, + CxtI) + : LTNumElts; + Result.first->second = NCost; + Cost += NCost; } return Cost; } - Kind = improveShuffleKindFromMask(Kind, Mask, Tp, Index, SubTp); + Kind = improveShuffleKindFromMask(Kind, Mask, SrcTy, Index, SubTp); bool IsExtractSubvector = Kind == TTI::SK_ExtractSubvector; // A subvector extract can be implemented with an ext (or trivial extract, if // from lane 0). This currently only handles low or high extracts to prevent @@ -4828,6 +5599,35 @@ InstructionCost AArch64TTIImpl::getShuffleCost( } Kind = TTI::SK_PermuteSingleSrc; } + // FIXME: This was added to keep the costs equal when adding DstTys. Update + // the code to handle length-changing shuffles. + if (Kind == TTI::SK_InsertSubvector) { + LT = getTypeLegalizationCost(DstTy); + SrcTy = DstTy; + } + + // Segmented shuffle matching. + if (Kind == TTI::SK_PermuteSingleSrc && isa<FixedVectorType>(SrcTy) && + !Mask.empty() && SrcTy->getPrimitiveSizeInBits().isNonZero() && + SrcTy->getPrimitiveSizeInBits().isKnownMultipleOf( + AArch64::SVEBitsPerBlock)) { + + FixedVectorType *VTy = cast<FixedVectorType>(SrcTy); + unsigned Segments = + VTy->getPrimitiveSizeInBits() / AArch64::SVEBitsPerBlock; + unsigned SegmentElts = VTy->getNumElements() / Segments; + + // dupq zd.t, zn.t[idx] + if ((ST->hasSVE2p1() || ST->hasSME2p1()) && + ST->isSVEorStreamingSVEAvailable() && + isDUPQMask(Mask, Segments, SegmentElts)) + return LT.first; + + // mov zd.q, vn + if (ST->isSVEorStreamingSVEAvailable() && + isDUPFirstSegmentMask(Mask, Segments, SegmentElts)) + return LT.first; + } // Check for broadcast loads, which are supported by the LD1R instruction. // In terms of code-size, the shuffle vector is free when a load + dup get @@ -4839,15 +5639,17 @@ InstructionCost AArch64TTIImpl::getShuffleCost( if (CostKind == TTI::TCK_CodeSize && Kind == TTI::SK_Broadcast) { bool IsLoad = !Args.empty() && isa<LoadInst>(Args[0]); if (IsLoad && LT.second.isVector() && - isLegalBroadcastLoad(Tp->getElementType(), + isLegalBroadcastLoad(SrcTy->getElementType(), LT.second.getVectorElementCount())) return 0; } // If we have 4 elements for the shuffle and a Mask, get the cost straight // from the perfect shuffle tables. - if (Mask.size() == 4 && Tp->getElementCount() == ElementCount::getFixed(4) && - (Tp->getScalarSizeInBits() == 16 || Tp->getScalarSizeInBits() == 32) && + if (Mask.size() == 4 && + SrcTy->getElementCount() == ElementCount::getFixed(4) && + (SrcTy->getScalarSizeInBits() == 16 || + SrcTy->getScalarSizeInBits() == 32) && all_of(Mask, [](int E) { return E < 8; })) return getPerfectShuffleCost(Mask); @@ -4867,6 +5669,12 @@ InstructionCost AArch64TTIImpl::getShuffleCost( (Kind == TTI::SK_PermuteTwoSrc || Kind == TTI::SK_PermuteSingleSrc) && (isZIPMask(Mask, LT.second.getVectorNumElements(), Unused) || isUZPMask(Mask, LT.second.getVectorNumElements(), Unused) || + isREVMask(Mask, LT.second.getScalarSizeInBits(), + LT.second.getVectorNumElements(), 16) || + isREVMask(Mask, LT.second.getScalarSizeInBits(), + LT.second.getVectorNumElements(), 32) || + isREVMask(Mask, LT.second.getScalarSizeInBits(), + LT.second.getVectorNumElements(), 64) || // Check for non-zero lane splats all_of(drop_begin(Mask), [&Mask](int M) { return M < 0 || M == Mask[0]; }))) @@ -4886,6 +5694,8 @@ InstructionCost AArch64TTIImpl::getShuffleCost( {TTI::SK_Broadcast, MVT::v2i64, 1}, {TTI::SK_Broadcast, MVT::v4f16, 1}, {TTI::SK_Broadcast, MVT::v8f16, 1}, + {TTI::SK_Broadcast, MVT::v4bf16, 1}, + {TTI::SK_Broadcast, MVT::v8bf16, 1}, {TTI::SK_Broadcast, MVT::v2f32, 1}, {TTI::SK_Broadcast, MVT::v4f32, 1}, {TTI::SK_Broadcast, MVT::v2f64, 1}, @@ -4900,6 +5710,8 @@ InstructionCost AArch64TTIImpl::getShuffleCost( {TTI::SK_Transpose, MVT::v2i64, 1}, {TTI::SK_Transpose, MVT::v4f16, 1}, {TTI::SK_Transpose, MVT::v8f16, 1}, + {TTI::SK_Transpose, MVT::v4bf16, 1}, + {TTI::SK_Transpose, MVT::v8bf16, 1}, {TTI::SK_Transpose, MVT::v2f32, 1}, {TTI::SK_Transpose, MVT::v4f32, 1}, {TTI::SK_Transpose, MVT::v2f64, 1}, @@ -4934,9 +5746,11 @@ InstructionCost AArch64TTIImpl::getShuffleCost( {TTI::SK_Reverse, MVT::v4f32, 2}, // REV64; EXT {TTI::SK_Reverse, MVT::v2f64, 1}, // EXT {TTI::SK_Reverse, MVT::v8f16, 2}, // REV64; EXT + {TTI::SK_Reverse, MVT::v8bf16, 2}, // REV64; EXT {TTI::SK_Reverse, MVT::v8i16, 2}, // REV64; EXT {TTI::SK_Reverse, MVT::v16i8, 2}, // REV64; EXT {TTI::SK_Reverse, MVT::v4f16, 1}, // REV64 + {TTI::SK_Reverse, MVT::v4bf16, 1}, // REV64 {TTI::SK_Reverse, MVT::v4i16, 1}, // REV64 {TTI::SK_Reverse, MVT::v8i8, 1}, // REV64 // Splice can all be lowered as `ext`. @@ -4950,8 +5764,8 @@ InstructionCost AArch64TTIImpl::getShuffleCost( {TTI::SK_Splice, MVT::v8bf16, 1}, {TTI::SK_Splice, MVT::v8i16, 1}, {TTI::SK_Splice, MVT::v16i8, 1}, - {TTI::SK_Splice, MVT::v4bf16, 1}, {TTI::SK_Splice, MVT::v4f16, 1}, + {TTI::SK_Splice, MVT::v4bf16, 1}, {TTI::SK_Splice, MVT::v4i16, 1}, {TTI::SK_Splice, MVT::v8i8, 1}, // Broadcast shuffle kinds for scalable vectors @@ -4995,8 +5809,8 @@ InstructionCost AArch64TTIImpl::getShuffleCost( return LT.first * Entry->Cost; } - if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(Tp)) - return getSpliceCost(Tp, Index); + if (Kind == TTI::SK_Splice && isa<ScalableVectorType>(SrcTy)) + return getSpliceCost(SrcTy, Index, CostKind); // Inserting a subvector can often be done with either a D, S or H register // move, so long as the inserted vector is "aligned". @@ -5014,8 +5828,8 @@ InstructionCost AArch64TTIImpl::getShuffleCost( // Restore optimal kind. if (IsExtractSubvector) Kind = TTI::SK_ExtractSubvector; - return BaseT::getShuffleCost(Kind, Tp, Mask, CostKind, Index, SubTp, Args, - CxtI); + return BaseT::getShuffleCost(Kind, DstTy, SrcTy, Mask, CostKind, Index, SubTp, + Args, CxtI); } static bool containsDecreasingPointers(Loop *TheLoop, @@ -5048,7 +5862,7 @@ unsigned AArch64TTIImpl::getEpilogueVectorizationMinVF() const { return ST->getEpilogueVectorizationMinVF(); } -bool AArch64TTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) { +bool AArch64TTIImpl::preferPredicateOverEpilogue(TailFoldingInfo *TFI) const { if (!ST->hasSVE()) return false; @@ -5109,10 +5923,11 @@ AArch64TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, // Scale represents reg2 * scale, thus account for 1 if // it is not equal to 0 or 1. return AM.Scale != 0 && AM.Scale != 1; - return -1; + return InstructionCost::getInvalid(); } -bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(const Instruction *I) { +bool AArch64TTIImpl::shouldTreatInstructionLikeSelect( + const Instruction *I) const { if (EnableOrLikeSelectOpt) { // For the binary operators (e.g. or) we need to be more careful than // selects, here we only transform them if they are already at a natural @@ -5130,8 +5945,9 @@ bool AArch64TTIImpl::shouldTreatInstructionLikeSelect(const Instruction *I) { return BaseT::shouldTreatInstructionLikeSelect(I); } -bool AArch64TTIImpl::isLSRCostLess(const TargetTransformInfo::LSRCost &C1, - const TargetTransformInfo::LSRCost &C2) { +bool AArch64TTIImpl::isLSRCostLess( + const TargetTransformInfo::LSRCost &C1, + const TargetTransformInfo::LSRCost &C2) const { // AArch64 specific here is adding the number of instructions to the // comparison (though not as the first consideration, as some targets do) // along with changing the priority of the base additions. @@ -5179,7 +5995,7 @@ static bool areExtractShuffleVectors(Value *Op1, Value *Op2, !match(Op2, m_Shuffle(m_Value(S2Op1), m_Undef(), m_Mask(M2)))) return false; - // If we allow splats, set S1Op1/S2Op1 to nullptr for the relavant arg so that + // If we allow splats, set S1Op1/S2Op1 to nullptr for the relevant arg so that // it is not checked as an extract below. if (AllowSplat && isSplatShuffle(Op1)) S1Op1 = nullptr; @@ -5583,7 +6399,6 @@ bool AArch64TTIImpl::isProfitableToSinkOperands( // the backend to generate a umull. unsigned Bitwidth = I->getType()->getScalarSizeInBits(); APInt UpperMask = APInt::getHighBitsSet(Bitwidth, Bitwidth / 2); - const DataLayout &DL = I->getDataLayout(); if (!MaskedValueIsZero(OperandInstr, UpperMask, DL)) continue; NumZExts++; |