summaryrefslogtreecommitdiff
path: root/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp')
-rw-r--r--llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp196
1 files changed, 178 insertions, 18 deletions
diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
index cf6de797727b..7fda6b8fb602 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
@@ -6,8 +6,8 @@
//
//===----------------------------------------------------------------------===//
-#include "AArch64ExpandImm.h"
#include "AArch64TargetTransformInfo.h"
+#include "AArch64ExpandImm.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
@@ -16,9 +16,11 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsAArch64.h"
+#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/Debug.h"
#include <algorithm>
using namespace llvm;
+using namespace llvm::PatternMatch;
#define DEBUG_TYPE "aarch64tti"
@@ -84,7 +86,8 @@ int AArch64TTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
int AArch64TTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
const APInt &Imm, Type *Ty,
- TTI::TargetCostKind CostKind) {
+ TTI::TargetCostKind CostKind,
+ Instruction *Inst) {
assert(Ty->isIntegerTy());
unsigned BitSize = Ty->getPrimitiveSizeInBits();
@@ -192,6 +195,10 @@ int AArch64TTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
return TTI::TCC_Free;
break;
+ case Intrinsic::experimental_gc_statepoint:
+ if ((Idx < 5) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
+ return TTI::TCC_Free;
+ break;
}
return AArch64TTIImpl::getIntImmCost(Imm, Ty, CostKind);
}
@@ -205,14 +212,43 @@ AArch64TTIImpl::getPopcntSupport(unsigned TyWidth) {
return TTI::PSK_Software;
}
+unsigned
+AArch64TTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
+ TTI::TargetCostKind CostKind) {
+ auto *RetTy = ICA.getReturnType();
+ switch (ICA.getID()) {
+ case Intrinsic::umin:
+ case Intrinsic::umax: {
+ auto LT = TLI->getTypeLegalizationCost(DL, RetTy);
+ // umin(x,y) -> sub(x,usubsat(x,y))
+ // umax(x,y) -> add(x,usubsat(y,x))
+ if (LT.second == MVT::v2i64)
+ return LT.first * 2;
+ LLVM_FALLTHROUGH;
+ }
+ case Intrinsic::smin:
+ case Intrinsic::smax: {
+ static const auto ValidMinMaxTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
+ MVT::v8i16, MVT::v2i32, MVT::v4i32};
+ auto LT = TLI->getTypeLegalizationCost(DL, RetTy);
+ if (any_of(ValidMinMaxTys, [&LT](MVT M) { return M == LT.second; }))
+ return LT.first;
+ break;
+ }
+ default:
+ break;
+ }
+ return BaseT::getIntrinsicInstrCost(ICA, CostKind);
+}
+
bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
ArrayRef<const Value *> Args) {
// A helper that returns a vector type from the given type. The number of
// elements in type Ty determine the vector width.
auto toVectorTy = [&](Type *ArgTy) {
- return FixedVectorType::get(ArgTy->getScalarType(),
- cast<FixedVectorType>(DstTy)->getNumElements());
+ return VectorType::get(ArgTy->getScalarType(),
+ cast<VectorType>(DstTy)->getElementCount());
};
// Exit early if DstTy is not a vector type whose elements are at least
@@ -261,8 +297,8 @@ bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
return false;
// Get the total number of vector elements in the legalized types.
- unsigned NumDstEls = DstTyL.first * DstTyL.second.getVectorNumElements();
- unsigned NumSrcEls = SrcTyL.first * SrcTyL.second.getVectorNumElements();
+ unsigned NumDstEls = DstTyL.first * DstTyL.second.getVectorMinNumElements();
+ unsigned NumSrcEls = SrcTyL.first * SrcTyL.second.getVectorMinNumElements();
// Return true if the legalized types have the same number of vector elements
// and the destination element type size is twice that of the source type.
@@ -270,6 +306,7 @@ bool AArch64TTIImpl::isWideningInstruction(Type *DstTy, unsigned Opcode,
}
int AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
+ TTI::CastContextHint CCH,
TTI::TargetCostKind CostKind,
const Instruction *I) {
int ISD = TLI->InstructionOpcodeToISD(Opcode);
@@ -306,7 +343,8 @@ int AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
EVT DstTy = TLI->getValueType(DL, Dst);
if (!SrcTy.isSimple() || !DstTy.isSimple())
- return AdjustCost(BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I));
+ return AdjustCost(
+ BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
static const TypeConversionCostTblEntry
ConversionTbl[] = {
@@ -410,7 +448,8 @@ int AArch64TTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src,
SrcTy.getSimpleVT()))
return AdjustCost(Entry->Cost);
- return AdjustCost(BaseT::getCastInstrCost(Opcode, Dst, Src, CostKind, I));
+ return AdjustCost(
+ BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I));
}
int AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
@@ -442,12 +481,14 @@ int AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
// we may get the extension for free. If not, get the default cost for the
// extend.
if (!VecLT.second.isVector() || !TLI->isTypeLegal(DstVT))
- return Cost + getCastInstrCost(Opcode, Dst, Src, CostKind);
+ return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
+ CostKind);
// The destination type should be larger than the element type. If not, get
// the default cost for the extend.
- if (DstVT.getSizeInBits() < SrcVT.getSizeInBits())
- return Cost + getCastInstrCost(Opcode, Dst, Src, CostKind);
+ if (DstVT.getFixedSizeInBits() < SrcVT.getFixedSizeInBits())
+ return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
+ CostKind);
switch (Opcode) {
default:
@@ -466,7 +507,8 @@ int AArch64TTIImpl::getExtractWithExtendCost(unsigned Opcode, Type *Dst,
}
// If we are unable to perform the extend for free, get the default cost.
- return Cost + getCastInstrCost(Opcode, Dst, Src, CostKind);
+ return Cost + getCastInstrCost(Opcode, Dst, Src, TTI::CastContextHint::None,
+ CostKind);
}
unsigned AArch64TTIImpl::getCFInstrCost(unsigned Opcode,
@@ -602,8 +644,20 @@ int AArch64TTIImpl::getArithmeticInstrCost(
}
return Cost;
- case ISD::ADD:
case ISD::MUL:
+ if (LT.second != MVT::v2i64)
+ return (Cost + 1) * LT.first;
+ // Since we do not have a MUL.2d instruction, a mul <2 x i64> is expensive
+ // as elements are extracted from the vectors and the muls scalarized.
+ // As getScalarizationOverhead is a bit too pessimistic, we estimate the
+ // cost for a i64 vector directly here, which is:
+ // - four i64 extracts,
+ // - two i64 inserts, and
+ // - two muls.
+ // So, for a v2i64 with LT.First = 1 the cost is 8, and for a v4i64 with
+ // LT.first = 2 the cost is 16.
+ return LT.first * 8;
+ case ISD::ADD:
case ISD::XOR:
case ISD::OR:
case ISD::AND:
@@ -642,19 +696,40 @@ int AArch64TTIImpl::getAddressComputationCost(Type *Ty, ScalarEvolution *SE,
}
int AArch64TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
- Type *CondTy,
+ Type *CondTy, CmpInst::Predicate VecPred,
TTI::TargetCostKind CostKind,
const Instruction *I) {
// TODO: Handle other cost kinds.
if (CostKind != TTI::TCK_RecipThroughput)
- return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, CostKind, I);
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind,
+ I);
int ISD = TLI->InstructionOpcodeToISD(Opcode);
// We don't lower some vector selects well that are wider than the register
// width.
- if (ValTy->isVectorTy() && ISD == ISD::SELECT) {
+ if (isa<FixedVectorType>(ValTy) && ISD == ISD::SELECT) {
// We would need this many instructions to hide the scalarization happening.
const int AmortizationCost = 20;
+
+ // If VecPred is not set, check if we can get a predicate from the context
+ // instruction, if its type matches the requested ValTy.
+ if (VecPred == CmpInst::BAD_ICMP_PREDICATE && I && I->getType() == ValTy) {
+ CmpInst::Predicate CurrentPred;
+ if (match(I, m_Select(m_Cmp(CurrentPred, m_Value(), m_Value()), m_Value(),
+ m_Value())))
+ VecPred = CurrentPred;
+ }
+ // Check if we have a compare/select chain that can be lowered using CMxx &
+ // BFI pair.
+ if (CmpInst::isIntPredicate(VecPred)) {
+ static const auto ValidMinMaxTys = {MVT::v8i8, MVT::v16i8, MVT::v4i16,
+ MVT::v8i16, MVT::v2i32, MVT::v4i32,
+ MVT::v2i64};
+ auto LT = TLI->getTypeLegalizationCost(DL, ValTy);
+ if (any_of(ValidMinMaxTys, [&LT](MVT M) { return M == LT.second; }))
+ return LT.first;
+ }
+
static const TypeConversionCostTblEntry
VectorSelectTbl[] = {
{ ISD::SELECT, MVT::v16i1, MVT::v16i16, 16 },
@@ -674,7 +749,9 @@ int AArch64TTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
return Entry->Cost;
}
}
- return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, CostKind, I);
+ // The base case handles scalable vectors fine for now, since it treats the
+ // cost as 1 * legalization cost.
+ return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind, I);
}
AArch64TTIImpl::TTI::MemCmpExpansionOptions
@@ -695,6 +772,30 @@ AArch64TTIImpl::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const {
return Options;
}
+unsigned AArch64TTIImpl::getGatherScatterOpCost(
+ unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask,
+ Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) {
+
+ if (!isa<ScalableVectorType>(DataTy))
+ return BaseT::getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask,
+ Alignment, CostKind, I);
+ auto *VT = cast<VectorType>(DataTy);
+ auto LT = TLI->getTypeLegalizationCost(DL, DataTy);
+ ElementCount LegalVF = LT.second.getVectorElementCount();
+ Optional<unsigned> MaxNumVScale = getMaxVScale();
+ assert(MaxNumVScale && "Expected valid max vscale value");
+
+ unsigned MemOpCost =
+ getMemoryOpCost(Opcode, VT->getElementType(), Alignment, 0, CostKind, I);
+ unsigned MaxNumElementsPerGather =
+ MaxNumVScale.getValue() * LegalVF.getKnownMinValue();
+ return LT.first * MaxNumElementsPerGather * MemOpCost;
+}
+
+bool AArch64TTIImpl::useNeonVector(const Type *Ty) const {
+ return isa<FixedVectorType>(Ty) && !ST->useSVEForFixedLengthVectors();
+}
+
int AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
MaybeAlign Alignment, unsigned AddressSpace,
TTI::TargetCostKind CostKind,
@@ -722,7 +823,7 @@ int AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Ty,
return LT.first * 2 * AmortizationCost;
}
- if (Ty->isVectorTy() &&
+ if (useNeonVector(Ty) &&
cast<VectorType>(Ty)->getElementType()->isIntegerTy(8)) {
unsigned ProfitableNumElements;
if (Opcode == Instruction::Store)
@@ -997,11 +1098,70 @@ bool AArch64TTIImpl::useReductionIntrinsic(unsigned Opcode, Type *Ty,
return false;
}
+int AArch64TTIImpl::getMinMaxReductionCost(VectorType *Ty, VectorType *CondTy,
+ bool IsPairwise, bool IsUnsigned,
+ TTI::TargetCostKind CostKind) {
+ if (!isa<ScalableVectorType>(Ty))
+ return BaseT::getMinMaxReductionCost(Ty, CondTy, IsPairwise, IsUnsigned,
+ CostKind);
+ assert((isa<ScalableVectorType>(Ty) && isa<ScalableVectorType>(CondTy)) &&
+ "Both vector needs to be scalable");
+
+ std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, Ty);
+ int LegalizationCost = 0;
+ if (LT.first > 1) {
+ Type *LegalVTy = EVT(LT.second).getTypeForEVT(Ty->getContext());
+ unsigned CmpOpcode =
+ Ty->isFPOrFPVectorTy() ? Instruction::FCmp : Instruction::ICmp;
+ LegalizationCost =
+ getCmpSelInstrCost(CmpOpcode, LegalVTy, LegalVTy,
+ CmpInst::BAD_ICMP_PREDICATE, CostKind) +
+ getCmpSelInstrCost(Instruction::Select, LegalVTy, LegalVTy,
+ CmpInst::BAD_ICMP_PREDICATE, CostKind);
+ LegalizationCost *= LT.first - 1;
+ }
+
+ return LegalizationCost + /*Cost of horizontal reduction*/ 2;
+}
+
+int AArch64TTIImpl::getArithmeticReductionCostSVE(
+ unsigned Opcode, VectorType *ValTy, bool IsPairwise,
+ TTI::TargetCostKind CostKind) {
+ assert(!IsPairwise && "Cannot be pair wise to continue");
+
+ std::pair<int, MVT> LT = TLI->getTypeLegalizationCost(DL, ValTy);
+ int LegalizationCost = 0;
+ if (LT.first > 1) {
+ Type *LegalVTy = EVT(LT.second).getTypeForEVT(ValTy->getContext());
+ LegalizationCost = getArithmeticInstrCost(Opcode, LegalVTy, CostKind);
+ LegalizationCost *= LT.first - 1;
+ }
+
+ int ISD = TLI->InstructionOpcodeToISD(Opcode);
+ assert(ISD && "Invalid opcode");
+ // Add the final reduction cost for the legal horizontal reduction
+ switch (ISD) {
+ case ISD::ADD:
+ case ISD::AND:
+ case ISD::OR:
+ case ISD::XOR:
+ case ISD::FADD:
+ return LegalizationCost + 2;
+ default:
+ // TODO: Replace for invalid when InstructionCost is used
+ // cases not supported by SVE
+ return 16;
+ }
+}
+
int AArch64TTIImpl::getArithmeticReductionCost(unsigned Opcode,
VectorType *ValTy,
bool IsPairwiseForm,
TTI::TargetCostKind CostKind) {
+ if (isa<ScalableVectorType>(ValTy))
+ return getArithmeticReductionCostSVE(Opcode, ValTy, IsPairwiseForm,
+ CostKind);
if (IsPairwiseForm)
return BaseT::getArithmeticReductionCost(Opcode, ValTy, IsPairwiseForm,
CostKind);
generated by cgit v1.3 (git 2.53.0) at 2026-07-29 11:12:28 +0000