diff options
Diffstat (limited to 'llvm/lib/Analysis/TargetTransformInfo.cpp')
| -rw-r--r-- | llvm/lib/Analysis/TargetTransformInfo.cpp | 717 |
1 files changed, 329 insertions, 388 deletions
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp index f2c63f789d892..2f051e53790b1 100644 --- a/llvm/lib/Analysis/TargetTransformInfo.cpp +++ b/llvm/lib/Analysis/TargetTransformInfo.cpp @@ -11,8 +11,8 @@ #include "llvm/Analysis/LoopIterator.h" #include "llvm/Analysis/TargetTransformInfoImpl.h" #include "llvm/IR/CFG.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/Dominators.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/IntrinsicInst.h" @@ -42,18 +42,109 @@ struct NoTTIImpl : TargetTransformInfoImplCRTPBase<NoTTIImpl> { explicit NoTTIImpl(const DataLayout &DL) : TargetTransformInfoImplCRTPBase<NoTTIImpl>(DL) {} }; -} +} // namespace bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) { // If the loop has irreducible control flow, it can not be converted to // Hardware loop. - LoopBlocksRPO RPOT(L); + LoopBlocksRPO RPOT(L); RPOT.perform(&LI); if (containsIrreducibleCFG<const BasicBlock *>(RPOT, LI)) return false; return true; } +IntrinsicCostAttributes::IntrinsicCostAttributes(const IntrinsicInst &I) : + II(&I), RetTy(I.getType()), IID(I.getIntrinsicID()) { + + FunctionType *FTy = I.getCalledFunction()->getFunctionType(); + ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end()); + Arguments.insert(Arguments.begin(), I.arg_begin(), I.arg_end()); + if (auto *FPMO = dyn_cast<FPMathOperator>(&I)) + FMF = FPMO->getFastMathFlags(); +} + +IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, + const CallBase &CI) : + II(dyn_cast<IntrinsicInst>(&CI)), RetTy(CI.getType()), IID(Id) { + + if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI)) + FMF = FPMO->getFastMathFlags(); + + FunctionType *FTy = + CI.getCalledFunction()->getFunctionType(); + ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end()); +} + +IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, + const CallBase &CI, + unsigned Factor) : + RetTy(CI.getType()), IID(Id), VF(Factor) { + + if (auto *FPMO = dyn_cast<FPMathOperator>(&CI)) + FMF = FPMO->getFastMathFlags(); + + Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end()); + FunctionType *FTy = + CI.getCalledFunction()->getFunctionType(); + ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end()); +} + +IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, + const CallBase &CI, + unsigned Factor, + unsigned ScalarCost) : + RetTy(CI.getType()), IID(Id), VF(Factor), ScalarizationCost(ScalarCost) { + + if (const auto *FPMO = dyn_cast<FPMathOperator>(&CI)) + FMF = FPMO->getFastMathFlags(); + + Arguments.insert(Arguments.begin(), CI.arg_begin(), CI.arg_end()); + FunctionType *FTy = + CI.getCalledFunction()->getFunctionType(); + ParamTys.insert(ParamTys.begin(), FTy->param_begin(), FTy->param_end()); +} + +IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, + ArrayRef<Type *> Tys, + FastMathFlags Flags) : + RetTy(RTy), IID(Id), FMF(Flags) { + ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end()); +} + +IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, + ArrayRef<Type *> Tys, + FastMathFlags Flags, + unsigned ScalarCost) : + RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) { + ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end()); +} + +IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, + ArrayRef<Type *> Tys, + FastMathFlags Flags, + unsigned ScalarCost, + const IntrinsicInst *I) : + II(I), RetTy(RTy), IID(Id), FMF(Flags), ScalarizationCost(ScalarCost) { + ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end()); +} + +IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *RTy, + ArrayRef<Type *> Tys) : + RetTy(RTy), IID(Id) { + ParamTys.insert(ParamTys.begin(), Tys.begin(), Tys.end()); +} + +IntrinsicCostAttributes::IntrinsicCostAttributes(Intrinsic::ID Id, Type *Ty, + ArrayRef<const Value *> Args) + : RetTy(Ty), IID(Id) { + + Arguments.insert(Arguments.begin(), Args.begin(), Args.end()); + ParamTys.reserve(Arguments.size()); + for (unsigned Idx = 0, Size = Arguments.size(); Idx != Size; ++Idx) + ParamTys.push_back(Arguments[Idx]->getType()); +} + bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE, LoopInfo &LI, DominatorTree &DT, bool ForceNestedLoop, @@ -146,28 +237,6 @@ TargetTransformInfo &TargetTransformInfo::operator=(TargetTransformInfo &&RHS) { return *this; } -int TargetTransformInfo::getOperationCost(unsigned Opcode, Type *Ty, - Type *OpTy) const { - int Cost = TTIImpl->getOperationCost(Opcode, Ty, OpTy); - assert(Cost >= 0 && "TTI should not produce negative costs!"); - return Cost; -} - -int TargetTransformInfo::getCallCost(FunctionType *FTy, int NumArgs, - const User *U) const { - int Cost = TTIImpl->getCallCost(FTy, NumArgs, U); - assert(Cost >= 0 && "TTI should not produce negative costs!"); - return Cost; -} - -int TargetTransformInfo::getCallCost(const Function *F, - ArrayRef<const Value *> Arguments, - const User *U) const { - int Cost = TTIImpl->getCallCost(F, Arguments, U); - assert(Cost >= 0 && "TTI should not produce negative costs!"); - return Cost; -} - unsigned TargetTransformInfo::getInliningThresholdMultiplier() const { return TTIImpl->getInliningThresholdMultiplier(); } @@ -177,34 +246,23 @@ int TargetTransformInfo::getInlinerVectorBonusPercent() const { } int TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr, - ArrayRef<const Value *> Operands) const { - return TTIImpl->getGEPCost(PointeeType, Ptr, Operands); -} - -int TargetTransformInfo::getExtCost(const Instruction *I, - const Value *Src) const { - return TTIImpl->getExtCost(I, Src); -} - -int TargetTransformInfo::getIntrinsicCost( - Intrinsic::ID IID, Type *RetTy, ArrayRef<const Value *> Arguments, - const User *U) const { - int Cost = TTIImpl->getIntrinsicCost(IID, RetTy, Arguments, U); - assert(Cost >= 0 && "TTI should not produce negative costs!"); - return Cost; + ArrayRef<const Value *> Operands, + TTI::TargetCostKind CostKind) const { + return TTIImpl->getGEPCost(PointeeType, Ptr, Operands, CostKind); } -unsigned -TargetTransformInfo::getEstimatedNumberOfCaseClusters( +unsigned TargetTransformInfo::getEstimatedNumberOfCaseClusters( const SwitchInst &SI, unsigned &JTSize, ProfileSummaryInfo *PSI, BlockFrequencyInfo *BFI) const { return TTIImpl->getEstimatedNumberOfCaseClusters(SI, JTSize, PSI, BFI); } int TargetTransformInfo::getUserCost(const User *U, - ArrayRef<const Value *> Operands) const { - int Cost = TTIImpl->getUserCost(U, Operands); - assert(Cost >= 0 && "TTI should not produce negative costs!"); + ArrayRef<const Value *> Operands, + enum TargetCostKind CostKind) const { + int Cost = TTIImpl->getUserCost(U, Operands, CostKind); + assert((CostKind == TTI::TCK_RecipThroughput || Cost >= 0) && + "TTI should not produce negative costs!"); return Cost; } @@ -212,6 +270,10 @@ bool TargetTransformInfo::hasBranchDivergence() const { return TTIImpl->hasBranchDivergence(); } +bool TargetTransformInfo::useGPUDivergenceAnalysis() const { + return TTIImpl->useGPUDivergenceAnalysis(); +} + bool TargetTransformInfo::isSourceOfDivergence(const Value *V) const { return TTIImpl->isSourceOfDivergence(V); } @@ -225,12 +287,17 @@ unsigned TargetTransformInfo::getFlatAddressSpace() const { } bool TargetTransformInfo::collectFlatAddressOperands( - SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const { + SmallVectorImpl<int> &OpIndexes, Intrinsic::ID IID) const { return TTIImpl->collectFlatAddressOperands(OpIndexes, IID); } -bool TargetTransformInfo::rewriteIntrinsicWithAddressSpace( - IntrinsicInst *II, Value *OldV, Value *NewV) const { +bool TargetTransformInfo::isNoopAddrSpaceCast(unsigned FromAS, + unsigned ToAS) const { + return TTIImpl->isNoopAddrSpaceCast(FromAS, ToAS); +} + +Value *TargetTransformInfo::rewriteIntrinsicWithAddressSpace( + IntrinsicInst *II, Value *OldV, Value *NewV) const { return TTIImpl->rewriteIntrinsicWithAddressSpace(II, OldV, NewV); } @@ -239,22 +306,32 @@ bool TargetTransformInfo::isLoweredToCall(const Function *F) const { } bool TargetTransformInfo::isHardwareLoopProfitable( - Loop *L, ScalarEvolution &SE, AssumptionCache &AC, - TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const { + Loop *L, ScalarEvolution &SE, AssumptionCache &AC, + TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const { return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); } -bool TargetTransformInfo::preferPredicateOverEpilogue(Loop *L, LoopInfo *LI, - ScalarEvolution &SE, AssumptionCache &AC, TargetLibraryInfo *TLI, - DominatorTree *DT, const LoopAccessInfo *LAI) const { +bool TargetTransformInfo::preferPredicateOverEpilogue( + Loop *L, LoopInfo *LI, ScalarEvolution &SE, AssumptionCache &AC, + TargetLibraryInfo *TLI, DominatorTree *DT, + const LoopAccessInfo *LAI) const { return TTIImpl->preferPredicateOverEpilogue(L, LI, SE, AC, TLI, DT, LAI); } +bool TargetTransformInfo::emitGetActiveLaneMask() const { + return TTIImpl->emitGetActiveLaneMask(); +} + void TargetTransformInfo::getUnrollingPreferences( Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const { return TTIImpl->getUnrollingPreferences(L, SE, UP); } +void TargetTransformInfo::getPeelingPreferences(Loop *L, ScalarEvolution &SE, + PeelingPreferences &PP) const { + return TTIImpl->getPeelingPreferences(L, SE, PP); +} + bool TargetTransformInfo::isLegalAddImmediate(int64_t Imm) const { return TTIImpl->isLegalAddImmediate(Imm); } @@ -265,8 +342,7 @@ bool TargetTransformInfo::isLegalICmpImmediate(int64_t Imm) const { bool TargetTransformInfo::isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, - bool HasBaseReg, - int64_t Scale, + bool HasBaseReg, int64_t Scale, unsigned AddrSpace, Instruction *I) const { return TTIImpl->isLegalAddressingMode(Ty, BaseGV, BaseOffset, HasBaseReg, @@ -277,6 +353,10 @@ bool TargetTransformInfo::isLSRCostLess(LSRCost &C1, LSRCost &C2) const { return TTIImpl->isLSRCostLess(C1, C2); } +bool TargetTransformInfo::isProfitableLSRChainElement(Instruction *I) const { + return TTIImpl->isProfitableLSRChainElement(I); +} + bool TargetTransformInfo::canMacroFuseCmp() const { return TTIImpl->canMacroFuseCmp(); } @@ -297,12 +377,12 @@ bool TargetTransformInfo::shouldFavorBackedgeIndex(const Loop *L) const { } bool TargetTransformInfo::isLegalMaskedStore(Type *DataType, - MaybeAlign Alignment) const { + Align Alignment) const { return TTIImpl->isLegalMaskedStore(DataType, Alignment); } bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType, - MaybeAlign Alignment) const { + Align Alignment) const { return TTIImpl->isLegalMaskedLoad(DataType, Alignment); } @@ -316,12 +396,12 @@ bool TargetTransformInfo::isLegalNTLoad(Type *DataType, Align Alignment) const { } bool TargetTransformInfo::isLegalMaskedGather(Type *DataType, - MaybeAlign Alignment) const { + Align Alignment) const { return TTIImpl->isLegalMaskedGather(DataType, Alignment); } bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType, - MaybeAlign Alignment) const { + Align Alignment) const { return TTIImpl->isLegalMaskedScatter(DataType, Alignment); } @@ -348,8 +428,7 @@ bool TargetTransformInfo::prefersVectorizedAddressing() const { int TargetTransformInfo::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset, - bool HasBaseReg, - int64_t Scale, + bool HasBaseReg, int64_t Scale, unsigned AddrSpace) const { int Cost = TTIImpl->getScalingFactorCost(Ty, BaseGV, BaseOffset, HasBaseReg, Scale, AddrSpace); @@ -378,7 +457,8 @@ bool TargetTransformInfo::isTypeLegal(Type *Ty) const { bool TargetTransformInfo::shouldBuildLookupTables() const { return TTIImpl->shouldBuildLookupTables(); } -bool TargetTransformInfo::shouldBuildLookupTablesForConstant(Constant *C) const { +bool TargetTransformInfo::shouldBuildLookupTablesForConstant( + Constant *C) const { return TTIImpl->shouldBuildLookupTablesForConstant(C); } @@ -386,14 +466,15 @@ bool TargetTransformInfo::useColdCCForColdCall(Function &F) const { return TTIImpl->useColdCCForColdCall(F); } -unsigned TargetTransformInfo:: -getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) const { - return TTIImpl->getScalarizationOverhead(Ty, Insert, Extract); +unsigned +TargetTransformInfo::getScalarizationOverhead(VectorType *Ty, + const APInt &DemandedElts, + bool Insert, bool Extract) const { + return TTIImpl->getScalarizationOverhead(Ty, DemandedElts, Insert, Extract); } -unsigned TargetTransformInfo:: -getOperandsScalarizationOverhead(ArrayRef<const Value *> Args, - unsigned VF) const { +unsigned TargetTransformInfo::getOperandsScalarizationOverhead( + ArrayRef<const Value *> Args, unsigned VF) const { return TTIImpl->getOperandsScalarizationOverhead(Args, VF); } @@ -401,7 +482,8 @@ bool TargetTransformInfo::supportsEfficientVectorElementLoadStore() const { return TTIImpl->supportsEfficientVectorElementLoadStore(); } -bool TargetTransformInfo::enableAggressiveInterleaving(bool LoopHasReductions) const { +bool TargetTransformInfo::enableAggressiveInterleaving( + bool LoopHasReductions) const { return TTIImpl->enableAggressiveInterleaving(LoopHasReductions); } @@ -427,8 +509,8 @@ bool TargetTransformInfo::allowsMisalignedMemoryAccesses(LLVMContext &Context, unsigned AddressSpace, unsigned Alignment, bool *Fast) const { - return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, AddressSpace, - Alignment, Fast); + return TTIImpl->allowsMisalignedMemoryAccesses(Context, BitWidth, + AddressSpace, Alignment, Fast); } TargetTransformInfo::PopcntSupportKind @@ -458,22 +540,27 @@ int TargetTransformInfo::getIntImmCodeSizeCost(unsigned Opcode, unsigned Idx, return Cost; } -int TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty) const { - int Cost = TTIImpl->getIntImmCost(Imm, Ty); +int TargetTransformInfo::getIntImmCost(const APInt &Imm, Type *Ty, + TTI::TargetCostKind CostKind) const { + int Cost = TTIImpl->getIntImmCost(Imm, Ty, CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } -int TargetTransformInfo::getIntImmCostInst(unsigned Opcode, unsigned Idx, - const APInt &Imm, Type *Ty) const { - int Cost = TTIImpl->getIntImmCostInst(Opcode, Idx, Imm, Ty); +int +TargetTransformInfo::getIntImmCostInst(unsigned Opcode, unsigned Idx, + const APInt &Imm, Type *Ty, + TTI::TargetCostKind CostKind) const { + int Cost = TTIImpl->getIntImmCostInst(Opcode, Idx, Imm, Ty, CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } -int TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, - const APInt &Imm, Type *Ty) const { - int Cost = TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty); +int +TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx, + const APInt &Imm, Type *Ty, + TTI::TargetCostKind CostKind) const { + int Cost = TTIImpl->getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } @@ -482,11 +569,12 @@ unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const { return TTIImpl->getNumberOfRegisters(ClassID); } -unsigned TargetTransformInfo::getRegisterClassForType(bool Vector, Type *Ty) const { +unsigned TargetTransformInfo::getRegisterClassForType(bool Vector, + Type *Ty) const { return TTIImpl->getRegisterClassForType(Vector, Ty); } -const char* TargetTransformInfo::getRegisterClassName(unsigned ClassID) const { +const char *TargetTransformInfo::getRegisterClassName(unsigned ClassID) const { return TTIImpl->getRegisterClassName(ClassID); } @@ -516,13 +604,13 @@ unsigned TargetTransformInfo::getCacheLineSize() const { return TTIImpl->getCacheLineSize(); } -llvm::Optional<unsigned> TargetTransformInfo::getCacheSize(CacheLevel Level) - const { +llvm::Optional<unsigned> +TargetTransformInfo::getCacheSize(CacheLevel Level) const { return TTIImpl->getCacheSize(Level); } -llvm::Optional<unsigned> TargetTransformInfo::getCacheAssociativity( - CacheLevel Level) const { +llvm::Optional<unsigned> +TargetTransformInfo::getCacheAssociativity(CacheLevel Level) const { return TTIImpl->getCacheAssociativity(Level); } @@ -530,24 +618,32 @@ unsigned TargetTransformInfo::getPrefetchDistance() const { return TTIImpl->getPrefetchDistance(); } -unsigned TargetTransformInfo::getMinPrefetchStride() const { - return TTIImpl->getMinPrefetchStride(); +unsigned TargetTransformInfo::getMinPrefetchStride( + unsigned NumMemAccesses, unsigned NumStridedMemAccesses, + unsigned NumPrefetches, bool HasCall) const { + return TTIImpl->getMinPrefetchStride(NumMemAccesses, NumStridedMemAccesses, + NumPrefetches, HasCall); } unsigned TargetTransformInfo::getMaxPrefetchIterationsAhead() const { return TTIImpl->getMaxPrefetchIterationsAhead(); } +bool TargetTransformInfo::enableWritePrefetching() const { + return TTIImpl->enableWritePrefetching(); +} + unsigned TargetTransformInfo::getMaxInterleaveFactor(unsigned VF) const { return TTIImpl->getMaxInterleaveFactor(VF); } TargetTransformInfo::OperandValueKind -TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) { +TargetTransformInfo::getOperandInfo(const Value *V, + OperandValueProperties &OpProps) { OperandValueKind OpInfo = OK_AnyValue; OpProps = OP_None; - if (auto *CI = dyn_cast<ConstantInt>(V)) { + if (const auto *CI = dyn_cast<ConstantInt>(V)) { if (CI->getValue().isPowerOf2()) OpProps = OP_PowerOf2; return OK_UniformConstantValue; @@ -556,7 +652,7 @@ TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) { // A broadcast shuffle creates a uniform value. // TODO: Add support for non-zero index broadcasts. // TODO: Add support for different source vector width. - if (auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V)) + if (const auto *ShuffleInst = dyn_cast<ShuffleVectorInst>(V)) if (ShuffleInst->isZeroEltSplat()) OpInfo = OK_UniformValue; @@ -571,7 +667,7 @@ TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) { if (auto *CI = dyn_cast<ConstantInt>(Splat)) if (CI->getValue().isPowerOf2()) OpProps = OP_PowerOf2; - } else if (auto *CDS = dyn_cast<ConstantDataSequential>(V)) { + } else if (const auto *CDS = dyn_cast<ConstantDataSequential>(V)) { OpProps = OP_PowerOf2; for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) { if (auto *CI = dyn_cast<ConstantInt>(CDS->getElementAsConstant(I))) @@ -592,28 +688,31 @@ TargetTransformInfo::getOperandInfo(Value *V, OperandValueProperties &OpProps) { } int TargetTransformInfo::getArithmeticInstrCost( - unsigned Opcode, Type *Ty, OperandValueKind Opd1Info, + unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind, + OperandValueKind Opd1Info, OperandValueKind Opd2Info, OperandValueProperties Opd1PropInfo, OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args, const Instruction *CxtI) const { int Cost = TTIImpl->getArithmeticInstrCost( - Opcode, Ty, Opd1Info, Opd2Info, Opd1PropInfo, Opd2PropInfo, Args, CxtI); + Opcode, Ty, CostKind, Opd1Info, Opd2Info, Opd1PropInfo, Opd2PropInfo, + Args, CxtI); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } -int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, Type *Ty, int Index, - Type *SubTp) const { +int TargetTransformInfo::getShuffleCost(ShuffleKind Kind, VectorType *Ty, + int Index, VectorType *SubTp) const { int Cost = TTIImpl->getShuffleCost(Kind, Ty, Index, SubTp); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } -int TargetTransformInfo::getCastInstrCost(unsigned Opcode, Type *Dst, - Type *Src, const Instruction *I) const { - assert ((I == nullptr || I->getOpcode() == Opcode) && - "Opcode should reflect passed instruction."); - int Cost = TTIImpl->getCastInstrCost(Opcode, Dst, Src, I); +int TargetTransformInfo::getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src, + TTI::TargetCostKind CostKind, + const Instruction *I) const { + assert((I == nullptr || I->getOpcode() == Opcode) && + "Opcode should reflect passed instruction."); + int Cost = TTIImpl->getCastInstrCost(Opcode, Dst, Src, CostKind, I); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } @@ -626,17 +725,20 @@ int TargetTransformInfo::getExtractWithExtendCost(unsigned Opcode, Type *Dst, return Cost; } -int TargetTransformInfo::getCFInstrCost(unsigned Opcode) const { - int Cost = TTIImpl->getCFInstrCost(Opcode); +int TargetTransformInfo::getCFInstrCost(unsigned Opcode, + TTI::TargetCostKind CostKind) const { + int Cost = TTIImpl->getCFInstrCost(Opcode, CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } int TargetTransformInfo::getCmpSelInstrCost(unsigned Opcode, Type *ValTy, - Type *CondTy, const Instruction *I) const { - assert ((I == nullptr || I->getOpcode() == Opcode) && - "Opcode should reflect passed instruction."); - int Cost = TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, I); + Type *CondTy, + TTI::TargetCostKind CostKind, + const Instruction *I) const { + assert((I == nullptr || I->getOpcode() == Opcode) && + "Opcode should reflect passed instruction."); + int Cost = TTIImpl->getCmpSelInstrCost(Opcode, ValTy, CondTy, CostKind, I); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } @@ -649,65 +751,59 @@ int TargetTransformInfo::getVectorInstrCost(unsigned Opcode, Type *Val, } int TargetTransformInfo::getMemoryOpCost(unsigned Opcode, Type *Src, - MaybeAlign Alignment, - unsigned AddressSpace, + Align Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind, const Instruction *I) const { - assert ((I == nullptr || I->getOpcode() == Opcode) && - "Opcode should reflect passed instruction."); - int Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, I); + assert((I == nullptr || I->getOpcode() == Opcode) && + "Opcode should reflect passed instruction."); + int Cost = TTIImpl->getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, + CostKind, I); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } -int TargetTransformInfo::getMaskedMemoryOpCost(unsigned Opcode, Type *Src, - unsigned Alignment, - unsigned AddressSpace) const { +int TargetTransformInfo::getMaskedMemoryOpCost( + unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace, + TTI::TargetCostKind CostKind) const { int Cost = - TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace); + TTIImpl->getMaskedMemoryOpCost(Opcode, Src, Alignment, AddressSpace, + CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } -int TargetTransformInfo::getGatherScatterOpCost(unsigned Opcode, Type *DataTy, - Value *Ptr, bool VariableMask, - unsigned Alignment) const { +int TargetTransformInfo::getGatherScatterOpCost( + unsigned Opcode, Type *DataTy, const Value *Ptr, bool VariableMask, + Align Alignment, TTI::TargetCostKind CostKind, const Instruction *I) const { int Cost = TTIImpl->getGatherScatterOpCost(Opcode, DataTy, Ptr, VariableMask, - Alignment); + Alignment, CostKind, I); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } int TargetTransformInfo::getInterleavedMemoryOpCost( unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices, - unsigned Alignment, unsigned AddressSpace, bool UseMaskForCond, - bool UseMaskForGaps) const { - int Cost = TTIImpl->getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices, - Alignment, AddressSpace, - UseMaskForCond, - UseMaskForGaps); - assert(Cost >= 0 && "TTI should not produce negative costs!"); - return Cost; -} - -int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, - ArrayRef<Type *> Tys, FastMathFlags FMF, - unsigned ScalarizationCostPassed) const { - int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Tys, FMF, - ScalarizationCostPassed); + Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind, + bool UseMaskForCond, bool UseMaskForGaps) const { + int Cost = TTIImpl->getInterleavedMemoryOpCost( + Opcode, VecTy, Factor, Indices, Alignment, AddressSpace, CostKind, + UseMaskForCond, UseMaskForGaps); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } -int TargetTransformInfo::getIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy, - ArrayRef<Value *> Args, FastMathFlags FMF, unsigned VF) const { - int Cost = TTIImpl->getIntrinsicInstrCost(ID, RetTy, Args, FMF, VF); +int +TargetTransformInfo::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA, + TTI::TargetCostKind CostKind) const { + int Cost = TTIImpl->getIntrinsicInstrCost(ICA, CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } int TargetTransformInfo::getCallInstrCost(Function *F, Type *RetTy, - ArrayRef<Type *> Tys) const { - int Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys); + ArrayRef<Type *> Tys, + TTI::TargetCostKind CostKind) const { + int Cost = TTIImpl->getCallInstrCost(F, RetTy, Tys, CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } @@ -730,18 +826,22 @@ int TargetTransformInfo::getMemcpyCost(const Instruction *I) const { return Cost; } -int TargetTransformInfo::getArithmeticReductionCost(unsigned Opcode, Type *Ty, - bool IsPairwiseForm) const { - int Cost = TTIImpl->getArithmeticReductionCost(Opcode, Ty, IsPairwiseForm); +int TargetTransformInfo::getArithmeticReductionCost(unsigned Opcode, + VectorType *Ty, + bool IsPairwiseForm, + TTI::TargetCostKind CostKind) const { + int Cost = TTIImpl->getArithmeticReductionCost(Opcode, Ty, IsPairwiseForm, + CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } -int TargetTransformInfo::getMinMaxReductionCost(Type *Ty, Type *CondTy, - bool IsPairwiseForm, - bool IsUnsigned) const { +int TargetTransformInfo::getMinMaxReductionCost( + VectorType *Ty, VectorType *CondTy, bool IsPairwiseForm, bool IsUnsigned, + TTI::TargetCostKind CostKind) const { int Cost = - TTIImpl->getMinMaxReductionCost(Ty, CondTy, IsPairwiseForm, IsUnsigned); + TTIImpl->getMinMaxReductionCost(Ty, CondTy, IsPairwiseForm, IsUnsigned, + CostKind); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } @@ -765,18 +865,19 @@ Value *TargetTransformInfo::getOrCreateResultFromMemIntrinsic( return TTIImpl->getOrCreateResultFromMemIntrinsic(Inst, ExpectedType); } -Type *TargetTransformInfo::getMemcpyLoopLoweringType(LLVMContext &Context, - Value *Length, - unsigned SrcAlign, - unsigned DestAlign) const { - return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAlign, - DestAlign); +Type *TargetTransformInfo::getMemcpyLoopLoweringType( + LLVMContext &Context, Value *Length, unsigned SrcAddrSpace, + unsigned DestAddrSpace, unsigned SrcAlign, unsigned DestAlign) const { + return TTIImpl->getMemcpyLoopLoweringType(Context, Length, SrcAddrSpace, + DestAddrSpace, SrcAlign, DestAlign); } void TargetTransformInfo::getMemcpyLoopResidualLoweringType( SmallVectorImpl<Type *> &OpsOut, LLVMContext &Context, - unsigned RemainingBytes, unsigned SrcAlign, unsigned DestAlign) const { + unsigned RemainingBytes, unsigned SrcAddrSpace, unsigned DestAddrSpace, + unsigned SrcAlign, unsigned DestAlign) const { TTIImpl->getMemcpyLoopResidualLoweringType(OpsOut, Context, RemainingBytes, + SrcAddrSpace, DestAddrSpace, SrcAlign, DestAlign); } @@ -814,13 +915,13 @@ bool TargetTransformInfo::isLegalToVectorizeStore(StoreInst *SI) const { } bool TargetTransformInfo::isLegalToVectorizeLoadChain( - unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const { + unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const { return TTIImpl->isLegalToVectorizeLoadChain(ChainSizeInBytes, Alignment, AddrSpace); } bool TargetTransformInfo::isLegalToVectorizeStoreChain( - unsigned ChainSizeInBytes, unsigned Alignment, unsigned AddrSpace) const { + unsigned ChainSizeInBytes, Align Alignment, unsigned AddrSpace) const { return TTIImpl->isLegalToVectorizeStoreChain(ChainSizeInBytes, Alignment, AddrSpace); } @@ -839,8 +940,8 @@ unsigned TargetTransformInfo::getStoreVectorFactor(unsigned VF, return TTIImpl->getStoreVectorFactor(VF, StoreSize, ChainSizeInBytes, VecTy); } -bool TargetTransformInfo::useReductionIntrinsic(unsigned Opcode, - Type *Ty, ReductionFlags Flags) const { +bool TargetTransformInfo::useReductionIntrinsic(unsigned Opcode, Type *Ty, + ReductionFlags Flags) const { return TTIImpl->useReductionIntrinsic(Opcode, Ty, Flags); } @@ -865,46 +966,21 @@ static bool matchPairwiseShuffleMask(ShuffleVectorInst *SI, bool IsLeft, else if (!SI) return false; - SmallVector<int, 32> Mask(SI->getType()->getVectorNumElements(), -1); + SmallVector<int, 32> Mask(SI->getType()->getNumElements(), -1); // Build a mask of 0, 2, ... (left) or 1, 3, ... (right) depending on whether // we look at the left or right side. for (unsigned i = 0, e = (1 << Level), val = !IsLeft; i != e; ++i, val += 2) Mask[i] = val; - SmallVector<int, 16> ActualMask = SI->getShuffleMask(); + ArrayRef<int> ActualMask = SI->getShuffleMask(); return Mask == ActualMask; } -namespace { -/// Kind of the reduction data. -enum ReductionKind { - RK_None, /// Not a reduction. - RK_Arithmetic, /// Binary reduction data. - RK_MinMax, /// Min/max reduction data. - RK_UnsignedMinMax, /// Unsigned min/max reduction data. -}; -/// Contains opcode + LHS/RHS parts of the reduction operations. -struct ReductionData { - ReductionData() = delete; - ReductionData(ReductionKind Kind, unsigned Opcode, Value *LHS, Value *RHS) - : Opcode(Opcode), LHS(LHS), RHS(RHS), Kind(Kind) { - assert(Kind != RK_None && "expected binary or min/max reduction only."); - } - unsigned Opcode = 0; - Value *LHS = nullptr; - Value *RHS = nullptr; - ReductionKind Kind = RK_None; - bool hasSameData(ReductionData &RD) const { - return Kind == RD.Kind && Opcode == RD.Opcode; - } -}; -} // namespace - -static Optional<ReductionData> getReductionData(Instruction *I) { +static Optional<TTI::ReductionData> getReductionData(Instruction *I) { Value *L, *R; if (m_BinOp(m_Value(L), m_Value(R)).match(I)) - return ReductionData(RK_Arithmetic, I->getOpcode(), L, R); + return TTI::ReductionData(TTI::RK_Arithmetic, I->getOpcode(), L, R); if (auto *SI = dyn_cast<SelectInst>(I)) { if (m_SMin(m_Value(L), m_Value(R)).match(SI) || m_SMax(m_Value(L), m_Value(R)).match(SI) || @@ -913,20 +989,20 @@ static Optional<ReductionData> getReductionData(Instruction *I) { m_UnordFMin(m_Value(L), m_Value(R)).match(SI) || m_UnordFMax(m_Value(L), m_Value(R)).match(SI)) { auto *CI = cast<CmpInst>(SI->getCondition()); - return ReductionData(RK_MinMax, CI->getOpcode(), L, R); + return TTI::ReductionData(TTI::RK_MinMax, CI->getOpcode(), L, R); } if (m_UMin(m_Value(L), m_Value(R)).match(SI) || m_UMax(m_Value(L), m_Value(R)).match(SI)) { auto *CI = cast<CmpInst>(SI->getCondition()); - return ReductionData(RK_UnsignedMinMax, CI->getOpcode(), L, R); + return TTI::ReductionData(TTI::RK_UnsignedMinMax, CI->getOpcode(), L, R); } } return llvm::None; } -static ReductionKind matchPairwiseReductionAtLevel(Instruction *I, - unsigned Level, - unsigned NumLevels) { +static TTI::ReductionKind matchPairwiseReductionAtLevel(Instruction *I, + unsigned Level, + unsigned NumLevels) { // Match one level of pairwise operations. // %rdx.shuf.0.0 = shufflevector <4 x float> %rdx, <4 x float> undef, // <4 x i32> <i32 0, i32 2 , i32 undef, i32 undef> @@ -934,24 +1010,24 @@ static ReductionKind matchPairwiseReductionAtLevel(Instruction *I, // <4 x i32> <i32 1, i32 3, i32 undef, i32 undef> // %bin.rdx.0 = fadd <4 x float> %rdx.shuf.0.0, %rdx.shuf.0.1 if (!I) - return RK_None; + return TTI::RK_None; assert(I->getType()->isVectorTy() && "Expecting a vector type"); - Optional<ReductionData> RD = getReductionData(I); + Optional<TTI::ReductionData> RD = getReductionData(I); if (!RD) - return RK_None; + return TTI::RK_None; ShuffleVectorInst *LS = dyn_cast<ShuffleVectorInst>(RD->LHS); if (!LS && Level) - return RK_None; + return TTI::RK_None; ShuffleVectorInst *RS = dyn_cast<ShuffleVectorInst>(RD->RHS); if (!RS && Level) - return RK_None; + return TTI::RK_None; // On level 0 we can omit one shufflevector instruction. if (!Level && !RS && !LS) - return RK_None; + return TTI::RK_None; // Shuffle inputs must match. Value *NextLevelOpL = LS ? LS->getOperand(0) : nullptr; @@ -960,7 +1036,7 @@ static ReductionKind matchPairwiseReductionAtLevel(Instruction *I, if (NextLevelOpR && NextLevelOpL) { // If we have two shuffles their operands must match. if (NextLevelOpL != NextLevelOpR) - return RK_None; + return TTI::RK_None; NextLevelOp = NextLevelOpL; } else if (Level == 0 && (NextLevelOpR || NextLevelOpL)) { @@ -971,46 +1047,48 @@ static ReductionKind matchPairwiseReductionAtLevel(Instruction *I, // %NextLevelOpL = shufflevector %R, <1, undef ...> // %BinOp = fadd %NextLevelOpL, %R if (NextLevelOpL && NextLevelOpL != RD->RHS) - return RK_None; + return TTI::RK_None; else if (NextLevelOpR && NextLevelOpR != RD->LHS) - return RK_None; + return TTI::RK_None; NextLevelOp = NextLevelOpL ? RD->RHS : RD->LHS; } else - return RK_None; + return TTI::RK_None; // Check that the next levels binary operation exists and matches with the // current one. if (Level + 1 != NumLevels) { - Optional<ReductionData> NextLevelRD = + if (!isa<Instruction>(NextLevelOp)) + return TTI::RK_None; + Optional<TTI::ReductionData> NextLevelRD = getReductionData(cast<Instruction>(NextLevelOp)); if (!NextLevelRD || !RD->hasSameData(*NextLevelRD)) - return RK_None; + return TTI::RK_None; } // Shuffle mask for pairwise operation must match. if (matchPairwiseShuffleMask(LS, /*IsLeft=*/true, Level)) { if (!matchPairwiseShuffleMask(RS, /*IsLeft=*/false, Level)) - return RK_None; + return TTI::RK_None; } else if (matchPairwiseShuffleMask(RS, /*IsLeft=*/true, Level)) { if (!matchPairwiseShuffleMask(LS, /*IsLeft=*/false, Level)) - return RK_None; + return TTI::RK_None; } else { - return RK_None; + return TTI::RK_None; } if (++Level == NumLevels) return RD->Kind; // Match next level. - return matchPairwiseReductionAtLevel(cast<Instruction>(NextLevelOp), Level, + return matchPairwiseReductionAtLevel(dyn_cast<Instruction>(NextLevelOp), Level, NumLevels); } -static ReductionKind matchPairwiseReduction(const ExtractElementInst *ReduxRoot, - unsigned &Opcode, Type *&Ty) { +TTI::ReductionKind TTI::matchPairwiseReduction( + const ExtractElementInst *ReduxRoot, unsigned &Opcode, VectorType *&Ty) { if (!EnableReduxCost) - return RK_None; + return TTI::RK_None; // Need to extract the first element. ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1)); @@ -1018,19 +1096,19 @@ static ReductionKind matchPairwiseReduction(const ExtractElementInst *ReduxRoot, if (CI) Idx = CI->getZExtValue(); if (Idx != 0) - return RK_None; + return TTI::RK_None; auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0)); if (!RdxStart) - return RK_None; - Optional<ReductionData> RD = getReductionData(RdxStart); + return TTI::RK_None; + Optional<TTI::ReductionData> RD = getReductionData(RdxStart); if (!RD) - return RK_None; + return TTI::RK_None; - Type *VecTy = RdxStart->getType(); - unsigned NumVecElems = VecTy->getVectorNumElements(); + auto *VecTy = cast<VectorType>(RdxStart->getType()); + unsigned NumVecElems = VecTy->getNumElements(); if (!isPowerOf2_32(NumVecElems)) - return RK_None; + return TTI::RK_None; // We look for a sequence of shuffle,shuffle,add triples like the following // that builds a pairwise reduction tree. @@ -1051,8 +1129,8 @@ static ReductionKind matchPairwiseReduction(const ExtractElementInst *ReduxRoot, // %bin.rdx8 = fadd <4 x float> %rdx.shuf.1.0, %rdx.shuf.1.1 // %r = extractelement <4 x float> %bin.rdx8, i32 0 if (matchPairwiseReductionAtLevel(RdxStart, 0, Log2_32(NumVecElems)) == - RK_None) - return RK_None; + TTI::RK_None) + return TTI::RK_None; Opcode = RD->Opcode; Ty = VecTy; @@ -1071,11 +1149,11 @@ getShuffleAndOtherOprd(Value *L, Value *R) { return std::make_pair(L, S); } -static ReductionKind -matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot, - unsigned &Opcode, Type *&Ty) { +TTI::ReductionKind TTI::matchVectorSplittingReduction( + const ExtractElementInst *ReduxRoot, unsigned &Opcode, VectorType *&Ty) { + if (!EnableReduxCost) - return RK_None; + return TTI::RK_None; // Need to extract the first element. ConstantInt *CI = dyn_cast<ConstantInt>(ReduxRoot->getOperand(1)); @@ -1083,19 +1161,19 @@ matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot, if (CI) Idx = CI->getZExtValue(); if (Idx != 0) - return RK_None; + return TTI::RK_None; auto *RdxStart = dyn_cast<Instruction>(ReduxRoot->getOperand(0)); if (!RdxStart) - return RK_None; - Optional<ReductionData> RD = getReductionData(RdxStart); + return TTI::RK_None; + Optional<TTI::ReductionData> RD = getReductionData(RdxStart); if (!RD) - return RK_None; + return TTI::RK_None; - Type *VecTy = ReduxRoot->getOperand(0)->getType(); - unsigned NumVecElems = VecTy->getVectorNumElements(); + auto *VecTy = cast<VectorType>(ReduxRoot->getOperand(0)->getType()); + unsigned NumVecElems = VecTy->getNumElements(); if (!isPowerOf2_32(NumVecElems)) - return RK_None; + return TTI::RK_None; // We look for a sequence of shuffles and adds like the following matching one // fadd, shuffle vector pair at a time. @@ -1115,10 +1193,10 @@ matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot, while (NumVecElemsRemain - 1) { // Check for the right reduction operation. if (!RdxOp) - return RK_None; - Optional<ReductionData> RDLevel = getReductionData(RdxOp); + return TTI::RK_None; + Optional<TTI::ReductionData> RDLevel = getReductionData(RdxOp); if (!RDLevel || !RDLevel->hasSameData(*RD)) - return RK_None; + return TTI::RK_None; Value *NextRdxOp; ShuffleVectorInst *Shuffle; @@ -1127,9 +1205,9 @@ matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot, // Check the current reduction operation and the shuffle use the same value. if (Shuffle == nullptr) - return RK_None; + return TTI::RK_None; if (Shuffle->getOperand(0) != NextRdxOp) - return RK_None; + return TTI::RK_None; // Check that shuffle masks matches. for (unsigned j = 0; j != MaskStart; ++j) @@ -1137,9 +1215,9 @@ matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot, // Fill the rest of the mask with -1 for undef. std::fill(&ShuffleMask[MaskStart], ShuffleMask.end(), -1); - SmallVector<int, 16> Mask = Shuffle->getShuffleMask(); + ArrayRef<int> Mask = Shuffle->getShuffleMask(); if (ShuffleMask != Mask) - return RK_None; + return TTI::RK_None; RdxOp = dyn_cast<Instruction>(NextRdxOp); NumVecElemsRemain /= 2; @@ -1152,15 +1230,13 @@ matchVectorSplittingReduction(const ExtractElementInst *ReduxRoot, } int TargetTransformInfo::getInstructionThroughput(const Instruction *I) const { + TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput; + switch (I->getOpcode()) { case Instruction::GetElementPtr: - return getUserCost(I); - case Instruction::Ret: case Instruction::PHI: - case Instruction::Br: { - return getCFInstrCost(I->getOpcode()); - } + case Instruction::Br: case Instruction::Add: case Instruction::FAdd: case Instruction::Sub: @@ -1178,48 +1254,13 @@ int TargetTransformInfo::getInstructionThroughput(const Instruction *I) const { case Instruction::AShr: case Instruction::And: case Instruction::Or: - case Instruction::Xor: { - TargetTransformInfo::OperandValueKind Op1VK, Op2VK; - TargetTransformInfo::OperandValueProperties Op1VP, Op2VP; - Op1VK = getOperandInfo(I->getOperand(0), Op1VP); - Op2VK = getOperandInfo(I->getOperand(1), Op2VP); - SmallVector<const Value *, 2> Operands(I->operand_values()); - return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK, - Op1VP, Op2VP, Operands, I); - } - case Instruction::FNeg: { - TargetTransformInfo::OperandValueKind Op1VK, Op2VK; - TargetTransformInfo::OperandValueProperties Op1VP, Op2VP; - Op1VK = getOperandInfo(I->getOperand(0), Op1VP); - Op2VK = OK_AnyValue; - Op2VP = OP_None; - SmallVector<const Value *, 2> Operands(I->operand_values()); - return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK, - Op1VP, Op2VP, Operands, I); - } - case Instruction::Select: { - const SelectInst *SI = cast<SelectInst>(I); - Type *CondTy = SI->getCondition()->getType(); - return getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy, I); - } + case Instruction::Xor: + case Instruction::FNeg: + case Instruction::Select: case Instruction::ICmp: - case Instruction::FCmp: { - Type *ValTy = I->getOperand(0)->getType(); - return getCmpSelInstrCost(I->getOpcode(), ValTy, I->getType(), I); - } - case Instruction::Store: { - const StoreInst *SI = cast<StoreInst>(I); - Type *ValTy = SI->getValueOperand()->getType(); - return getMemoryOpCost(I->getOpcode(), ValTy, - MaybeAlign(SI->getAlignment()), - SI->getPointerAddressSpace(), I); - } - case Instruction::Load: { - const LoadInst *LI = cast<LoadInst>(I); - return getMemoryOpCost(I->getOpcode(), I->getType(), - MaybeAlign(LI->getAlignment()), - LI->getPointerAddressSpace(), I); - } + case Instruction::FCmp: + case Instruction::Store: + case Instruction::Load: case Instruction::ZExt: case Instruction::SExt: case Instruction::FPToUI: @@ -1232,113 +1273,13 @@ int TargetTransformInfo::getInstructionThroughput(const Instruction *I) const { case Instruction::Trunc: case Instruction::FPTrunc: case Instruction::BitCast: - case Instruction::AddrSpaceCast: { - Type *SrcTy = I->getOperand(0)->getType(); - return getCastInstrCost(I->getOpcode(), I->getType(), SrcTy, I); - } - case Instruction::ExtractElement: { - const ExtractElementInst * EEI = cast<ExtractElementInst>(I); - ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1)); - unsigned Idx = -1; - if (CI) - Idx = CI->getZExtValue(); - - // Try to match a reduction sequence (series of shufflevector and vector - // adds followed by a extractelement). - unsigned ReduxOpCode; - Type *ReduxType; - - switch (matchVectorSplittingReduction(EEI, ReduxOpCode, ReduxType)) { - case RK_Arithmetic: - return getArithmeticReductionCost(ReduxOpCode, ReduxType, - /*IsPairwiseForm=*/false); - case RK_MinMax: - return getMinMaxReductionCost( - ReduxType, CmpInst::makeCmpResultType(ReduxType), - /*IsPairwiseForm=*/false, /*IsUnsigned=*/false); - case RK_UnsignedMinMax: - return getMinMaxReductionCost( - ReduxType, CmpInst::makeCmpResultType(ReduxType), - /*IsPairwiseForm=*/false, /*IsUnsigned=*/true); - case RK_None: - break; - } - - switch (matchPairwiseReduction(EEI, ReduxOpCode, ReduxType)) { - case RK_Arithmetic: - return getArithmeticReductionCost(ReduxOpCode, ReduxType, - /*IsPairwiseForm=*/true); - case RK_MinMax: - return getMinMaxReductionCost( - ReduxType, CmpInst::makeCmpResultType(ReduxType), - /*IsPairwiseForm=*/true, /*IsUnsigned=*/false); - case RK_UnsignedMinMax: - return getMinMaxReductionCost( - ReduxType, CmpInst::makeCmpResultType(ReduxType), - /*IsPairwiseForm=*/true, /*IsUnsigned=*/true); - case RK_None: - break; - } - - return getVectorInstrCost(I->getOpcode(), - EEI->getOperand(0)->getType(), Idx); - } - case Instruction::InsertElement: { - const InsertElementInst * IE = cast<InsertElementInst>(I); - ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2)); - unsigned Idx = -1; - if (CI) - Idx = CI->getZExtValue(); - return getVectorInstrCost(I->getOpcode(), - IE->getType(), Idx); - } + case Instruction::AddrSpaceCast: + case Instruction::ExtractElement: + case Instruction::InsertElement: case Instruction::ExtractValue: - return 0; // Model all ExtractValue nodes as free. - case Instruction::ShuffleVector: { - const ShuffleVectorInst *Shuffle = cast<ShuffleVectorInst>(I); - Type *Ty = Shuffle->getType(); - Type *SrcTy = Shuffle->getOperand(0)->getType(); - - // TODO: Identify and add costs for insert subvector, etc. - int SubIndex; - if (Shuffle->isExtractSubvectorMask(SubIndex)) - return TTIImpl->getShuffleCost(SK_ExtractSubvector, SrcTy, SubIndex, Ty); - - if (Shuffle->changesLength()) - return -1; - - if (Shuffle->isIdentity()) - return 0; - - if (Shuffle->isReverse()) - return TTIImpl->getShuffleCost(SK_Reverse, Ty, 0, nullptr); - - if (Shuffle->isSelect()) - return TTIImpl->getShuffleCost(SK_Select, Ty, 0, nullptr); - - if (Shuffle->isTranspose()) - return TTIImpl->getShuffleCost(SK_Transpose, Ty, 0, nullptr); - - if (Shuffle->isZeroEltSplat()) - return TTIImpl->getShuffleCost(SK_Broadcast, Ty, 0, nullptr); - - if (Shuffle->isSingleSource()) - return TTIImpl->getShuffleCost(SK_PermuteSingleSrc, Ty, 0, nullptr); - - return TTIImpl->getShuffleCost(SK_PermuteTwoSrc, Ty, 0, nullptr); - } + case Instruction::ShuffleVector: case Instruction::Call: - if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) { - SmallVector<Value *, 4> Args(II->arg_operands()); - - FastMathFlags FMF; - if (auto *FPMO = dyn_cast<FPMathOperator>(II)) - FMF = FPMO->getFastMathFlags(); - - return getIntrinsicInstrCost(II->getIntrinsicID(), II->getType(), - Args, FMF); - } - return -1; + return getUserCost(I, CostKind); default: // We don't have any information on this instruction. return -1; |