aboutsummaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp69
1 files changed, 42 insertions, 27 deletions
diff --git a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 9743fa0e7402..6ca93e15719f 100644
--- a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -2346,9 +2346,8 @@ emitTransformedIndex(IRBuilderBase &B, Value *Index, Value *StartValue,
auto *Offset = CreateMul(Index, Step);
return CreateAdd(StartValue, Offset);
}
- case InductionDescriptor::IK_PtrInduction: {
- return B.CreateGEP(B.getInt8Ty(), StartValue, CreateMul(Index, Step));
- }
+ case InductionDescriptor::IK_PtrInduction:
+ return B.CreatePtrAdd(StartValue, CreateMul(Index, Step));
case InductionDescriptor::IK_FpInduction: {
assert(!isa<VectorType>(Index->getType()) &&
"Vector indices not supported for FP inductions yet");
@@ -6950,10 +6949,25 @@ LoopVectorizationCostModel::getInstructionCost(Instruction *I, ElementCount VF,
Op2Info.Kind = TargetTransformInfo::OK_UniformValue;
SmallVector<const Value *, 4> Operands(I->operand_values());
- return TTI.getArithmeticInstrCost(
+ auto InstrCost = TTI.getArithmeticInstrCost(
I->getOpcode(), VectorTy, CostKind,
{TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
Op2Info, Operands, I);
+
+ // Some targets can replace frem with vector library calls.
+ InstructionCost VecCallCost = InstructionCost::getInvalid();
+ if (I->getOpcode() == Instruction::FRem) {
+ LibFunc Func;
+ if (TLI->getLibFunc(I->getOpcode(), I->getType(), Func) &&
+ TLI->isFunctionVectorizable(TLI->getName(Func), VF)) {
+ SmallVector<Type *, 4> OpTypes;
+ for (auto &Op : I->operands())
+ OpTypes.push_back(Op->getType());
+ VecCallCost =
+ TTI.getCallInstrCost(nullptr, VectorTy, OpTypes, CostKind);
+ }
+ }
+ return std::min(InstrCost, VecCallCost);
}
case Instruction::FNeg: {
return TTI.getArithmeticInstrCost(
@@ -8247,7 +8261,8 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
},
Range);
if (ShouldUseVectorIntrinsic)
- return new VPWidenCallRecipe(*CI, make_range(Ops.begin(), Ops.end()), ID);
+ return new VPWidenCallRecipe(*CI, make_range(Ops.begin(), Ops.end()), ID,
+ CI->getDebugLoc());
Function *Variant = nullptr;
std::optional<unsigned> MaskPos;
@@ -8300,7 +8315,8 @@ VPWidenCallRecipe *VPRecipeBuilder::tryToWidenCall(CallInst *CI,
}
return new VPWidenCallRecipe(*CI, make_range(Ops.begin(), Ops.end()),
- Intrinsic::not_intrinsic, Variant);
+ Intrinsic::not_intrinsic, CI->getDebugLoc(),
+ Variant);
}
return nullptr;
@@ -8949,16 +8965,17 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
"AnyOf reductions are not allowed for in-loop reductions");
// Collect the chain of "link" recipes for the reduction starting at PhiR.
- SetVector<VPRecipeBase *> Worklist;
+ SetVector<VPSingleDefRecipe *> Worklist;
Worklist.insert(PhiR);
for (unsigned I = 0; I != Worklist.size(); ++I) {
- VPRecipeBase *Cur = Worklist[I];
- for (VPUser *U : Cur->getVPSingleValue()->users()) {
- auto *UserRecipe = dyn_cast<VPRecipeBase>(U);
- if (!UserRecipe)
+ VPSingleDefRecipe *Cur = Worklist[I];
+ for (VPUser *U : Cur->users()) {
+ auto *UserRecipe = dyn_cast<VPSingleDefRecipe>(U);
+ if (!UserRecipe) {
+ assert(isa<VPLiveOut>(U) &&
+ "U must either be a VPSingleDef or VPLiveOut");
continue;
- assert(UserRecipe->getNumDefinedValues() == 1 &&
- "recipes must define exactly one result value");
+ }
Worklist.insert(UserRecipe);
}
}
@@ -8968,10 +8985,8 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
// (PreviousLink) to tell which of the two operands of a Link will remain
// scalar and which will be reduced. For minmax by select(cmp), Link will be
// the select instructions.
- VPRecipeBase *PreviousLink = PhiR; // Aka Worklist[0].
- for (VPRecipeBase *CurrentLink : Worklist.getArrayRef().drop_front()) {
- VPValue *PreviousLinkV = PreviousLink->getVPSingleValue();
-
+ VPSingleDefRecipe *PreviousLink = PhiR; // Aka Worklist[0].
+ for (VPSingleDefRecipe *CurrentLink : Worklist.getArrayRef().drop_front()) {
Instruction *CurrentLinkI = CurrentLink->getUnderlyingInstr();
// Index of the first operand which holds a non-mask vector operand.
@@ -8986,7 +9001,7 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
"Expected instruction to be a call to the llvm.fmuladd intrinsic");
assert(((MinVF.isScalar() && isa<VPReplicateRecipe>(CurrentLink)) ||
isa<VPWidenCallRecipe>(CurrentLink)) &&
- CurrentLink->getOperand(2) == PreviousLinkV &&
+ CurrentLink->getOperand(2) == PreviousLink &&
"expected a call where the previous link is the added operand");
// If the instruction is a call to the llvm.fmuladd intrinsic then we
@@ -9017,15 +9032,15 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
// Note that for non-commutable operands (cmp-selects), the semantics of
// the cmp-select are captured in the recurrence kind.
unsigned VecOpId =
- CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLinkV
+ CurrentLink->getOperand(IndexOfFirstOperand) == PreviousLink
? IndexOfFirstOperand + 1
: IndexOfFirstOperand;
VecOp = CurrentLink->getOperand(VecOpId);
- assert(VecOp != PreviousLinkV &&
+ assert(VecOp != PreviousLink &&
CurrentLink->getOperand(CurrentLink->getNumOperands() - 1 -
(VecOpId - IndexOfFirstOperand)) ==
- PreviousLinkV &&
- "PreviousLinkV must be the operand other than VecOp");
+ PreviousLink &&
+ "PreviousLink must be the operand other than VecOp");
}
BasicBlock *BB = CurrentLinkI->getParent();
@@ -9037,19 +9052,19 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
}
VPReductionRecipe *RedRecipe = new VPReductionRecipe(
- RdxDesc, CurrentLinkI, PreviousLinkV, VecOp, CondOp);
+ RdxDesc, CurrentLinkI, PreviousLink, VecOp, CondOp);
// Append the recipe to the end of the VPBasicBlock because we need to
// ensure that it comes after all of it's inputs, including CondOp.
// Note that this transformation may leave over dead recipes (including
// CurrentLink), which will be cleaned by a later VPlan transform.
LinkVPBB->appendRecipe(RedRecipe);
- CurrentLink->getVPSingleValue()->replaceAllUsesWith(RedRecipe);
+ CurrentLink->replaceAllUsesWith(RedRecipe);
PreviousLink = RedRecipe;
}
}
- Builder.setInsertPoint(&*LatchVPBB->begin());
- for (VPRecipeBase &R :
- Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
+ Builder.setInsertPoint(&*LatchVPBB->begin());
+ for (VPRecipeBase &R :
+ Plan->getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
VPReductionPHIRecipe *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
if (!PhiR)
continue;
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-13 04:36:45 +0000