aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp')
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp104
1 files changed, 69 insertions, 35 deletions
diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
index 52b5ae083d0e..ded5bc04beb5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
@@ -31,19 +31,18 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
VPBasicBlock *VPBB = Base->getEntryBasicBlock();
// Introduce each ingredient into VPlan.
- for (auto I = VPBB->begin(), E = VPBB->end(); I != E;) {
- VPRecipeBase *Ingredient = &*I++;
- VPValue *VPV = Ingredient->getVPSingleValue();
+ for (VPRecipeBase &Ingredient : llvm::make_early_inc_range(*VPBB)) {
+ VPValue *VPV = Ingredient.getVPSingleValue();
Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
if (DeadInstructions.count(Inst)) {
VPValue DummyValue;
VPV->replaceAllUsesWith(&DummyValue);
- Ingredient->eraseFromParent();
+ Ingredient.eraseFromParent();
continue;
}
VPRecipeBase *NewRecipe = nullptr;
- if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(Ingredient)) {
+ if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
InductionDescriptor II = Inductions.lookup(Phi);
if (II.getKind() == InductionDescriptor::IK_IntInduction ||
@@ -55,25 +54,25 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
continue;
}
} else {
- assert(isa<VPInstruction>(Ingredient) &&
+ assert(isa<VPInstruction>(&Ingredient) &&
"only VPInstructions expected here");
assert(!isa<PHINode>(Inst) && "phis should be handled above");
// Create VPWidenMemoryInstructionRecipe for loads and stores.
if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
NewRecipe = new VPWidenMemoryInstructionRecipe(
*Load, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
- nullptr /*Mask*/);
+ nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/);
} else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
NewRecipe = new VPWidenMemoryInstructionRecipe(
*Store, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
- Plan->getOrAddVPValue(Store->getValueOperand()),
- nullptr /*Mask*/);
+ Plan->getOrAddVPValue(Store->getValueOperand()), nullptr /*Mask*/,
+ false /*Consecutive*/, false /*Reverse*/);
} else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
NewRecipe = new VPWidenGEPRecipe(
GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop);
} else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
- NewRecipe = new VPWidenCallRecipe(
- *CI, Plan->mapToVPValues(CI->arg_operands()));
+ NewRecipe =
+ new VPWidenCallRecipe(*CI, Plan->mapToVPValues(CI->args()));
} else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
bool InvariantCond =
SE.isLoopInvariant(SE.getSCEV(SI->getOperand(0)), OrigLoop);
@@ -85,13 +84,13 @@ void VPlanTransforms::VPInstructionsToVPRecipes(
}
}
- NewRecipe->insertBefore(Ingredient);
+ NewRecipe->insertBefore(&Ingredient);
if (NewRecipe->getNumDefinedValues() == 1)
VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
else
assert(NewRecipe->getNumDefinedValues() == 0 &&
"Only recpies with zero or one defined values expected");
- Ingredient->eraseFromParent();
+ Ingredient.eraseFromParent();
Plan->removeVPValueFor(Inst);
for (auto *Def : NewRecipe->definedValues()) {
Plan->addVPValue(Inst, Def);
@@ -106,44 +105,76 @@ bool VPlanTransforms::sinkScalarOperands(VPlan &Plan) {
bool Changed = false;
// First, collect the operands of all predicated replicate recipes as seeds
// for sinking.
- SetVector<VPValue *> WorkList;
+ SetVector<std::pair<VPBasicBlock *, VPValue *>> WorkList;
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
for (auto &Recipe : *VPBB) {
auto *RepR = dyn_cast<VPReplicateRecipe>(&Recipe);
if (!RepR || !RepR->isPredicated())
continue;
- WorkList.insert(RepR->op_begin(), RepR->op_end());
+ for (VPValue *Op : RepR->operands())
+ WorkList.insert(std::make_pair(RepR->getParent(), Op));
}
}
// Try to sink each replicate recipe in the worklist.
while (!WorkList.empty()) {
- auto *C = WorkList.pop_back_val();
+ VPBasicBlock *SinkTo;
+ VPValue *C;
+ std::tie(SinkTo, C) = WorkList.pop_back_val();
auto *SinkCandidate = dyn_cast_or_null<VPReplicateRecipe>(C->Def);
- if (!SinkCandidate || SinkCandidate->isUniform())
- continue;
-
- // All users of SinkCandidate must be in the same block in order to perform
- // sinking. Therefore the destination block for sinking must match the block
- // containing the first user.
- auto *FirstUser = dyn_cast<VPRecipeBase>(*SinkCandidate->user_begin());
- if (!FirstUser)
- continue;
- VPBasicBlock *SinkTo = FirstUser->getParent();
- if (SinkCandidate->getParent() == SinkTo ||
+ if (!SinkCandidate || SinkCandidate->isUniform() ||
+ SinkCandidate->getParent() == SinkTo ||
SinkCandidate->mayHaveSideEffects() ||
SinkCandidate->mayReadOrWriteMemory())
continue;
- // All recipe users of the sink candidate must be in the same block SinkTo.
- if (any_of(SinkCandidate->users(), [SinkTo](VPUser *U) {
- auto *UI = dyn_cast<VPRecipeBase>(U);
- return !UI || UI->getParent() != SinkTo;
- }))
+ bool NeedsDuplicating = false;
+ // All recipe users of the sink candidate must be in the same block SinkTo
+ // or all users outside of SinkTo must be uniform-after-vectorization (
+ // i.e., only first lane is used) . In the latter case, we need to duplicate
+ // SinkCandidate. At the moment, we identify such UAV's by looking for the
+ // address operands of widened memory recipes.
+ auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
+ SinkCandidate](VPUser *U) {
+ auto *UI = dyn_cast<VPRecipeBase>(U);
+ if (!UI)
+ return false;
+ if (UI->getParent() == SinkTo)
+ return true;
+ auto *WidenI = dyn_cast<VPWidenMemoryInstructionRecipe>(UI);
+ if (WidenI && WidenI->getAddr() == SinkCandidate) {
+ NeedsDuplicating = true;
+ return true;
+ }
+ return false;
+ };
+ if (!all_of(SinkCandidate->users(), CanSinkWithUser))
continue;
+ if (NeedsDuplicating) {
+ Instruction *I = cast<Instruction>(SinkCandidate->getUnderlyingValue());
+ auto *Clone =
+ new VPReplicateRecipe(I, SinkCandidate->operands(), true, false);
+ // TODO: add ".cloned" suffix to name of Clone's VPValue.
+
+ Clone->insertBefore(SinkCandidate);
+ SmallVector<VPUser *, 4> Users(SinkCandidate->user_begin(),
+ SinkCandidate->user_end());
+ for (auto *U : Users) {
+ auto *UI = cast<VPRecipeBase>(U);
+ if (UI->getParent() == SinkTo)
+ continue;
+
+ for (unsigned Idx = 0; Idx != UI->getNumOperands(); Idx++) {
+ if (UI->getOperand(Idx) != SinkCandidate)
+ continue;
+ UI->setOperand(Idx, Clone);
+ }
+ }
+ }
SinkCandidate->moveBefore(*SinkTo, SinkTo->getFirstNonPhi());
- WorkList.insert(SinkCandidate->op_begin(), SinkCandidate->op_end());
+ for (VPValue *Op : SinkCandidate->operands())
+ WorkList.insert(std::make_pair(SinkTo, Op));
Changed = true;
}
return Changed;
@@ -234,12 +265,15 @@ bool VPlanTransforms::mergeReplicateRegions(VPlan &Plan) {
for (VPRecipeBase &Phi1ToMove : make_early_inc_range(reverse(*Merge1))) {
VPValue *PredInst1 =
cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
- for (VPUser *U : Phi1ToMove.getVPSingleValue()->users()) {
+ VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
+ SmallVector<VPUser *> Users(Phi1ToMoveV->user_begin(),
+ Phi1ToMoveV->user_end());
+ for (VPUser *U : Users) {
auto *UI = dyn_cast<VPRecipeBase>(U);
if (!UI || UI->getParent() != Then2)
continue;
for (unsigned I = 0, E = U->getNumOperands(); I != E; ++I) {
- if (Phi1ToMove.getVPSingleValue() != U->getOperand(I))
+ if (Phi1ToMoveV != U->getOperand(I))
continue;
U->setOperand(I, PredInst1);
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-12 01:20:59 +0000