summaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms/Vectorize/VPlan.h
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms/Vectorize/VPlan.h')
-rw-r--r--llvm/lib/Transforms/Vectorize/VPlan.h118
1 files changed, 80 insertions, 38 deletions
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 810dd5030f95..f4a1883e35d5 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -39,6 +39,7 @@
#include "llvm/ADT/ilist.h"
#include "llvm/ADT/ilist_node.h"
#include "llvm/Analysis/VectorUtils.h"
+#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/Support/InstructionCost.h"
#include <algorithm>
@@ -51,6 +52,7 @@ namespace llvm {
class BasicBlock;
class DominatorTree;
+class InductionDescriptor;
class InnerLoopVectorizer;
class LoopInfo;
class raw_ostream;
@@ -500,6 +502,8 @@ public:
const VPBlocksTy &getSuccessors() const { return Successors; }
VPBlocksTy &getSuccessors() { return Successors; }
+ iterator_range<VPBlockBase **> successors() { return Successors; }
+
const VPBlocksTy &getPredecessors() const { return Predecessors; }
VPBlocksTy &getPredecessors() { return Predecessors; }
@@ -795,6 +799,7 @@ private:
typedef unsigned char OpcodeTy;
OpcodeTy Opcode;
FastMathFlags FMF;
+ DebugLoc DL;
/// Utility method serving execute(): generates a single instance of the
/// modeled instruction.
@@ -804,12 +809,14 @@ protected:
void setUnderlyingInstr(Instruction *I) { setUnderlyingValue(I); }
public:
- VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands)
+ VPInstruction(unsigned Opcode, ArrayRef<VPValue *> Operands, DebugLoc DL)
: VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands),
- VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode) {}
+ VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode),
+ DL(DL) {}
- VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands)
- : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands)) {}
+ VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands,
+ DebugLoc DL = {})
+ : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands), DL) {}
/// Method to support type inquiry through isa, cast, and dyn_cast.
static inline bool classof(const VPValue *V) {
@@ -818,7 +825,7 @@ public:
VPInstruction *clone() const {
SmallVector<VPValue *, 2> Operands(operands());
- return new VPInstruction(Opcode, Operands);
+ return new VPInstruction(Opcode, Operands, DL);
}
/// Method to support type inquiry through isa, cast, and dyn_cast.
@@ -1003,21 +1010,22 @@ public:
/// A recipe for handling phi nodes of integer and floating-point inductions,
/// producing their vector and scalar values.
-class VPWidenIntOrFpInductionRecipe : public VPRecipeBase {
+class VPWidenIntOrFpInductionRecipe : public VPRecipeBase, public VPValue {
PHINode *IV;
+ const InductionDescriptor &IndDesc;
public:
- VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, Instruction *Cast,
- TruncInst *Trunc = nullptr)
- : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV) {
- if (Trunc)
- new VPValue(Trunc, this);
- else
- new VPValue(IV, this);
+ VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start,
+ const InductionDescriptor &IndDesc)
+ : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), VPValue(IV, this),
+ IV(IV), IndDesc(IndDesc) {}
+
+ VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start,
+ const InductionDescriptor &IndDesc,
+ TruncInst *Trunc)
+ : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), VPValue(Trunc, this),
+ IV(IV), IndDesc(IndDesc) {}
- if (Cast)
- new VPValue(Cast, this);
- }
~VPWidenIntOrFpInductionRecipe() override = default;
/// Method to support type inquiry through isa, cast, and dyn_cast.
@@ -1038,13 +1046,6 @@ public:
/// Returns the start value of the induction.
VPValue *getStartValue() { return getOperand(0); }
- /// Returns the cast VPValue, if one is attached, or nullptr otherwise.
- VPValue *getCastValue() {
- if (getNumDefinedValues() != 2)
- return nullptr;
- return getVPValue(1);
- }
-
/// Returns the first defined value as TruncInst, if it is one or nullptr
/// otherwise.
TruncInst *getTruncInst() {
@@ -1053,6 +1054,9 @@ public:
const TruncInst *getTruncInst() const {
return dyn_cast_or_null<TruncInst>(getVPValue(0)->getUnderlyingValue());
}
+
+ /// Returns the induction descriptor for the recipe.
+ const InductionDescriptor &getInductionDescriptor() const { return IndDesc; }
};
/// A recipe for handling first order recurrences and pointer inductions. For
@@ -1169,7 +1173,7 @@ struct VPFirstOrderRecurrencePHIRecipe : public VPWidenPHIRecipe {
/// operand.
class VPReductionPHIRecipe : public VPWidenPHIRecipe {
/// Descriptor for the reduction.
- RecurrenceDescriptor &RdxDesc;
+ const RecurrenceDescriptor &RdxDesc;
/// The phi is part of an in-loop reduction.
bool IsInLoop;
@@ -1180,7 +1184,7 @@ class VPReductionPHIRecipe : public VPWidenPHIRecipe {
public:
/// Create a new VPReductionPHIRecipe for the reduction \p Phi described by \p
/// RdxDesc.
- VPReductionPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc,
+ VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc,
VPValue &Start, bool IsInLoop = false,
bool IsOrdered = false)
: VPWidenPHIRecipe(VPVReductionPHISC, VPReductionPHISC, Phi, &Start),
@@ -1210,7 +1214,9 @@ public:
VPSlotTracker &SlotTracker) const override;
#endif
- RecurrenceDescriptor &getRecurrenceDescriptor() { return RdxDesc; }
+ const RecurrenceDescriptor &getRecurrenceDescriptor() const {
+ return RdxDesc;
+ }
/// Returns true, if the phi is part of an ordered reduction.
bool isOrdered() const { return IsOrdered; }
@@ -1340,13 +1346,13 @@ public:
/// The Operands are {ChainOp, VecOp, [Condition]}.
class VPReductionRecipe : public VPRecipeBase, public VPValue {
/// The recurrence decriptor for the reduction in question.
- RecurrenceDescriptor *RdxDesc;
+ const RecurrenceDescriptor *RdxDesc;
/// Pointer to the TTI, needed to create the target reduction
const TargetTransformInfo *TTI;
public:
- VPReductionRecipe(RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp,
- VPValue *VecOp, VPValue *CondOp,
+ VPReductionRecipe(const RecurrenceDescriptor *R, Instruction *I,
+ VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp,
const TargetTransformInfo *TTI)
: VPRecipeBase(VPRecipeBase::VPReductionSC, {ChainOp, VecOp}),
VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), TTI(TTI) {
@@ -2252,6 +2258,12 @@ public:
return map_range(Operands, Fn);
}
+ /// Returns true if \p VPV is uniform after vectorization.
+ bool isUniformAfterVectorization(VPValue *VPV) const {
+ auto RepR = dyn_cast_or_null<VPReplicateRecipe>(VPV->getDef());
+ return !VPV->getDef() || (RepR && RepR->isUniform());
+ }
+
private:
/// Add to the given dominator tree the header block and every new basic block
/// that was created between it and the latch block, inclusive.
@@ -2340,18 +2352,23 @@ public:
/// Insert disconnected VPBlockBase \p NewBlock after \p BlockPtr. Add \p
/// NewBlock as successor of \p BlockPtr and \p BlockPtr as predecessor of \p
- /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. If \p BlockPtr
- /// has more than one successor, its conditional bit is propagated to \p
- /// NewBlock. \p NewBlock must have neither successors nor predecessors.
+ /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. \p BlockPtr's
+ /// successors are moved from \p BlockPtr to \p NewBlock and \p BlockPtr's
+ /// conditional bit is propagated to \p NewBlock. \p NewBlock must have
+ /// neither successors nor predecessors.
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) {
assert(NewBlock->getSuccessors().empty() &&
- "Can't insert new block with successors.");
- // TODO: move successors from BlockPtr to NewBlock when this functionality
- // is necessary. For now, setBlockSingleSuccessor will assert if BlockPtr
- // already has successors.
- BlockPtr->setOneSuccessor(NewBlock);
- NewBlock->setPredecessors({BlockPtr});
+ NewBlock->getPredecessors().empty() &&
+ "Can't insert new block with predecessors or successors.");
NewBlock->setParent(BlockPtr->getParent());
+ SmallVector<VPBlockBase *> Succs(BlockPtr->successors());
+ for (VPBlockBase *Succ : Succs) {
+ disconnectBlocks(BlockPtr, Succ);
+ connectBlocks(NewBlock, Succ);
+ }
+ NewBlock->setCondBit(BlockPtr->getCondBit());
+ BlockPtr->setCondBit(nullptr);
+ connectBlocks(BlockPtr, NewBlock);
}
/// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p
@@ -2394,6 +2411,31 @@ public:
To->removePredecessor(From);
}
+ /// Try to merge \p Block into its single predecessor, if \p Block is a
+ /// VPBasicBlock and its predecessor has a single successor. Returns a pointer
+ /// to the predecessor \p Block was merged into or nullptr otherwise.
+ static VPBasicBlock *tryToMergeBlockIntoPredecessor(VPBlockBase *Block) {
+ auto *VPBB = dyn_cast<VPBasicBlock>(Block);
+ auto *PredVPBB =
+ dyn_cast_or_null<VPBasicBlock>(Block->getSinglePredecessor());
+ if (!VPBB || !PredVPBB || PredVPBB->getNumSuccessors() != 1)
+ return nullptr;
+
+ for (VPRecipeBase &R : make_early_inc_range(*VPBB))
+ R.moveBefore(*PredVPBB, PredVPBB->end());
+ VPBlockUtils::disconnectBlocks(PredVPBB, VPBB);
+ auto *ParentRegion = cast<VPRegionBlock>(Block->getParent());
+ if (ParentRegion->getExit() == Block)
+ ParentRegion->setExit(PredVPBB);
+ SmallVector<VPBlockBase *> Successors(Block->successors());
+ for (auto *Succ : Successors) {
+ VPBlockUtils::disconnectBlocks(Block, Succ);
+ VPBlockUtils::connectBlocks(PredVPBB, Succ);
+ }
+ delete Block;
+ return PredVPBB;
+ }
+
/// Returns true if the edge \p FromBlock -> \p ToBlock is a back-edge.
static bool isBackEdge(const VPBlockBase *FromBlock,
const VPBlockBase *ToBlock, const VPLoopInfo *VPLI) {
generated by cgit v1.3 (git 2.53.0) at 2026-05-14 06:12:57 +0000