aboutsummaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp')
-rw-r--r--llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp258
1 files changed, 146 insertions, 112 deletions
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index 61e62adbe327..4a5ffef2b08e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -171,8 +171,11 @@ Instruction *InstCombinerImpl::scalarizePHI(ExtractElementInst &EI,
}
}
- for (auto *E : Extracts)
+ for (auto *E : Extracts) {
replaceInstUsesWith(*E, scalarPHI);
+ // Add old extract to worklist for DCE.
+ addToWorklist(E);
+ }
return &EI;
}
@@ -384,7 +387,7 @@ static APInt findDemandedEltsByAllUsers(Value *V) {
/// return it with the canonical type if it isn't already canonical. We
/// arbitrarily pick 64 bit as our canonical type. The actual bitwidth doesn't
/// matter, we just want a consistent type to simplify CSE.
-ConstantInt *getPreferredVectorIndex(ConstantInt *IndexC) {
+static ConstantInt *getPreferredVectorIndex(ConstantInt *IndexC) {
const unsigned IndexBW = IndexC->getType()->getBitWidth();
if (IndexBW == 64 || IndexC->getValue().getActiveBits() > 64)
return nullptr;
@@ -543,16 +546,16 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
->getNumElements();
if (SrcIdx < 0)
- return replaceInstUsesWith(EI, UndefValue::get(EI.getType()));
+ return replaceInstUsesWith(EI, PoisonValue::get(EI.getType()));
if (SrcIdx < (int)LHSWidth)
Src = SVI->getOperand(0);
else {
SrcIdx -= LHSWidth;
Src = SVI->getOperand(1);
}
- Type *Int32Ty = Type::getInt32Ty(EI.getContext());
+ Type *Int64Ty = Type::getInt64Ty(EI.getContext());
return ExtractElementInst::Create(
- Src, ConstantInt::get(Int32Ty, SrcIdx, false));
+ Src, ConstantInt::get(Int64Ty, SrcIdx, false));
}
} else if (auto *CI = dyn_cast<CastInst>(I)) {
// Canonicalize extractelement(cast) -> cast(extractelement).
@@ -594,6 +597,7 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) {
SrcVec, DemandedElts, UndefElts, 0 /* Depth */,
true /* AllowMultipleUsers */)) {
if (V != SrcVec) {
+ Worklist.addValue(SrcVec);
SrcVec->replaceAllUsesWith(V);
return &EI;
}
@@ -640,11 +644,11 @@ static bool collectSingleShuffleElements(Value *V, Value *LHS, Value *RHS,
return false;
unsigned InsertedIdx = cast<ConstantInt>(IdxOp)->getZExtValue();
- if (isa<UndefValue>(ScalarOp)) { // inserting undef into vector.
+ if (isa<PoisonValue>(ScalarOp)) { // inserting poison into vector.
// We can handle this if the vector we are inserting into is
// transitively ok.
if (collectSingleShuffleElements(VecOp, LHS, RHS, Mask)) {
- // If so, update the mask to reflect the inserted undef.
+ // If so, update the mask to reflect the inserted poison.
Mask[InsertedIdx] = -1;
return true;
}
@@ -680,7 +684,7 @@ static bool collectSingleShuffleElements(Value *V, Value *LHS, Value *RHS,
/// If we have insertion into a vector that is wider than the vector that we
/// are extracting from, try to widen the source vector to allow a single
/// shufflevector to replace one or more insert/extract pairs.
-static void replaceExtractElements(InsertElementInst *InsElt,
+static bool replaceExtractElements(InsertElementInst *InsElt,
ExtractElementInst *ExtElt,
InstCombinerImpl &IC) {
auto *InsVecType = cast<FixedVectorType>(InsElt->getType());
@@ -691,7 +695,7 @@ static void replaceExtractElements(InsertElementInst *InsElt,
// The inserted-to vector must be wider than the extracted-from vector.
if (InsVecType->getElementType() != ExtVecType->getElementType() ||
NumExtElts >= NumInsElts)
- return;
+ return false;
// Create a shuffle mask to widen the extended-from vector using poison
// values. The mask selects all of the values of the original vector followed
@@ -719,7 +723,7 @@ static void replaceExtractElements(InsertElementInst *InsElt,
// that will delete our widening shuffle. This would trigger another attempt
// here to create that shuffle, and we spin forever.
if (InsertionBlock != InsElt->getParent())
- return;
+ return false;
// TODO: This restriction matches the check in visitInsertElementInst() and
// prevents an infinite loop caused by not turning the extract/insert pair
@@ -727,7 +731,7 @@ static void replaceExtractElements(InsertElementInst *InsElt,
// folds for shufflevectors because we're afraid to generate shuffle masks
// that the backend can't handle.
if (InsElt->hasOneUse() && isa<InsertElementInst>(InsElt->user_back()))
- return;
+ return false;
auto *WideVec = new ShuffleVectorInst(ExtVecOp, ExtendMask);
@@ -747,9 +751,14 @@ static void replaceExtractElements(InsertElementInst *InsElt,
if (!OldExt || OldExt->getParent() != WideVec->getParent())
continue;
auto *NewExt = ExtractElementInst::Create(WideVec, OldExt->getOperand(1));
- NewExt->insertAfter(OldExt);
+ IC.InsertNewInstWith(NewExt, *OldExt);
IC.replaceInstUsesWith(*OldExt, NewExt);
+ // Add the old extracts to the worklist for DCE. We can't remove the
+ // extracts directly, because they may still be used by the calling code.
+ IC.addToWorklist(OldExt);
}
+
+ return true;
}
/// We are building a shuffle to create V, which is a sequence of insertelement,
@@ -764,7 +773,7 @@ using ShuffleOps = std::pair<Value *, Value *>;
static ShuffleOps collectShuffleElements(Value *V, SmallVectorImpl<int> &Mask,
Value *PermittedRHS,
- InstCombinerImpl &IC) {
+ InstCombinerImpl &IC, bool &Rerun) {
assert(V->getType()->isVectorTy() && "Invalid shuffle!");
unsigned NumElts = cast<FixedVectorType>(V->getType())->getNumElements();
@@ -795,13 +804,14 @@ static ShuffleOps collectShuffleElements(Value *V, SmallVectorImpl<int> &Mask,
// otherwise we'd end up with a shuffle of three inputs.
if (EI->getOperand(0) == PermittedRHS || PermittedRHS == nullptr) {
Value *RHS = EI->getOperand(0);
- ShuffleOps LR = collectShuffleElements(VecOp, Mask, RHS, IC);
+ ShuffleOps LR = collectShuffleElements(VecOp, Mask, RHS, IC, Rerun);
assert(LR.second == nullptr || LR.second == RHS);
if (LR.first->getType() != RHS->getType()) {
// Although we are giving up for now, see if we can create extracts
// that match the inserts for another round of combining.
- replaceExtractElements(IEI, EI, IC);
+ if (replaceExtractElements(IEI, EI, IC))
+ Rerun = true;
// We tried our best, but we can't find anything compatible with RHS
// further up the chain. Return a trivial shuffle.
@@ -1129,6 +1139,11 @@ Instruction *InstCombinerImpl::foldAggregateConstructionIntoAggregateReuse(
/// It should be transformed to:
/// %0 = insertvalue { i8, i32 } undef, i8 %y, 0
Instruction *InstCombinerImpl::visitInsertValueInst(InsertValueInst &I) {
+ if (Value *V = simplifyInsertValueInst(
+ I.getAggregateOperand(), I.getInsertedValueOperand(), I.getIndices(),
+ SQ.getWithInstruction(&I)))
+ return replaceInstUsesWith(I, V);
+
bool IsRedundant = false;
ArrayRef<unsigned int> FirstIndices = I.getIndices();
@@ -1235,22 +1250,22 @@ static Instruction *foldInsSequenceIntoSplat(InsertElementInst &InsElt) {
if (FirstIE == &InsElt)
return nullptr;
- // If we are not inserting into an undef vector, make sure we've seen an
+ // If we are not inserting into a poison vector, make sure we've seen an
// insert into every element.
// TODO: If the base vector is not undef, it might be better to create a splat
// and then a select-shuffle (blend) with the base vector.
- if (!match(FirstIE->getOperand(0), m_Undef()))
+ if (!match(FirstIE->getOperand(0), m_Poison()))
if (!ElementPresent.all())
return nullptr;
// Create the insert + shuffle.
- Type *Int32Ty = Type::getInt32Ty(InsElt.getContext());
+ Type *Int64Ty = Type::getInt64Ty(InsElt.getContext());
PoisonValue *PoisonVec = PoisonValue::get(VecTy);
- Constant *Zero = ConstantInt::get(Int32Ty, 0);
+ Constant *Zero = ConstantInt::get(Int64Ty, 0);
if (!cast<ConstantInt>(FirstIE->getOperand(2))->isZero())
FirstIE = InsertElementInst::Create(PoisonVec, SplatVal, Zero, "", &InsElt);
- // Splat from element 0, but replace absent elements with undef in the mask.
+ // Splat from element 0, but replace absent elements with poison in the mask.
SmallVector<int, 16> Mask(NumElements, 0);
for (unsigned i = 0; i != NumElements; ++i)
if (!ElementPresent[i])
@@ -1339,7 +1354,7 @@ static Instruction *foldInsEltIntoIdentityShuffle(InsertElementInst &InsElt) {
// (demanded elements analysis may unset it later).
return nullptr;
} else {
- assert(OldMask[i] == UndefMaskElem &&
+ assert(OldMask[i] == PoisonMaskElem &&
"Unexpected shuffle mask element for identity shuffle");
NewMask[i] = IdxC;
}
@@ -1465,10 +1480,10 @@ static Instruction *foldConstantInsEltIntoShuffle(InsertElementInst &InsElt) {
}
++ValI;
}
- // Remaining values are filled with 'undef' values.
+ // Remaining values are filled with 'poison' values.
for (unsigned I = 0; I < NumElts; ++I) {
if (!Values[I]) {
- Values[I] = UndefValue::get(InsElt.getType()->getElementType());
+ Values[I] = PoisonValue::get(InsElt.getType()->getElementType());
Mask[I] = I;
}
}
@@ -1676,16 +1691,22 @@ Instruction *InstCombinerImpl::visitInsertElementInst(InsertElementInst &IE) {
// Try to form a shuffle from a chain of extract-insert ops.
if (isShuffleRootCandidate(IE)) {
- SmallVector<int, 16> Mask;
- ShuffleOps LR = collectShuffleElements(&IE, Mask, nullptr, *this);
-
- // The proposed shuffle may be trivial, in which case we shouldn't
- // perform the combine.
- if (LR.first != &IE && LR.second != &IE) {
- // We now have a shuffle of LHS, RHS, Mask.
- if (LR.second == nullptr)
- LR.second = UndefValue::get(LR.first->getType());
- return new ShuffleVectorInst(LR.first, LR.second, Mask);
+ bool Rerun = true;
+ while (Rerun) {
+ Rerun = false;
+
+ SmallVector<int, 16> Mask;
+ ShuffleOps LR =
+ collectShuffleElements(&IE, Mask, nullptr, *this, Rerun);
+
+ // The proposed shuffle may be trivial, in which case we shouldn't
+ // perform the combine.
+ if (LR.first != &IE && LR.second != &IE) {
+ // We now have a shuffle of LHS, RHS, Mask.
+ if (LR.second == nullptr)
+ LR.second = PoisonValue::get(LR.first->getType());
+ return new ShuffleVectorInst(LR.first, LR.second, Mask);
+ }
}
}
}
@@ -1815,9 +1836,9 @@ static bool canEvaluateShuffled(Value *V, ArrayRef<int> Mask,
/// Rebuild a new instruction just like 'I' but with the new operands given.
/// In the event of type mismatch, the type of the operands is correct.
-static Value *buildNew(Instruction *I, ArrayRef<Value*> NewOps) {
- // We don't want to use the IRBuilder here because we want the replacement
- // instructions to appear next to 'I', not the builder's insertion point.
+static Value *buildNew(Instruction *I, ArrayRef<Value*> NewOps,
+ IRBuilderBase &Builder) {
+ Builder.SetInsertPoint(I);
switch (I->getOpcode()) {
case Instruction::Add:
case Instruction::FAdd:
@@ -1839,28 +1860,29 @@ static Value *buildNew(Instruction *I, ArrayRef<Value*> NewOps) {
case Instruction::Xor: {
BinaryOperator *BO = cast<BinaryOperator>(I);
assert(NewOps.size() == 2 && "binary operator with #ops != 2");
- BinaryOperator *New =
- BinaryOperator::Create(cast<BinaryOperator>(I)->getOpcode(),
- NewOps[0], NewOps[1], "", BO);
- if (isa<OverflowingBinaryOperator>(BO)) {
- New->setHasNoUnsignedWrap(BO->hasNoUnsignedWrap());
- New->setHasNoSignedWrap(BO->hasNoSignedWrap());
- }
- if (isa<PossiblyExactOperator>(BO)) {
- New->setIsExact(BO->isExact());
+ Value *New = Builder.CreateBinOp(cast<BinaryOperator>(I)->getOpcode(),
+ NewOps[0], NewOps[1]);
+ if (auto *NewI = dyn_cast<Instruction>(New)) {
+ if (isa<OverflowingBinaryOperator>(BO)) {
+ NewI->setHasNoUnsignedWrap(BO->hasNoUnsignedWrap());
+ NewI->setHasNoSignedWrap(BO->hasNoSignedWrap());
+ }
+ if (isa<PossiblyExactOperator>(BO)) {
+ NewI->setIsExact(BO->isExact());
+ }
+ if (isa<FPMathOperator>(BO))
+ NewI->copyFastMathFlags(I);
}
- if (isa<FPMathOperator>(BO))
- New->copyFastMathFlags(I);
return New;
}
case Instruction::ICmp:
assert(NewOps.size() == 2 && "icmp with #ops != 2");
- return new ICmpInst(I, cast<ICmpInst>(I)->getPredicate(),
- NewOps[0], NewOps[1]);
+ return Builder.CreateICmp(cast<ICmpInst>(I)->getPredicate(), NewOps[0],
+ NewOps[1]);
case Instruction::FCmp:
assert(NewOps.size() == 2 && "fcmp with #ops != 2");
- return new FCmpInst(I, cast<FCmpInst>(I)->getPredicate(),
- NewOps[0], NewOps[1]);
+ return Builder.CreateFCmp(cast<FCmpInst>(I)->getPredicate(), NewOps[0],
+ NewOps[1]);
case Instruction::Trunc:
case Instruction::ZExt:
case Instruction::SExt:
@@ -1876,27 +1898,26 @@ static Value *buildNew(Instruction *I, ArrayRef<Value*> NewOps) {
I->getType()->getScalarType(),
cast<VectorType>(NewOps[0]->getType())->getElementCount());
assert(NewOps.size() == 1 && "cast with #ops != 1");
- return CastInst::Create(cast<CastInst>(I)->getOpcode(), NewOps[0], DestTy,
- "", I);
+ return Builder.CreateCast(cast<CastInst>(I)->getOpcode(), NewOps[0],
+ DestTy);
}
case Instruction::GetElementPtr: {
Value *Ptr = NewOps[0];
ArrayRef<Value*> Idx = NewOps.slice(1);
- GetElementPtrInst *GEP = GetElementPtrInst::Create(
- cast<GetElementPtrInst>(I)->getSourceElementType(), Ptr, Idx, "", I);
- GEP->setIsInBounds(cast<GetElementPtrInst>(I)->isInBounds());
- return GEP;
+ return Builder.CreateGEP(cast<GEPOperator>(I)->getSourceElementType(),
+ Ptr, Idx, "",
+ cast<GEPOperator>(I)->isInBounds());
}
}
llvm_unreachable("failed to rebuild vector instructions");
}
-static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask) {
+static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask,
+ IRBuilderBase &Builder) {
// Mask.size() does not need to be equal to the number of vector elements.
assert(V->getType()->isVectorTy() && "can't reorder non-vector elements");
Type *EltTy = V->getType()->getScalarType();
- Type *I32Ty = IntegerType::getInt32Ty(V->getContext());
if (match(V, m_Undef()))
return UndefValue::get(FixedVectorType::get(EltTy, Mask.size()));
@@ -1950,15 +1971,14 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask) {
// as well. E.g. GetElementPtr may have scalar operands even if the
// return value is a vector, so we need to examine the operand type.
if (I->getOperand(i)->getType()->isVectorTy())
- V = evaluateInDifferentElementOrder(I->getOperand(i), Mask);
+ V = evaluateInDifferentElementOrder(I->getOperand(i), Mask, Builder);
else
V = I->getOperand(i);
NewOps.push_back(V);
NeedsRebuild |= (V != I->getOperand(i));
}
- if (NeedsRebuild) {
- return buildNew(I, NewOps);
- }
+ if (NeedsRebuild)
+ return buildNew(I, NewOps, Builder);
return I;
}
case Instruction::InsertElement: {
@@ -1979,11 +1999,12 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask) {
// If element is not in Mask, no need to handle the operand 1 (element to
// be inserted). Just evaluate values in operand 0 according to Mask.
if (!Found)
- return evaluateInDifferentElementOrder(I->getOperand(0), Mask);
+ return evaluateInDifferentElementOrder(I->getOperand(0), Mask, Builder);
- Value *V = evaluateInDifferentElementOrder(I->getOperand(0), Mask);
- return InsertElementInst::Create(V, I->getOperand(1),
- ConstantInt::get(I32Ty, Index), "", I);
+ Value *V = evaluateInDifferentElementOrder(I->getOperand(0), Mask,
+ Builder);
+ Builder.SetInsertPoint(I);
+ return Builder.CreateInsertElement(V, I->getOperand(1), Index);
}
}
llvm_unreachable("failed to reorder elements of vector instruction!");
@@ -2140,7 +2161,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
ConstantExpr::getShuffleVector(IdC, C, Mask);
bool MightCreatePoisonOrUB =
- is_contained(Mask, UndefMaskElem) &&
+ is_contained(Mask, PoisonMaskElem) &&
(Instruction::isIntDivRem(BOpcode) || Instruction::isShift(BOpcode));
if (MightCreatePoisonOrUB)
NewC = InstCombiner::getSafeVectorConstantForBinop(BOpcode, NewC, true);
@@ -2154,7 +2175,7 @@ static Instruction *foldSelectShuffleWith1Binop(ShuffleVectorInst &Shuf) {
// An undef shuffle mask element may propagate as an undef constant element in
// the new binop. That would produce poison where the original code might not.
// If we already made a safe constant, then there's no danger.
- if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, PoisonMaskElem) && !MightCreatePoisonOrUB)
NewBO->dropPoisonGeneratingFlags();
return NewBO;
}
@@ -2178,8 +2199,7 @@ static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
// Insert into element 0 of an undef vector.
UndefValue *UndefVec = UndefValue::get(Shuf.getType());
- Constant *Zero = Builder.getInt32(0);
- Value *NewIns = Builder.CreateInsertElement(UndefVec, X, Zero);
+ Value *NewIns = Builder.CreateInsertElement(UndefVec, X, (uint64_t)0);
// Splat from element 0. Any mask element that is undefined remains undefined.
// For example:
@@ -2189,7 +2209,7 @@ static Instruction *canonicalizeInsertSplat(ShuffleVectorInst &Shuf,
cast<FixedVectorType>(Shuf.getType())->getNumElements();
SmallVector<int, 16> NewMask(NumMaskElts, 0);
for (unsigned i = 0; i != NumMaskElts; ++i)
- if (Mask[i] == UndefMaskElem)
+ if (Mask[i] == PoisonMaskElem)
NewMask[i] = Mask[i];
return new ShuffleVectorInst(NewIns, NewMask);
@@ -2274,7 +2294,7 @@ Instruction *InstCombinerImpl::foldSelectShuffle(ShuffleVectorInst &Shuf) {
// mask element, the result is undefined, but it is not poison or undefined
// behavior. That is not necessarily true for div/rem/shift.
bool MightCreatePoisonOrUB =
- is_contained(Mask, UndefMaskElem) &&
+ is_contained(Mask, PoisonMaskElem) &&
(Instruction::isIntDivRem(BOpc) || Instruction::isShift(BOpc));
if (MightCreatePoisonOrUB)
NewC = InstCombiner::getSafeVectorConstantForBinop(BOpc, NewC,
@@ -2325,7 +2345,7 @@ Instruction *InstCombinerImpl::foldSelectShuffle(ShuffleVectorInst &Shuf) {
NewI->andIRFlags(B1);
if (DropNSW)
NewI->setHasNoSignedWrap(false);
- if (is_contained(Mask, UndefMaskElem) && !MightCreatePoisonOrUB)
+ if (is_contained(Mask, PoisonMaskElem) && !MightCreatePoisonOrUB)
NewI->dropPoisonGeneratingFlags();
}
return replaceInstUsesWith(Shuf, NewBO);
@@ -2361,7 +2381,7 @@ static Instruction *foldTruncShuffle(ShuffleVectorInst &Shuf,
SrcType->getScalarSizeInBits() / DestType->getScalarSizeInBits();
ArrayRef<int> Mask = Shuf.getShuffleMask();
for (unsigned i = 0, e = Mask.size(); i != e; ++i) {
- if (Mask[i] == UndefMaskElem)
+ if (Mask[i] == PoisonMaskElem)
continue;
uint64_t LSBIndex = IsBigEndian ? (i + 1) * TruncRatio - 1 : i * TruncRatio;
assert(LSBIndex <= INT32_MAX && "Overflowed 32-bits");
@@ -2407,37 +2427,51 @@ static Instruction *narrowVectorSelect(ShuffleVectorInst &Shuf,
return SelectInst::Create(NarrowCond, NarrowX, NarrowY);
}
-/// Canonicalize FP negate after shuffle.
-static Instruction *foldFNegShuffle(ShuffleVectorInst &Shuf,
- InstCombiner::BuilderTy &Builder) {
- Instruction *FNeg0;
+/// Canonicalize FP negate/abs after shuffle.
+static Instruction *foldShuffleOfUnaryOps(ShuffleVectorInst &Shuf,
+ InstCombiner::BuilderTy &Builder) {
+ auto *S0 = dyn_cast<Instruction>(Shuf.getOperand(0));
Value *X;
- if (!match(Shuf.getOperand(0), m_CombineAnd(m_Instruction(FNeg0),
- m_FNeg(m_Value(X)))))
+ if (!S0 || !match(S0, m_CombineOr(m_FNeg(m_Value(X)), m_FAbs(m_Value(X)))))
return nullptr;
- // shuffle (fneg X), Mask --> fneg (shuffle X, Mask)
- if (FNeg0->hasOneUse() && match(Shuf.getOperand(1), m_Undef())) {
+ bool IsFNeg = S0->getOpcode() == Instruction::FNeg;
+
+ // Match 1-input (unary) shuffle.
+ // shuffle (fneg/fabs X), Mask --> fneg/fabs (shuffle X, Mask)
+ if (S0->hasOneUse() && match(Shuf.getOperand(1), m_Undef())) {
Value *NewShuf = Builder.CreateShuffleVector(X, Shuf.getShuffleMask());
- return UnaryOperator::CreateFNegFMF(NewShuf, FNeg0);
+ if (IsFNeg)
+ return UnaryOperator::CreateFNegFMF(NewShuf, S0);
+
+ Function *FAbs = Intrinsic::getDeclaration(Shuf.getModule(),
+ Intrinsic::fabs, Shuf.getType());
+ CallInst *NewF = CallInst::Create(FAbs, {NewShuf});
+ NewF->setFastMathFlags(S0->getFastMathFlags());
+ return NewF;
}
- Instruction *FNeg1;
+ // Match 2-input (binary) shuffle.
+ auto *S1 = dyn_cast<Instruction>(Shuf.getOperand(1));
Value *Y;
- if (!match(Shuf.getOperand(1), m_CombineAnd(m_Instruction(FNeg1),
- m_FNeg(m_Value(Y)))))
+ if (!S1 || !match(S1, m_CombineOr(m_FNeg(m_Value(Y)), m_FAbs(m_Value(Y)))) ||
+ S0->getOpcode() != S1->getOpcode() ||
+ (!S0->hasOneUse() && !S1->hasOneUse()))
return nullptr;
- // shuffle (fneg X), (fneg Y), Mask --> fneg (shuffle X, Y, Mask)
- if (FNeg0->hasOneUse() || FNeg1->hasOneUse()) {
- Value *NewShuf = Builder.CreateShuffleVector(X, Y, Shuf.getShuffleMask());
- Instruction *NewFNeg = UnaryOperator::CreateFNeg(NewShuf);
- NewFNeg->copyIRFlags(FNeg0);
- NewFNeg->andIRFlags(FNeg1);
- return NewFNeg;
+ // shuf (fneg/fabs X), (fneg/fabs Y), Mask --> fneg/fabs (shuf X, Y, Mask)
+ Value *NewShuf = Builder.CreateShuffleVector(X, Y, Shuf.getShuffleMask());
+ Instruction *NewF;
+ if (IsFNeg) {
+ NewF = UnaryOperator::CreateFNeg(NewShuf);
+ } else {
+ Function *FAbs = Intrinsic::getDeclaration(Shuf.getModule(),
+ Intrinsic::fabs, Shuf.getType());
+ NewF = CallInst::Create(FAbs, {NewShuf});
}
-
- return nullptr;
+ NewF->copyIRFlags(S0);
+ NewF->andIRFlags(S1);
+ return NewF;
}
/// Canonicalize casts after shuffle.
@@ -2533,7 +2567,7 @@ static Instruction *foldIdentityExtractShuffle(ShuffleVectorInst &Shuf) {
for (unsigned i = 0; i != NumElts; ++i) {
int ExtractMaskElt = Shuf.getMaskValue(i);
int MaskElt = Mask[i];
- NewMask[i] = ExtractMaskElt == UndefMaskElem ? ExtractMaskElt : MaskElt;
+ NewMask[i] = ExtractMaskElt == PoisonMaskElem ? ExtractMaskElt : MaskElt;
}
return new ShuffleVectorInst(X, Y, NewMask);
}
@@ -2699,7 +2733,8 @@ static Instruction *foldIdentityPaddedShuffles(ShuffleVectorInst &Shuf) {
// splatting the first element of the result of the BinOp
Instruction *InstCombinerImpl::simplifyBinOpSplats(ShuffleVectorInst &SVI) {
if (!match(SVI.getOperand(1), m_Undef()) ||
- !match(SVI.getShuffleMask(), m_ZeroMask()))
+ !match(SVI.getShuffleMask(), m_ZeroMask()) ||
+ !SVI.getOperand(0)->hasOneUse())
return nullptr;
Value *Op0 = SVI.getOperand(0);
@@ -2759,7 +2794,6 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
}
ArrayRef<int> Mask = SVI.getShuffleMask();
- Type *Int32Ty = Type::getInt32Ty(SVI.getContext());
// Peek through a bitcasted shuffle operand by scaling the mask. If the
// simulated shuffle can simplify, then this shuffle is unnecessary:
@@ -2815,7 +2849,7 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
if (Instruction *I = narrowVectorSelect(SVI, Builder))
return I;
- if (Instruction *I = foldFNegShuffle(SVI, Builder))
+ if (Instruction *I = foldShuffleOfUnaryOps(SVI, Builder))
return I;
if (Instruction *I = foldCastShuffle(SVI, Builder))
@@ -2840,7 +2874,7 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
return I;
if (match(RHS, m_Undef()) && canEvaluateShuffled(LHS, Mask)) {
- Value *V = evaluateInDifferentElementOrder(LHS, Mask);
+ Value *V = evaluateInDifferentElementOrder(LHS, Mask, Builder);
return replaceInstUsesWith(SVI, V);
}
@@ -2916,15 +2950,15 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
unsigned SrcElemsPerTgtElem = TgtElemBitWidth / SrcElemBitWidth;
assert(SrcElemsPerTgtElem);
BegIdx /= SrcElemsPerTgtElem;
- bool BCAlreadyExists = NewBCs.find(CastSrcTy) != NewBCs.end();
+ bool BCAlreadyExists = NewBCs.contains(CastSrcTy);
auto *NewBC =
BCAlreadyExists
? NewBCs[CastSrcTy]
: Builder.CreateBitCast(V, CastSrcTy, SVI.getName() + ".bc");
if (!BCAlreadyExists)
NewBCs[CastSrcTy] = NewBC;
- auto *Ext = Builder.CreateExtractElement(
- NewBC, ConstantInt::get(Int32Ty, BegIdx), SVI.getName() + ".extract");
+ auto *Ext = Builder.CreateExtractElement(NewBC, BegIdx,
+ SVI.getName() + ".extract");
// The shufflevector isn't being replaced: the bitcast that used it
// is. InstCombine will visit the newly-created instructions.
replaceInstUsesWith(*BC, Ext);
@@ -3042,7 +3076,7 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
for (unsigned i = 0; i < VWidth; ++i) {
int eltMask;
if (Mask[i] < 0) {
- // This element is an undef value.
+ // This element is a poison value.
eltMask = -1;
} else if (Mask[i] < (int)LHSWidth) {
// This element is from left hand side vector operand.
@@ -3051,27 +3085,27 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
// new mask value for the element.
if (newLHS != LHS) {
eltMask = LHSMask[Mask[i]];
- // If the value selected is an undef value, explicitly specify it
+ // If the value selected is an poison value, explicitly specify it
// with a -1 mask value.
- if (eltMask >= (int)LHSOp0Width && isa<UndefValue>(LHSOp1))
+ if (eltMask >= (int)LHSOp0Width && isa<PoisonValue>(LHSOp1))
eltMask = -1;
} else
eltMask = Mask[i];
} else {
// This element is from right hand side vector operand
//
- // If the value selected is an undef value, explicitly specify it
+ // If the value selected is a poison value, explicitly specify it
// with a -1 mask value. (case 1)
- if (match(RHS, m_Undef()))
+ if (match(RHS, m_Poison()))
eltMask = -1;
// If RHS is going to be replaced (case 3 or 4), calculate the
// new mask value for the element.
else if (newRHS != RHS) {
eltMask = RHSMask[Mask[i]-LHSWidth];
- // If the value selected is an undef value, explicitly specify it
+ // If the value selected is an poison value, explicitly specify it
// with a -1 mask value.
if (eltMask >= (int)RHSOp0Width) {
- assert(match(RHSShuffle->getOperand(1), m_Undef()) &&
+ assert(match(RHSShuffle->getOperand(1), m_Poison()) &&
"should have been check above");
eltMask = -1;
}
@@ -3102,7 +3136,7 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
// or is a splat, do the replacement.
if (isSplat || newMask == LHSMask || newMask == RHSMask || newMask == Mask) {
if (!newRHS)
- newRHS = UndefValue::get(newLHS->getType());
+ newRHS = PoisonValue::get(newLHS->getType());
return new ShuffleVectorInst(newLHS, newRHS, newMask);
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-19 05:49:30 +0000