diff options
Diffstat (limited to 'llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp')
| -rw-r--r-- | llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp | 538 |
1 files changed, 433 insertions, 105 deletions
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 726bb545be12..bfa7bfa2290a 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -67,7 +67,6 @@ #include "llvm/Support/KnownBits.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/InstCombine/InstCombineWorklist.h" #include "llvm/Transforms/InstCombine/InstCombiner.h" #include "llvm/Transforms/Utils/AssumeBundleBuilder.h" #include "llvm/Transforms/Utils/Local.h" @@ -79,11 +78,12 @@ #include <utility> #include <vector> +#define DEBUG_TYPE "instcombine" +#include "llvm/Transforms/Utils/InstructionWorklist.h" + using namespace llvm; using namespace PatternMatch; -#define DEBUG_TYPE "instcombine" - STATISTIC(NumSimplified, "Number of library calls simplified"); static cl::opt<unsigned> GuardWideningWindow( @@ -513,7 +513,7 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) { // If the input to cttz/ctlz is known to be non-zero, // then change the 'ZeroIsUndef' parameter to 'true' // because we know the zero behavior can't affect the result. - if (!Known.One.isNullValue() || + if (!Known.One.isZero() || isKnownNonZero(Op0, IC.getDataLayout(), 0, &IC.getAssumptionCache(), &II, &IC.getDominatorTree())) { if (!match(II.getArgOperand(1), m_One())) @@ -656,8 +656,8 @@ static Value *simplifyNeonTbl1(const IntrinsicInst &II, // comparison to the first NumOperands. static bool haveSameOperands(const IntrinsicInst &I, const IntrinsicInst &E, unsigned NumOperands) { - assert(I.getNumArgOperands() >= NumOperands && "Not enough operands"); - assert(E.getNumArgOperands() >= NumOperands && "Not enough operands"); + assert(I.arg_size() >= NumOperands && "Not enough operands"); + assert(E.arg_size() >= NumOperands && "Not enough operands"); for (unsigned i = 0; i < NumOperands; i++) if (I.getArgOperand(i) != E.getArgOperand(i)) return false; @@ -682,11 +682,11 @@ removeTriviallyEmptyRange(IntrinsicInst &EndI, InstCombinerImpl &IC, BasicBlock::reverse_iterator BI(EndI), BE(EndI.getParent()->rend()); for (; BI != BE; ++BI) { if (auto *I = dyn_cast<IntrinsicInst>(&*BI)) { - if (isa<DbgInfoIntrinsic>(I) || + if (I->isDebugOrPseudoInst() || I->getIntrinsicID() == EndI.getIntrinsicID()) continue; if (IsStart(*I)) { - if (haveSameOperands(EndI, *I, EndI.getNumArgOperands())) { + if (haveSameOperands(EndI, *I, EndI.arg_size())) { IC.eraseInstFromFunction(*I); IC.eraseInstFromFunction(EndI); return true; @@ -710,7 +710,7 @@ Instruction *InstCombinerImpl::visitVAEndInst(VAEndInst &I) { } static CallInst *canonicalizeConstantArg0ToArg1(CallInst &Call) { - assert(Call.getNumArgOperands() > 1 && "Need at least 2 args to swap"); + assert(Call.arg_size() > 1 && "Need at least 2 args to swap"); Value *Arg0 = Call.getArgOperand(0), *Arg1 = Call.getArgOperand(1); if (isa<Constant>(Arg0) && !isa<Constant>(Arg1)) { Call.setArgOperand(0, Arg1); @@ -754,6 +754,45 @@ static Optional<bool> getKnownSign(Value *Op, Instruction *CxtI, ICmpInst::ICMP_SLT, Op, Constant::getNullValue(Op->getType()), CxtI, DL); } +/// Try to canonicalize min/max(X + C0, C1) as min/max(X, C1 - C0) + C0. This +/// can trigger other combines. +static Instruction *moveAddAfterMinMax(IntrinsicInst *II, + InstCombiner::BuilderTy &Builder) { + Intrinsic::ID MinMaxID = II->getIntrinsicID(); + assert((MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin || + MinMaxID == Intrinsic::umax || MinMaxID == Intrinsic::umin) && + "Expected a min or max intrinsic"); + + // TODO: Match vectors with undef elements, but undef may not propagate. + Value *Op0 = II->getArgOperand(0), *Op1 = II->getArgOperand(1); + Value *X; + const APInt *C0, *C1; + if (!match(Op0, m_OneUse(m_Add(m_Value(X), m_APInt(C0)))) || + !match(Op1, m_APInt(C1))) + return nullptr; + + // Check for necessary no-wrap and overflow constraints. + bool IsSigned = MinMaxID == Intrinsic::smax || MinMaxID == Intrinsic::smin; + auto *Add = cast<BinaryOperator>(Op0); + if ((IsSigned && !Add->hasNoSignedWrap()) || + (!IsSigned && !Add->hasNoUnsignedWrap())) + return nullptr; + + // If the constant difference overflows, then instsimplify should reduce the + // min/max to the add or C1. + bool Overflow; + APInt CDiff = + IsSigned ? C1->ssub_ov(*C0, Overflow) : C1->usub_ov(*C0, Overflow); + assert(!Overflow && "Expected simplify of min/max"); + + // min/max (add X, C0), C1 --> add (min/max X, C1 - C0), C0 + // Note: the "mismatched" no-overflow setting does not propagate. + Constant *NewMinMaxC = ConstantInt::get(II->getType(), CDiff); + Value *NewMinMax = Builder.CreateBinaryIntrinsic(MinMaxID, X, NewMinMaxC); + return IsSigned ? BinaryOperator::CreateNSWAdd(NewMinMax, Add->getOperand(1)) + : BinaryOperator::CreateNUWAdd(NewMinMax, Add->getOperand(1)); +} + /// If we have a clamp pattern like max (min X, 42), 41 -- where the output /// can only be one of two possible constant values -- turn that into a select /// of constants. @@ -795,6 +834,63 @@ static Instruction *foldClampRangeOfTwo(IntrinsicInst *II, return SelectInst::Create(Cmp, ConstantInt::get(II->getType(), *C0), I1); } +/// Reduce a sequence of min/max intrinsics with a common operand. +static Instruction *factorizeMinMaxTree(IntrinsicInst *II) { + // Match 3 of the same min/max ops. Example: umin(umin(), umin()). + auto *LHS = dyn_cast<IntrinsicInst>(II->getArgOperand(0)); + auto *RHS = dyn_cast<IntrinsicInst>(II->getArgOperand(1)); + Intrinsic::ID MinMaxID = II->getIntrinsicID(); + if (!LHS || !RHS || LHS->getIntrinsicID() != MinMaxID || + RHS->getIntrinsicID() != MinMaxID || + (!LHS->hasOneUse() && !RHS->hasOneUse())) + return nullptr; + + Value *A = LHS->getArgOperand(0); + Value *B = LHS->getArgOperand(1); + Value *C = RHS->getArgOperand(0); + Value *D = RHS->getArgOperand(1); + + // Look for a common operand. + Value *MinMaxOp = nullptr; + Value *ThirdOp = nullptr; + if (LHS->hasOneUse()) { + // If the LHS is only used in this chain and the RHS is used outside of it, + // reuse the RHS min/max because that will eliminate the LHS. + if (D == A || C == A) { + // min(min(a, b), min(c, a)) --> min(min(c, a), b) + // min(min(a, b), min(a, d)) --> min(min(a, d), b) + MinMaxOp = RHS; + ThirdOp = B; + } else if (D == B || C == B) { + // min(min(a, b), min(c, b)) --> min(min(c, b), a) + // min(min(a, b), min(b, d)) --> min(min(b, d), a) + MinMaxOp = RHS; + ThirdOp = A; + } + } else { + assert(RHS->hasOneUse() && "Expected one-use operand"); + // Reuse the LHS. This will eliminate the RHS. + if (D == A || D == B) { + // min(min(a, b), min(c, a)) --> min(min(a, b), c) + // min(min(a, b), min(c, b)) --> min(min(a, b), c) + MinMaxOp = LHS; + ThirdOp = C; + } else if (C == A || C == B) { + // min(min(a, b), min(b, d)) --> min(min(a, b), d) + // min(min(a, b), min(c, b)) --> min(min(a, b), d) + MinMaxOp = LHS; + ThirdOp = D; + } + } + + if (!MinMaxOp || !ThirdOp) + return nullptr; + + Module *Mod = II->getModule(); + Function *MinMax = Intrinsic::getDeclaration(Mod, MinMaxID, II->getType()); + return CallInst::Create(MinMax, { MinMaxOp, ThirdOp }); +} + /// CallInst simplification. This mostly only handles folding of intrinsic /// instructions. For normal calls, it allows visitCallBase to do the heavy /// lifting. @@ -896,7 +992,7 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (auto *IIFVTy = dyn_cast<FixedVectorType>(II->getType())) { auto VWidth = IIFVTy->getNumElements(); APInt UndefElts(VWidth, 0); - APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth)); + APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, UndefElts)) { if (V != II) return replaceInstUsesWith(*II, V); @@ -1007,21 +1103,45 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { } } - if (match(I0, m_Not(m_Value(X)))) { - // max (not X), (not Y) --> not (min X, Y) - Intrinsic::ID InvID = getInverseMinMaxIntrinsic(IID); - if (match(I1, m_Not(m_Value(Y))) && + if (IID == Intrinsic::smax || IID == Intrinsic::smin) { + // smax (neg nsw X), (neg nsw Y) --> neg nsw (smin X, Y) + // smin (neg nsw X), (neg nsw Y) --> neg nsw (smax X, Y) + // TODO: Canonicalize neg after min/max if I1 is constant. + if (match(I0, m_NSWNeg(m_Value(X))) && match(I1, m_NSWNeg(m_Value(Y))) && (I0->hasOneUse() || I1->hasOneUse())) { + Intrinsic::ID InvID = getInverseMinMaxIntrinsic(IID); Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, X, Y); - return BinaryOperator::CreateNot(InvMaxMin); + return BinaryOperator::CreateNSWNeg(InvMaxMin); } - // max (not X), C --> not(min X, ~C) - if (match(I1, m_Constant(C)) && I0->hasOneUse()) { - Constant *NotC = ConstantExpr::getNot(C); - Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, X, NotC); + } + + // If we can eliminate ~A and Y is free to invert: + // max ~A, Y --> ~(min A, ~Y) + // + // Examples: + // max ~A, ~Y --> ~(min A, Y) + // max ~A, C --> ~(min A, ~C) + // max ~A, (max ~Y, ~Z) --> ~min( A, (min Y, Z)) + auto moveNotAfterMinMax = [&](Value *X, Value *Y) -> Instruction * { + Value *A; + if (match(X, m_OneUse(m_Not(m_Value(A)))) && + !isFreeToInvert(A, A->hasOneUse()) && + isFreeToInvert(Y, Y->hasOneUse())) { + Value *NotY = Builder.CreateNot(Y); + Intrinsic::ID InvID = getInverseMinMaxIntrinsic(IID); + Value *InvMaxMin = Builder.CreateBinaryIntrinsic(InvID, A, NotY); return BinaryOperator::CreateNot(InvMaxMin); } - } + return nullptr; + }; + + if (Instruction *I = moveNotAfterMinMax(I0, I1)) + return I; + if (Instruction *I = moveNotAfterMinMax(I1, I0)) + return I; + + if (Instruction *I = moveAddAfterMinMax(II, Builder)) + return I; // smax(X, -X) --> abs(X) // smin(X, -X) --> -abs(X) @@ -1051,11 +1171,17 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (Instruction *Sel = foldClampRangeOfTwo(II, Builder)) return Sel; + if (Instruction *SAdd = matchSAddSubSat(*II)) + return SAdd; + if (match(I1, m_ImmConstant())) if (auto *Sel = dyn_cast<SelectInst>(I0)) if (Instruction *R = FoldOpIntoSelect(*II, Sel)) return R; + if (Instruction *NewMinMax = factorizeMinMaxTree(II)) + return NewMinMax; + break; } case Intrinsic::bswap: { @@ -1098,6 +1224,19 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (Power->equalsInt(2)) return BinaryOperator::CreateFMulFMF(II->getArgOperand(0), II->getArgOperand(0), II); + + if (!Power->getValue()[0]) { + Value *X; + // If power is even: + // powi(-x, p) -> powi(x, p) + // powi(fabs(x), p) -> powi(x, p) + // powi(copysign(x, y), p) -> powi(x, p) + if (match(II->getArgOperand(0), m_FNeg(m_Value(X))) || + match(II->getArgOperand(0), m_FAbs(m_Value(X))) || + match(II->getArgOperand(0), + m_Intrinsic<Intrinsic::copysign>(m_Value(X), m_Value()))) + return replaceOperand(*II, 0, X); + } } break; @@ -1637,14 +1776,66 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { break; } case Intrinsic::stackrestore: { - // If the save is right next to the restore, remove the restore. This can - // happen when variable allocas are DCE'd. + enum class ClassifyResult { + None, + Alloca, + StackRestore, + CallWithSideEffects, + }; + auto Classify = [](const Instruction *I) { + if (isa<AllocaInst>(I)) + return ClassifyResult::Alloca; + + if (auto *CI = dyn_cast<CallInst>(I)) { + if (auto *II = dyn_cast<IntrinsicInst>(CI)) { + if (II->getIntrinsicID() == Intrinsic::stackrestore) + return ClassifyResult::StackRestore; + + if (II->mayHaveSideEffects()) + return ClassifyResult::CallWithSideEffects; + } else { + // Consider all non-intrinsic calls to be side effects + return ClassifyResult::CallWithSideEffects; + } + } + + return ClassifyResult::None; + }; + + // If the stacksave and the stackrestore are in the same BB, and there is + // no intervening call, alloca, or stackrestore of a different stacksave, + // remove the restore. This can happen when variable allocas are DCE'd. if (IntrinsicInst *SS = dyn_cast<IntrinsicInst>(II->getArgOperand(0))) { - if (SS->getIntrinsicID() == Intrinsic::stacksave) { - // Skip over debug info. - if (SS->getNextNonDebugInstruction() == II) { - return eraseInstFromFunction(CI); + if (SS->getIntrinsicID() == Intrinsic::stacksave && + SS->getParent() == II->getParent()) { + BasicBlock::iterator BI(SS); + bool CannotRemove = false; + for (++BI; &*BI != II; ++BI) { + switch (Classify(&*BI)) { + case ClassifyResult::None: + // So far so good, look at next instructions. + break; + + case ClassifyResult::StackRestore: + // If we found an intervening stackrestore for a different + // stacksave, we can't remove the stackrestore. Otherwise, continue. + if (cast<IntrinsicInst>(*BI).getArgOperand(0) != SS) + CannotRemove = true; + break; + + case ClassifyResult::Alloca: + case ClassifyResult::CallWithSideEffects: + // If we found an alloca, a non-intrinsic call, or an intrinsic + // call with side effects, we can't remove the stackrestore. + CannotRemove = true; + break; + } + if (CannotRemove) + break; } + + if (!CannotRemove) + return eraseInstFromFunction(CI); } } @@ -1654,29 +1845,25 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { Instruction *TI = II->getParent()->getTerminator(); bool CannotRemove = false; for (++BI; &*BI != TI; ++BI) { - if (isa<AllocaInst>(BI)) { - CannotRemove = true; + switch (Classify(&*BI)) { + case ClassifyResult::None: + // So far so good, look at next instructions. break; - } - if (CallInst *BCI = dyn_cast<CallInst>(BI)) { - if (auto *II2 = dyn_cast<IntrinsicInst>(BCI)) { - // If there is a stackrestore below this one, remove this one. - if (II2->getIntrinsicID() == Intrinsic::stackrestore) - return eraseInstFromFunction(CI); - // Bail if we cross over an intrinsic with side effects, such as - // llvm.stacksave, or llvm.read_register. - if (II2->mayHaveSideEffects()) { - CannotRemove = true; - break; - } - } else { - // If we found a non-intrinsic call, we can't remove the stack - // restore. - CannotRemove = true; - break; - } + case ClassifyResult::StackRestore: + // If there is a stackrestore below this one, remove this one. + return eraseInstFromFunction(CI); + + case ClassifyResult::Alloca: + case ClassifyResult::CallWithSideEffects: + // If we found an alloca, a non-intrinsic call, or an intrinsic call + // with side effects (such as llvm.stacksave and llvm.read_register), + // we can't remove the stack restore. + CannotRemove = true; + break; } + if (CannotRemove) + break; } // If the stack restore is in a return, resume, or unwind block and if there @@ -1963,6 +2150,46 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { } break; } + case Intrinsic::experimental_vector_reverse: { + Value *BO0, *BO1, *X, *Y; + Value *Vec = II->getArgOperand(0); + if (match(Vec, m_OneUse(m_BinOp(m_Value(BO0), m_Value(BO1))))) { + auto *OldBinOp = cast<BinaryOperator>(Vec); + if (match(BO0, m_Intrinsic<Intrinsic::experimental_vector_reverse>( + m_Value(X)))) { + // rev(binop rev(X), rev(Y)) --> binop X, Y + if (match(BO1, m_Intrinsic<Intrinsic::experimental_vector_reverse>( + m_Value(Y)))) + return replaceInstUsesWith(CI, + BinaryOperator::CreateWithCopiedFlags( + OldBinOp->getOpcode(), X, Y, OldBinOp, + OldBinOp->getName(), II)); + // rev(binop rev(X), BO1Splat) --> binop X, BO1Splat + if (isSplatValue(BO1)) + return replaceInstUsesWith(CI, + BinaryOperator::CreateWithCopiedFlags( + OldBinOp->getOpcode(), X, BO1, + OldBinOp, OldBinOp->getName(), II)); + } + // rev(binop BO0Splat, rev(Y)) --> binop BO0Splat, Y + if (match(BO1, m_Intrinsic<Intrinsic::experimental_vector_reverse>( + m_Value(Y))) && + isSplatValue(BO0)) + return replaceInstUsesWith(CI, BinaryOperator::CreateWithCopiedFlags( + OldBinOp->getOpcode(), BO0, Y, + OldBinOp, OldBinOp->getName(), II)); + } + // rev(unop rev(X)) --> unop X + if (match(Vec, m_OneUse(m_UnOp( + m_Intrinsic<Intrinsic::experimental_vector_reverse>( + m_Value(X)))))) { + auto *OldUnOp = cast<UnaryOperator>(Vec); + auto *NewUnOp = UnaryOperator::CreateWithCopiedFlags( + OldUnOp->getOpcode(), X, OldUnOp, OldUnOp->getName(), II); + return replaceInstUsesWith(CI, NewUnOp); + } + break; + } case Intrinsic::vector_reduce_or: case Intrinsic::vector_reduce_and: { // Canonicalize logical or/and reductions: @@ -1973,21 +2200,26 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { // %val = bitcast <ReduxWidth x i1> to iReduxWidth // %res = cmp eq iReduxWidth %val, 11111 Value *Arg = II->getArgOperand(0); - Type *RetTy = II->getType(); - if (RetTy == Builder.getInt1Ty()) - if (auto *FVTy = dyn_cast<FixedVectorType>(Arg->getType())) { - Value *Res = Builder.CreateBitCast( - Arg, Builder.getIntNTy(FVTy->getNumElements())); - if (IID == Intrinsic::vector_reduce_and) { - Res = Builder.CreateICmpEQ( - Res, ConstantInt::getAllOnesValue(Res->getType())); - } else { - assert(IID == Intrinsic::vector_reduce_or && - "Expected or reduction."); - Res = Builder.CreateIsNotNull(Res); + Value *Vect; + if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) { + if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType())) + if (FTy->getElementType() == Builder.getInt1Ty()) { + Value *Res = Builder.CreateBitCast( + Vect, Builder.getIntNTy(FTy->getNumElements())); + if (IID == Intrinsic::vector_reduce_and) { + Res = Builder.CreateICmpEQ( + Res, ConstantInt::getAllOnesValue(Res->getType())); + } else { + assert(IID == Intrinsic::vector_reduce_or && + "Expected or reduction."); + Res = Builder.CreateIsNotNull(Res); + } + if (Arg != Vect) + Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res, + II->getType()); + return replaceInstUsesWith(CI, Res); } - return replaceInstUsesWith(CI, Res); - } + } LLVM_FALLTHROUGH; } case Intrinsic::vector_reduce_add: { @@ -2017,12 +2249,117 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { } LLVM_FALLTHROUGH; } - case Intrinsic::vector_reduce_mul: - case Intrinsic::vector_reduce_xor: - case Intrinsic::vector_reduce_umax: + case Intrinsic::vector_reduce_xor: { + if (IID == Intrinsic::vector_reduce_xor) { + // Exclusive disjunction reduction over the vector with + // (potentially-extended) i1 element type is actually a + // (potentially-extended) arithmetic `add` reduction over the original + // non-extended value: + // vector_reduce_xor(?ext(<n x i1>)) + // --> + // ?ext(vector_reduce_add(<n x i1>)) + Value *Arg = II->getArgOperand(0); + Value *Vect; + if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) { + if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType())) + if (FTy->getElementType() == Builder.getInt1Ty()) { + Value *Res = Builder.CreateAddReduce(Vect); + if (Arg != Vect) + Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res, + II->getType()); + return replaceInstUsesWith(CI, Res); + } + } + } + LLVM_FALLTHROUGH; + } + case Intrinsic::vector_reduce_mul: { + if (IID == Intrinsic::vector_reduce_mul) { + // Multiplicative reduction over the vector with (potentially-extended) + // i1 element type is actually a (potentially zero-extended) + // logical `and` reduction over the original non-extended value: + // vector_reduce_mul(?ext(<n x i1>)) + // --> + // zext(vector_reduce_and(<n x i1>)) + Value *Arg = II->getArgOperand(0); + Value *Vect; + if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) { + if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType())) + if (FTy->getElementType() == Builder.getInt1Ty()) { + Value *Res = Builder.CreateAndReduce(Vect); + if (Res->getType() != II->getType()) + Res = Builder.CreateZExt(Res, II->getType()); + return replaceInstUsesWith(CI, Res); + } + } + } + LLVM_FALLTHROUGH; + } case Intrinsic::vector_reduce_umin: - case Intrinsic::vector_reduce_smax: + case Intrinsic::vector_reduce_umax: { + if (IID == Intrinsic::vector_reduce_umin || + IID == Intrinsic::vector_reduce_umax) { + // UMin/UMax reduction over the vector with (potentially-extended) + // i1 element type is actually a (potentially-extended) + // logical `and`/`or` reduction over the original non-extended value: + // vector_reduce_u{min,max}(?ext(<n x i1>)) + // --> + // ?ext(vector_reduce_{and,or}(<n x i1>)) + Value *Arg = II->getArgOperand(0); + Value *Vect; + if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) { + if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType())) + if (FTy->getElementType() == Builder.getInt1Ty()) { + Value *Res = IID == Intrinsic::vector_reduce_umin + ? Builder.CreateAndReduce(Vect) + : Builder.CreateOrReduce(Vect); + if (Arg != Vect) + Res = Builder.CreateCast(cast<CastInst>(Arg)->getOpcode(), Res, + II->getType()); + return replaceInstUsesWith(CI, Res); + } + } + } + LLVM_FALLTHROUGH; + } case Intrinsic::vector_reduce_smin: + case Intrinsic::vector_reduce_smax: { + if (IID == Intrinsic::vector_reduce_smin || + IID == Intrinsic::vector_reduce_smax) { + // SMin/SMax reduction over the vector with (potentially-extended) + // i1 element type is actually a (potentially-extended) + // logical `and`/`or` reduction over the original non-extended value: + // vector_reduce_s{min,max}(<n x i1>) + // --> + // vector_reduce_{or,and}(<n x i1>) + // and + // vector_reduce_s{min,max}(sext(<n x i1>)) + // --> + // sext(vector_reduce_{or,and}(<n x i1>)) + // and + // vector_reduce_s{min,max}(zext(<n x i1>)) + // --> + // zext(vector_reduce_{and,or}(<n x i1>)) + Value *Arg = II->getArgOperand(0); + Value *Vect; + if (match(Arg, m_ZExtOrSExtOrSelf(m_Value(Vect)))) { + if (auto *FTy = dyn_cast<FixedVectorType>(Vect->getType())) + if (FTy->getElementType() == Builder.getInt1Ty()) { + Instruction::CastOps ExtOpc = Instruction::CastOps::CastOpsEnd; + if (Arg != Vect) + ExtOpc = cast<CastInst>(Arg)->getOpcode(); + Value *Res = ((IID == Intrinsic::vector_reduce_smin) == + (ExtOpc == Instruction::CastOps::ZExt)) + ? Builder.CreateAndReduce(Vect) + : Builder.CreateOrReduce(Vect); + if (Arg != Vect) + Res = Builder.CreateCast(ExtOpc, Res, II->getType()); + return replaceInstUsesWith(CI, Res); + } + } + } + LLVM_FALLTHROUGH; + } case Intrinsic::vector_reduce_fmax: case Intrinsic::vector_reduce_fmin: case Intrinsic::vector_reduce_fadd: @@ -2228,7 +2565,7 @@ static IntrinsicInst *findInitTrampoline(Value *Callee) { } void InstCombinerImpl::annotateAnyAllocSite(CallBase &Call, const TargetLibraryInfo *TLI) { - unsigned NumArgs = Call.getNumArgOperands(); + unsigned NumArgs = Call.arg_size(); ConstantInt *Op0C = dyn_cast<ConstantInt>(Call.getOperand(0)); ConstantInt *Op1C = (NumArgs == 1) ? nullptr : dyn_cast<ConstantInt>(Call.getOperand(1)); @@ -2239,55 +2576,46 @@ void InstCombinerImpl::annotateAnyAllocSite(CallBase &Call, const TargetLibraryI if (isMallocLikeFn(&Call, TLI) && Op0C) { if (isOpNewLikeFn(&Call, TLI)) - Call.addAttribute(AttributeList::ReturnIndex, - Attribute::getWithDereferenceableBytes( - Call.getContext(), Op0C->getZExtValue())); + Call.addRetAttr(Attribute::getWithDereferenceableBytes( + Call.getContext(), Op0C->getZExtValue())); else - Call.addAttribute(AttributeList::ReturnIndex, - Attribute::getWithDereferenceableOrNullBytes( - Call.getContext(), Op0C->getZExtValue())); + Call.addRetAttr(Attribute::getWithDereferenceableOrNullBytes( + Call.getContext(), Op0C->getZExtValue())); } else if (isAlignedAllocLikeFn(&Call, TLI)) { if (Op1C) - Call.addAttribute(AttributeList::ReturnIndex, - Attribute::getWithDereferenceableOrNullBytes( - Call.getContext(), Op1C->getZExtValue())); + Call.addRetAttr(Attribute::getWithDereferenceableOrNullBytes( + Call.getContext(), Op1C->getZExtValue())); // Add alignment attribute if alignment is a power of two constant. if (Op0C && Op0C->getValue().ult(llvm::Value::MaximumAlignment) && isKnownNonZero(Call.getOperand(1), DL, 0, &AC, &Call, &DT)) { uint64_t AlignmentVal = Op0C->getZExtValue(); if (llvm::isPowerOf2_64(AlignmentVal)) { - Call.removeAttribute(AttributeList::ReturnIndex, Attribute::Alignment); - Call.addAttribute(AttributeList::ReturnIndex, - Attribute::getWithAlignment(Call.getContext(), - Align(AlignmentVal))); + Call.removeRetAttr(Attribute::Alignment); + Call.addRetAttr(Attribute::getWithAlignment(Call.getContext(), + Align(AlignmentVal))); } } } else if (isReallocLikeFn(&Call, TLI) && Op1C) { - Call.addAttribute(AttributeList::ReturnIndex, - Attribute::getWithDereferenceableOrNullBytes( - Call.getContext(), Op1C->getZExtValue())); + Call.addRetAttr(Attribute::getWithDereferenceableOrNullBytes( + Call.getContext(), Op1C->getZExtValue())); } else if (isCallocLikeFn(&Call, TLI) && Op0C && Op1C) { bool Overflow; const APInt &N = Op0C->getValue(); APInt Size = N.umul_ov(Op1C->getValue(), Overflow); if (!Overflow) - Call.addAttribute(AttributeList::ReturnIndex, - Attribute::getWithDereferenceableOrNullBytes( - Call.getContext(), Size.getZExtValue())); + Call.addRetAttr(Attribute::getWithDereferenceableOrNullBytes( + Call.getContext(), Size.getZExtValue())); } else if (isStrdupLikeFn(&Call, TLI)) { uint64_t Len = GetStringLength(Call.getOperand(0)); if (Len) { // strdup if (NumArgs == 1) - Call.addAttribute(AttributeList::ReturnIndex, - Attribute::getWithDereferenceableOrNullBytes( - Call.getContext(), Len)); + Call.addRetAttr(Attribute::getWithDereferenceableOrNullBytes( + Call.getContext(), Len)); // strndup else if (NumArgs == 2 && Op1C) - Call.addAttribute( - AttributeList::ReturnIndex, - Attribute::getWithDereferenceableOrNullBytes( - Call.getContext(), std::min(Len, Op1C->getZExtValue() + 1))); + Call.addRetAttr(Attribute::getWithDereferenceableOrNullBytes( + Call.getContext(), std::min(Len, Op1C->getZExtValue() + 1))); } } } @@ -2489,7 +2817,7 @@ Instruction *InstCombinerImpl::visitCallBase(CallBase &Call) { // isKnownNonNull -> nonnull attribute if (!GCR.hasRetAttr(Attribute::NonNull) && isKnownNonZero(DerivedPtr, DL, 0, &AC, &Call, &DT)) { - GCR.addAttribute(AttributeList::ReturnIndex, Attribute::NonNull); + GCR.addRetAttr(Attribute::NonNull); // We discovered new fact, re-check users. Worklist.pushUsersToWorkList(GCR); } @@ -2646,19 +2974,19 @@ bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) { if (!CastInst::isBitOrNoopPointerCastable(ActTy, ParamTy, DL)) return false; // Cannot transform this parameter value. - if (AttrBuilder(CallerPAL.getParamAttributes(i)) + if (AttrBuilder(CallerPAL.getParamAttrs(i)) .overlaps(AttributeFuncs::typeIncompatible(ParamTy))) return false; // Attribute not compatible with transformed value. if (Call.isInAllocaArgument(i)) return false; // Cannot transform to and from inalloca. - if (CallerPAL.hasParamAttribute(i, Attribute::SwiftError)) + if (CallerPAL.hasParamAttr(i, Attribute::SwiftError)) return false; // If the parameter is passed as a byval argument, then we have to have a // sized type and the sized type has to have the same size as the old type. - if (ParamTy != ActTy && CallerPAL.hasParamAttribute(i, Attribute::ByVal)) { + if (ParamTy != ActTy && CallerPAL.hasParamAttr(i, Attribute::ByVal)) { PointerType *ParamPTy = dyn_cast<PointerType>(ParamTy); if (!ParamPTy || !ParamPTy->getElementType()->isSized()) return false; @@ -2699,7 +3027,7 @@ bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) { // that are compatible with being a vararg call argument. unsigned SRetIdx; if (CallerPAL.hasAttrSomewhere(Attribute::StructRet, &SRetIdx) && - SRetIdx > FT->getNumParams()) + SRetIdx - AttributeList::FirstArgIndex >= FT->getNumParams()) return false; } @@ -2728,12 +3056,12 @@ bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) { Args.push_back(NewArg); // Add any parameter attributes. - if (CallerPAL.hasParamAttribute(i, Attribute::ByVal)) { - AttrBuilder AB(CallerPAL.getParamAttributes(i)); + if (CallerPAL.hasParamAttr(i, Attribute::ByVal)) { + AttrBuilder AB(CallerPAL.getParamAttrs(i)); AB.addByValAttr(NewArg->getType()->getPointerElementType()); ArgAttrs.push_back(AttributeSet::get(Ctx, AB)); } else - ArgAttrs.push_back(CallerPAL.getParamAttributes(i)); + ArgAttrs.push_back(CallerPAL.getParamAttrs(i)); } // If the function takes more arguments than the call was taking, add them @@ -2760,12 +3088,12 @@ bool InstCombinerImpl::transformConstExprCastCall(CallBase &Call) { Args.push_back(NewArg); // Add any parameter attributes. - ArgAttrs.push_back(CallerPAL.getParamAttributes(i)); + ArgAttrs.push_back(CallerPAL.getParamAttrs(i)); } } } - AttributeSet FnAttrs = CallerPAL.getFnAttributes(); + AttributeSet FnAttrs = CallerPAL.getFnAttrs(); if (NewRetTy->isVoidTy()) Caller->setName(""); // Void type should not have a name. @@ -2866,7 +3194,7 @@ InstCombinerImpl::transformCallThroughTrampoline(CallBase &Call, for (FunctionType::param_iterator I = NestFTy->param_begin(), E = NestFTy->param_end(); I != E; ++NestArgNo, ++I) { - AttributeSet AS = NestAttrs.getParamAttributes(NestArgNo); + AttributeSet AS = NestAttrs.getParamAttrs(NestArgNo); if (AS.hasAttribute(Attribute::Nest)) { // Record the parameter type and any other attributes. NestTy = *I; @@ -2902,7 +3230,7 @@ InstCombinerImpl::transformCallThroughTrampoline(CallBase &Call, // Add the original argument and attributes. NewArgs.push_back(*I); - NewArgAttrs.push_back(Attrs.getParamAttributes(ArgNo)); + NewArgAttrs.push_back(Attrs.getParamAttrs(ArgNo)); ++ArgNo; ++I; @@ -2948,8 +3276,8 @@ InstCombinerImpl::transformCallThroughTrampoline(CallBase &Call, NestF : ConstantExpr::getBitCast(NestF, PointerType::getUnqual(NewFTy)); AttributeList NewPAL = - AttributeList::get(FTy->getContext(), Attrs.getFnAttributes(), - Attrs.getRetAttributes(), NewArgAttrs); + AttributeList::get(FTy->getContext(), Attrs.getFnAttrs(), + Attrs.getRetAttrs(), NewArgAttrs); SmallVector<OperandBundleDef, 1> OpBundles; Call.getOperandBundlesAsDefs(OpBundles); |