diff options
Diffstat (limited to 'llvm/lib/Transforms')
30 files changed, 488 insertions, 308 deletions
diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 8e1f782f7cd8..b2618e35b085 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -291,42 +291,15 @@ static const Value *getPointerOperand(const Instruction *I, } /// Helper function to create a pointer based on \p Ptr, and advanced by \p -/// Offset bytes. To aid later analysis the method tries to build -/// getelement pointer instructions that traverse the natural type of \p Ptr if -/// possible. If that fails, the remaining offset is adjusted byte-wise, hence -/// through a cast to i8*. -/// -/// TODO: This could probably live somewhere more prominantly if it doesn't -/// already exist. -static Value *constructPointer(Type *PtrElemTy, Value *Ptr, int64_t Offset, - IRBuilder<NoFolder> &IRB, const DataLayout &DL) { - assert(Offset >= 0 && "Negative offset not supported yet!"); +/// Offset bytes. +static Value *constructPointer(Value *Ptr, int64_t Offset, + IRBuilder<NoFolder> &IRB) { LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset << "-bytes\n"); - if (Offset) { - Type *Ty = PtrElemTy; - APInt IntOffset(DL.getIndexTypeSizeInBits(Ptr->getType()), Offset); - SmallVector<APInt> IntIndices = DL.getGEPIndicesForOffset(Ty, IntOffset); - - SmallVector<Value *, 4> ValIndices; - std::string GEPName = Ptr->getName().str(); - for (const APInt &Index : IntIndices) { - ValIndices.push_back(IRB.getInt(Index)); - GEPName += "." + std::to_string(Index.getZExtValue()); - } - - // Create a GEP for the indices collected above. - Ptr = IRB.CreateGEP(PtrElemTy, Ptr, ValIndices, GEPName); - - // If an offset is left we use byte-wise adjustment. - if (IntOffset != 0) { - Ptr = IRB.CreateGEP(IRB.getInt8Ty(), Ptr, IRB.getInt(IntOffset), - GEPName + ".b" + Twine(IntOffset.getZExtValue())); - } - } - - LLVM_DEBUG(dbgs() << "Constructed pointer: " << *Ptr << "\n"); + if (Offset) + Ptr = IRB.CreateGEP(IRB.getInt8Ty(), Ptr, IRB.getInt64(Offset), + Ptr->getName() + ".b" + Twine(Offset)); return Ptr; } @@ -7487,16 +7460,15 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { if (auto *PrivStructType = dyn_cast<StructType>(PrivType)) { const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType); for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) { - Value *Ptr = constructPointer( - PrivType, &Base, PrivStructLayout->getElementOffset(u), IRB, DL); + Value *Ptr = + constructPointer(&Base, PrivStructLayout->getElementOffset(u), IRB); new StoreInst(F.getArg(ArgNo + u), Ptr, &IP); } } else if (auto *PrivArrayType = dyn_cast<ArrayType>(PrivType)) { Type *PointeeTy = PrivArrayType->getElementType(); uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy); for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { - Value *Ptr = - constructPointer(PrivType, &Base, u * PointeeTySize, IRB, DL); + Value *Ptr = constructPointer(&Base, u * PointeeTySize, IRB); new StoreInst(F.getArg(ArgNo + u), Ptr, &IP); } } else { @@ -7521,8 +7493,8 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType); for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) { Type *PointeeTy = PrivStructType->getElementType(u); - Value *Ptr = constructPointer( - PrivType, Base, PrivStructLayout->getElementOffset(u), IRB, DL); + Value *Ptr = + constructPointer(Base, PrivStructLayout->getElementOffset(u), IRB); LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP); L->setAlignment(Alignment); ReplacementValues.push_back(L); @@ -7531,8 +7503,7 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { Type *PointeeTy = PrivArrayType->getElementType(); uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy); for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { - Value *Ptr = - constructPointer(PrivType, Base, u * PointeeTySize, IRB, DL); + Value *Ptr = constructPointer(Base, u * PointeeTySize, IRB); LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP); L->setAlignment(Alignment); ReplacementValues.push_back(L); diff --git a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp index b2665161c090..4176d561363f 100644 --- a/llvm/lib/Transforms/IPO/OpenMPOpt.cpp +++ b/llvm/lib/Transforms/IPO/OpenMPOpt.cpp @@ -2053,6 +2053,9 @@ private: LLVM_DEBUG(dbgs() << "[Attributor] Done with " << SCC.size() << " functions, result: " << Changed << ".\n"); + if (Changed == ChangeStatus::CHANGED) + OMPInfoCache.invalidateAnalyses(); + return Changed == ChangeStatus::CHANGED; } @@ -3763,7 +3766,7 @@ struct AAKernelInfoFunction : AAKernelInfo { ConstantInt *ExecModeC = KernelInfo::getExecModeFromKernelEnvironment(KernelEnvC); ConstantInt *AssumedExecModeC = ConstantInt::get( - ExecModeC->getType(), + ExecModeC->getIntegerType(), ExecModeC->getSExtValue() | OMP_TGT_EXEC_MODE_GENERIC_SPMD); if (ExecModeC->getSExtValue() & OMP_TGT_EXEC_MODE_SPMD) SPMDCompatibilityTracker.indicateOptimisticFixpoint(); @@ -3792,7 +3795,7 @@ struct AAKernelInfoFunction : AAKernelInfo { ConstantInt *MayUseNestedParallelismC = KernelInfo::getMayUseNestedParallelismFromKernelEnvironment(KernelEnvC); ConstantInt *AssumedMayUseNestedParallelismC = ConstantInt::get( - MayUseNestedParallelismC->getType(), NestedParallelism); + MayUseNestedParallelismC->getIntegerType(), NestedParallelism); setMayUseNestedParallelismOfKernelEnvironment( AssumedMayUseNestedParallelismC); @@ -3801,7 +3804,7 @@ struct AAKernelInfoFunction : AAKernelInfo { KernelInfo::getUseGenericStateMachineFromKernelEnvironment( KernelEnvC); ConstantInt *AssumedUseGenericStateMachineC = - ConstantInt::get(UseGenericStateMachineC->getType(), false); + ConstantInt::get(UseGenericStateMachineC->getIntegerType(), false); setUseGenericStateMachineOfKernelEnvironment( AssumedUseGenericStateMachineC); } @@ -4280,8 +4283,9 @@ struct AAKernelInfoFunction : AAKernelInfo { // kernel is executed in. assert(ExecModeVal == OMP_TGT_EXEC_MODE_GENERIC && "Initially non-SPMD kernel has SPMD exec mode!"); - setExecModeOfKernelEnvironment(ConstantInt::get( - ExecModeC->getType(), ExecModeVal | OMP_TGT_EXEC_MODE_GENERIC_SPMD)); + setExecModeOfKernelEnvironment( + ConstantInt::get(ExecModeC->getIntegerType(), + ExecModeVal | OMP_TGT_EXEC_MODE_GENERIC_SPMD)); ++NumOpenMPTargetRegionKernelsSPMD; @@ -4332,7 +4336,7 @@ struct AAKernelInfoFunction : AAKernelInfo { // If not SPMD mode, indicate we use a custom state machine now. setUseGenericStateMachineOfKernelEnvironment( - ConstantInt::get(UseStateMachineC->getType(), false)); + ConstantInt::get(UseStateMachineC->getIntegerType(), false)); // If we don't actually need a state machine we are done here. This can // happen if there simply are no parallel regions. In the resulting kernel @@ -4658,7 +4662,7 @@ struct AAKernelInfoFunction : AAKernelInfo { KernelInfo::getMayUseNestedParallelismFromKernelEnvironment( AA.KernelEnvC); ConstantInt *NewMayUseNestedParallelismC = ConstantInt::get( - MayUseNestedParallelismC->getType(), AA.NestedParallelism); + MayUseNestedParallelismC->getIntegerType(), AA.NestedParallelism); AA.setMayUseNestedParallelismOfKernelEnvironment( NewMayUseNestedParallelismC); } diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index 6c6f0a0eca72..2fd8668d15e2 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -794,10 +794,9 @@ SampleProfileLoader::findIndirectCallFunctionSamples( return R; auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL); - auto T = FS->findCallTargetMapAt(CallSite); Sum = 0; - if (T) - for (const auto &T_C : T.get()) + if (auto T = FS->findCallTargetMapAt(CallSite)) + for (const auto &T_C : *T) Sum += T_C.second; if (const FunctionSamplesMap *M = FS->findFunctionSamplesMapAt(CallSite)) { if (M->empty()) @@ -1679,7 +1678,8 @@ void SampleProfileLoader::generateMDProfMetadata(Function &F) { if (!FS) continue; auto CallSite = FunctionSamples::getCallSiteIdentifier(DIL); - auto T = FS->findCallTargetMapAt(CallSite); + ErrorOr<SampleRecord::CallTargetMap> T = + FS->findCallTargetMapAt(CallSite); if (!T || T.get().empty()) continue; if (FunctionSamples::ProfileIsProbeBased) { @@ -2261,9 +2261,8 @@ void SampleProfileMatcher::countProfileCallsiteMismatches( // Compute number of samples in the original profile. uint64_t CallsiteSamples = 0; - auto CTM = FS.findCallTargetMapAt(Loc); - if (CTM) { - for (const auto &I : CTM.get()) + if (auto CTM = FS.findCallTargetMapAt(Loc)) { + for (const auto &I : *CTM) CallsiteSamples += I.second; } const auto *FSMap = FS.findFunctionSamplesMapAt(Loc); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp index 5e362f4117d0..63b1e0f64a88 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp @@ -3956,35 +3956,50 @@ Value *InstCombinerImpl::foldXorOfICmps(ICmpInst *LHS, ICmpInst *RHS, const APInt *LC, *RC; if (match(LHS1, m_APInt(LC)) && match(RHS1, m_APInt(RC)) && LHS0->getType() == RHS0->getType() && - LHS0->getType()->isIntOrIntVectorTy() && - (LHS->hasOneUse() || RHS->hasOneUse())) { + LHS0->getType()->isIntOrIntVectorTy()) { // Convert xor of signbit tests to signbit test of xor'd values: // (X > -1) ^ (Y > -1) --> (X ^ Y) < 0 // (X < 0) ^ (Y < 0) --> (X ^ Y) < 0 // (X > -1) ^ (Y < 0) --> (X ^ Y) > -1 // (X < 0) ^ (Y > -1) --> (X ^ Y) > -1 bool TrueIfSignedL, TrueIfSignedR; - if (isSignBitCheck(PredL, *LC, TrueIfSignedL) && + if ((LHS->hasOneUse() || RHS->hasOneUse()) && + isSignBitCheck(PredL, *LC, TrueIfSignedL) && isSignBitCheck(PredR, *RC, TrueIfSignedR)) { Value *XorLR = Builder.CreateXor(LHS0, RHS0); return TrueIfSignedL == TrueIfSignedR ? Builder.CreateIsNeg(XorLR) : Builder.CreateIsNotNeg(XorLR); } - // (X > C) ^ (X < C + 2) --> X != C + 1 - // (X < C + 2) ^ (X > C) --> X != C + 1 - // Considering the correctness of this pattern, we should avoid that C is - // non-negative and C + 2 is negative, although it will be matched by other - // patterns. - const APInt *C1, *C2; - if ((PredL == CmpInst::ICMP_SGT && match(LHS1, m_APInt(C1)) && - PredR == CmpInst::ICMP_SLT && match(RHS1, m_APInt(C2))) || - (PredL == CmpInst::ICMP_SLT && match(LHS1, m_APInt(C2)) && - PredR == CmpInst::ICMP_SGT && match(RHS1, m_APInt(C1)))) - if (LHS0 == RHS0 && *C1 + 2 == *C2 && - (C1->isNegative() || C2->isNonNegative())) - return Builder.CreateICmpNE(LHS0, - ConstantInt::get(LHS0->getType(), *C1 + 1)); + // Fold (icmp pred1 X, C1) ^ (icmp pred2 X, C2) + // into a single comparison using range-based reasoning. + if (LHS0 == RHS0) { + ConstantRange CR1 = ConstantRange::makeExactICmpRegion(PredL, *LC); + ConstantRange CR2 = ConstantRange::makeExactICmpRegion(PredR, *RC); + auto CRUnion = CR1.exactUnionWith(CR2); + auto CRIntersect = CR1.exactIntersectWith(CR2); + if (CRUnion && CRIntersect) + if (auto CR = CRUnion->exactIntersectWith(CRIntersect->inverse())) { + if (CR->isFullSet()) + return ConstantInt::getTrue(I.getType()); + if (CR->isEmptySet()) + return ConstantInt::getFalse(I.getType()); + + CmpInst::Predicate NewPred; + APInt NewC, Offset; + CR->getEquivalentICmp(NewPred, NewC, Offset); + + if ((Offset.isZero() && (LHS->hasOneUse() || RHS->hasOneUse())) || + (LHS->hasOneUse() && RHS->hasOneUse())) { + Value *NewV = LHS0; + Type *Ty = LHS0->getType(); + if (!Offset.isZero()) + NewV = Builder.CreateAdd(NewV, ConstantInt::get(Ty, Offset)); + return Builder.CreateICmp(NewPred, NewV, + ConstantInt::get(Ty, NewC)); + } + } + } } // Instead of trying to imitate the folds for and/or, decompose this 'xor' diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index 1539fa9a3269..3b7fe7fa2266 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -357,9 +357,9 @@ Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) { // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask); - APInt UndefElts(DemandedElts.getBitWidth(), 0); - if (Value *V = - SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, UndefElts)) + APInt PoisonElts(DemandedElts.getBitWidth(), 0); + if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, + PoisonElts)) return replaceOperand(II, 0, V); return nullptr; @@ -439,12 +439,12 @@ Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) { // Use masked off lanes to simplify operands via SimplifyDemandedVectorElts APInt DemandedElts = possiblyDemandedEltsInMask(ConstMask); - APInt UndefElts(DemandedElts.getBitWidth(), 0); - if (Value *V = - SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, UndefElts)) + APInt PoisonElts(DemandedElts.getBitWidth(), 0); + if (Value *V = SimplifyDemandedVectorElts(II.getOperand(0), DemandedElts, + PoisonElts)) return replaceOperand(II, 0, V); - if (Value *V = - SimplifyDemandedVectorElts(II.getOperand(1), DemandedElts, UndefElts)) + if (Value *V = SimplifyDemandedVectorElts(II.getOperand(1), DemandedElts, + PoisonElts)) return replaceOperand(II, 1, V); return nullptr; @@ -1526,9 +1526,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { // support. if (auto *IIFVTy = dyn_cast<FixedVectorType>(II->getType())) { auto VWidth = IIFVTy->getNumElements(); - APInt UndefElts(VWidth, 0); + APInt PoisonElts(VWidth, 0); APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); - if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, UndefElts)) { + if (Value *V = SimplifyDemandedVectorElts(II, AllOnesEltMask, PoisonElts)) { if (V != II) return replaceInstUsesWith(*II, V); return II; @@ -1539,6 +1539,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (Instruction *I = foldCommutativeIntrinsicOverSelects(*II)) return I; + if (Instruction *I = foldCommutativeIntrinsicOverPhis(*II)) + return I; + if (CallInst *NewCall = canonicalizeConstantArg0ToArg1(CI)) return NewCall; } @@ -1793,6 +1796,23 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) { if (Instruction *NewMinMax = factorizeMinMaxTree(II)) return NewMinMax; + // Try to fold minmax with constant RHS based on range information + const APInt *RHSC; + if (match(I1, m_APIntAllowUndef(RHSC))) { + ICmpInst::Predicate Pred = + ICmpInst::getNonStrictPredicate(MinMaxIntrinsic::getPredicate(IID)); + bool IsSigned = MinMaxIntrinsic::isSigned(IID); + ConstantRange LHS_CR = computeConstantRangeIncludingKnownBits( + I0, IsSigned, SQ.getWithInstruction(II)); + if (!LHS_CR.isFullSet()) { + if (LHS_CR.icmp(Pred, *RHSC)) + return replaceInstUsesWith(*II, I0); + if (LHS_CR.icmp(ICmpInst::getSwappedPredicate(Pred), *RHSC)) + return replaceInstUsesWith(*II, + ConstantInt::get(II->getType(), *RHSC)); + } + } + break; } case Intrinsic::bitreverse: { @@ -4237,3 +4257,22 @@ InstCombinerImpl::foldCommutativeIntrinsicOverSelects(IntrinsicInst &II) { return nullptr; } + +Instruction * +InstCombinerImpl::foldCommutativeIntrinsicOverPhis(IntrinsicInst &II) { + assert(II.isCommutative() && "Instruction should be commutative"); + + PHINode *LHS = dyn_cast<PHINode>(II.getOperand(0)); + PHINode *RHS = dyn_cast<PHINode>(II.getOperand(1)); + + if (!LHS || !RHS) + return nullptr; + + if (auto P = matchSymmetricPhiNodesPair(LHS, RHS)) { + replaceOperand(II, 0, P->first); + replaceOperand(II, 1, P->second); + return &II; + } + + return nullptr; +} diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h index 1d50fa9b6bf7..9e76a0cf17b1 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -278,6 +278,16 @@ private: IntrinsicInst &Tramp); Instruction *foldCommutativeIntrinsicOverSelects(IntrinsicInst &II); + // Match a pair of Phi Nodes like + // phi [a, BB0], [b, BB1] & phi [b, BB0], [a, BB1] + // Return the matched two operands. + std::optional<std::pair<Value *, Value *>> + matchSymmetricPhiNodesPair(PHINode *LHS, PHINode *RHS); + + // Tries to fold (op phi(a, b) phi(b, a)) -> (op a, b) + // while op is a commutative intrinsic call. + Instruction *foldCommutativeIntrinsicOverPhis(IntrinsicInst &II); + Value *simplifyMaskedLoad(IntrinsicInst &II); Instruction *simplifyMaskedStore(IntrinsicInst &II); Instruction *simplifyMaskedGather(IntrinsicInst &II); @@ -492,6 +502,11 @@ public: /// X % (C0 * C1) Value *SimplifyAddWithRemainder(BinaryOperator &I); + // Tries to fold (Binop phi(a, b) phi(b, a)) -> (Binop a, b) + // while Binop is commutative. + Value *SimplifyPhiCommutativeBinaryOp(BinaryOperator &I, Value *LHS, + Value *RHS); + // Binary Op helper for select operations where the expression can be // efficiently reorganized. Value *SimplifySelectsFeedingBinaryOp(BinaryOperator &I, Value *LHS, @@ -550,7 +565,7 @@ public: bool SimplifyDemandedInstructionBits(Instruction &Inst, KnownBits &Known); Value *SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, - APInt &UndefElts, unsigned Depth = 0, + APInt &PoisonElts, unsigned Depth = 0, bool AllowMultipleUsers = false) override; /// Canonicalize the position of binops relative to shufflevector. diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index e5566578869d..f0ea3d9fcad5 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -350,6 +350,13 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) { if (match(&I, m_c_Mul(m_OneUse(m_Neg(m_Value(X))), m_Value(Y)))) return BinaryOperator::CreateNeg(Builder.CreateMul(X, Y)); + // (-X * Y) * -X --> (X * Y) * X + // (-X << Y) * -X --> (X << Y) * X + if (match(Op1, m_Neg(m_Value(X)))) { + if (Value *NegOp0 = Negator::Negate(false, /*IsNSW*/ false, Op0, *this)) + return BinaryOperator::CreateMul(NegOp0, X); + } + // (X / Y) * Y = X - (X % Y) // (X / Y) * -Y = (X % Y) - X { diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index 2dda46986f0f..20bf00344b14 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -2440,9 +2440,9 @@ Instruction *InstCombinerImpl::foldVectorSelect(SelectInst &Sel) { return nullptr; unsigned NumElts = VecTy->getNumElements(); - APInt UndefElts(NumElts, 0); + APInt PoisonElts(NumElts, 0); APInt AllOnesEltMask(APInt::getAllOnes(NumElts)); - if (Value *V = SimplifyDemandedVectorElts(&Sel, AllOnesEltMask, UndefElts)) { + if (Value *V = SimplifyDemandedVectorElts(&Sel, AllOnesEltMask, PoisonElts)) { if (V != &Sel) return replaceInstUsesWith(Sel, V); return &Sel; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp index 846116a929b1..a8a5f9831e15 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp @@ -1319,8 +1319,8 @@ Value *InstCombinerImpl::simplifyShrShlDemandedBits( } /// The specified value produces a vector with any number of elements. -/// This method analyzes which elements of the operand are undef or poison and -/// returns that information in UndefElts. +/// This method analyzes which elements of the operand are poison and +/// returns that information in PoisonElts. /// /// DemandedElts contains the set of elements that are actually used by the /// caller, and by default (AllowMultipleUsers equals false) the value is @@ -1333,7 +1333,7 @@ Value *InstCombinerImpl::simplifyShrShlDemandedBits( /// returned. This returns null if no change was made. Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, APInt DemandedElts, - APInt &UndefElts, + APInt &PoisonElts, unsigned Depth, bool AllowMultipleUsers) { // Cannot analyze scalable type. The number of vector elements is not a @@ -1345,18 +1345,18 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, APInt EltMask(APInt::getAllOnes(VWidth)); assert((DemandedElts & ~EltMask) == 0 && "Invalid DemandedElts!"); - if (match(V, m_Undef())) { - // If the entire vector is undef or poison, just return this info. - UndefElts = EltMask; + if (match(V, m_Poison())) { + // If the entire vector is poison, just return this info. + PoisonElts = EltMask; return nullptr; } if (DemandedElts.isZero()) { // If nothing is demanded, provide poison. - UndefElts = EltMask; + PoisonElts = EltMask; return PoisonValue::get(V->getType()); } - UndefElts = 0; + PoisonElts = 0; if (auto *C = dyn_cast<Constant>(V)) { // Check if this is identity. If so, return 0 since we are not simplifying @@ -1370,7 +1370,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, for (unsigned i = 0; i != VWidth; ++i) { if (!DemandedElts[i]) { // If not demanded, set to poison. Elts.push_back(Poison); - UndefElts.setBit(i); + PoisonElts.setBit(i); continue; } @@ -1378,8 +1378,8 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, if (!Elt) return nullptr; Elts.push_back(Elt); - if (isa<UndefValue>(Elt)) // Already undef or poison. - UndefElts.setBit(i); + if (isa<PoisonValue>(Elt)) // Already poison. + PoisonElts.setBit(i); } // If we changed the constant, return it. @@ -1400,7 +1400,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, // They'll be handled when it's their turn to be visited by // the main instcombine process. if (Depth != 0) - // TODO: Just compute the UndefElts information recursively. + // TODO: Just compute the PoisonElts information recursively. return nullptr; // Conservatively assume that all elements are needed. @@ -1422,8 +1422,8 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } }; - APInt UndefElts2(VWidth, 0); - APInt UndefElts3(VWidth, 0); + APInt PoisonElts2(VWidth, 0); + APInt PoisonElts3(VWidth, 0); switch (I->getOpcode()) { default: break; @@ -1449,17 +1449,17 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, if (i == 0 ? match(I->getOperand(i), m_Undef()) : match(I->getOperand(i), m_Poison())) { // If the entire vector is undefined, just return this info. - UndefElts = EltMask; + PoisonElts = EltMask; return nullptr; } if (I->getOperand(i)->getType()->isVectorTy()) { - APInt UndefEltsOp(VWidth, 0); - simplifyAndSetOp(I, i, DemandedElts, UndefEltsOp); + APInt PoisonEltsOp(VWidth, 0); + simplifyAndSetOp(I, i, DemandedElts, PoisonEltsOp); // gep(x, undef) is not undef, so skip considering idx ops here // Note that we could propagate poison, but we can't distinguish between // undef & poison bits ATM if (i == 0) - UndefElts |= UndefEltsOp; + PoisonElts |= PoisonEltsOp; } } @@ -1472,7 +1472,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, if (!Idx) { // Note that we can't propagate undef elt info, because we don't know // which elt is getting updated. - simplifyAndSetOp(I, 0, DemandedElts, UndefElts2); + simplifyAndSetOp(I, 0, DemandedElts, PoisonElts2); break; } @@ -1487,7 +1487,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, // was extracted from the same index in another vector with the same type, // replace this insert with that other vector. // Note: This is attempted before the call to simplifyAndSetOp because that - // may change UndefElts to a value that does not match with Vec. + // may change PoisonElts to a value that does not match with Vec. Value *Vec; if (PreInsertDemandedElts == 0 && match(I->getOperand(1), @@ -1496,7 +1496,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return Vec; } - simplifyAndSetOp(I, 0, PreInsertDemandedElts, UndefElts); + simplifyAndSetOp(I, 0, PreInsertDemandedElts, PoisonElts); // If this is inserting an element that isn't demanded, remove this // insertelement. @@ -1506,7 +1506,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } // The inserted element is defined. - UndefElts.clearBit(IdxNo); + PoisonElts.clearBit(IdxNo); break; } case Instruction::ShuffleVector: { @@ -1520,17 +1520,17 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, // operand. if (all_of(Shuffle->getShuffleMask(), [](int Elt) { return Elt == 0; }) && DemandedElts.isAllOnes()) { - if (!match(I->getOperand(1), m_Undef())) { + if (!isa<PoisonValue>(I->getOperand(1))) { I->setOperand(1, PoisonValue::get(I->getOperand(1)->getType())); MadeChange = true; } APInt LeftDemanded(OpWidth, 1); - APInt LHSUndefElts(OpWidth, 0); - simplifyAndSetOp(I, 0, LeftDemanded, LHSUndefElts); - if (LHSUndefElts[0]) - UndefElts = EltMask; + APInt LHSPoisonElts(OpWidth, 0); + simplifyAndSetOp(I, 0, LeftDemanded, LHSPoisonElts); + if (LHSPoisonElts[0]) + PoisonElts = EltMask; else - UndefElts.clearAllBits(); + PoisonElts.clearAllBits(); break; } @@ -1549,11 +1549,11 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } } - APInt LHSUndefElts(OpWidth, 0); - simplifyAndSetOp(I, 0, LeftDemanded, LHSUndefElts); + APInt LHSPoisonElts(OpWidth, 0); + simplifyAndSetOp(I, 0, LeftDemanded, LHSPoisonElts); - APInt RHSUndefElts(OpWidth, 0); - simplifyAndSetOp(I, 1, RightDemanded, RHSUndefElts); + APInt RHSPoisonElts(OpWidth, 0); + simplifyAndSetOp(I, 1, RightDemanded, RHSPoisonElts); // If this shuffle does not change the vector length and the elements // demanded by this shuffle are an identity mask, then this shuffle is @@ -1579,7 +1579,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return Shuffle->getOperand(0); } - bool NewUndefElts = false; + bool NewPoisonElts = false; unsigned LHSIdx = -1u, LHSValIdx = -1u; unsigned RHSIdx = -1u, RHSValIdx = -1u; bool LHSUniform = true; @@ -1587,23 +1587,23 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, for (unsigned i = 0; i < VWidth; i++) { unsigned MaskVal = Shuffle->getMaskValue(i); if (MaskVal == -1u) { - UndefElts.setBit(i); + PoisonElts.setBit(i); } else if (!DemandedElts[i]) { - NewUndefElts = true; - UndefElts.setBit(i); + NewPoisonElts = true; + PoisonElts.setBit(i); } else if (MaskVal < OpWidth) { - if (LHSUndefElts[MaskVal]) { - NewUndefElts = true; - UndefElts.setBit(i); + if (LHSPoisonElts[MaskVal]) { + NewPoisonElts = true; + PoisonElts.setBit(i); } else { LHSIdx = LHSIdx == -1u ? i : OpWidth; LHSValIdx = LHSValIdx == -1u ? MaskVal : OpWidth; LHSUniform = LHSUniform && (MaskVal == i); } } else { - if (RHSUndefElts[MaskVal - OpWidth]) { - NewUndefElts = true; - UndefElts.setBit(i); + if (RHSPoisonElts[MaskVal - OpWidth]) { + NewPoisonElts = true; + PoisonElts.setBit(i); } else { RHSIdx = RHSIdx == -1u ? i : OpWidth; RHSValIdx = RHSValIdx == -1u ? MaskVal - OpWidth : OpWidth; @@ -1646,11 +1646,11 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return New; } } - if (NewUndefElts) { + if (NewPoisonElts) { // Add additional discovered undefs. SmallVector<int, 16> Elts; for (unsigned i = 0; i < VWidth; ++i) { - if (UndefElts[i]) + if (PoisonElts[i]) Elts.push_back(PoisonMaskElem); else Elts.push_back(Shuffle->getMaskValue(i)); @@ -1665,12 +1665,12 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, // on the current demanded elements. SelectInst *Sel = cast<SelectInst>(I); if (Sel->getCondition()->getType()->isVectorTy()) { - // TODO: We are not doing anything with UndefElts based on this call. + // TODO: We are not doing anything with PoisonElts based on this call. // It is overwritten below based on the other select operands. If an // element of the select condition is known undef, then we are free to // choose the output value from either arm of the select. If we know that // one of those values is undef, then the output can be undef. - simplifyAndSetOp(I, 0, DemandedElts, UndefElts); + simplifyAndSetOp(I, 0, DemandedElts, PoisonElts); } // Next, see if we can transform the arms of the select. @@ -1692,12 +1692,12 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } } - simplifyAndSetOp(I, 1, DemandedLHS, UndefElts2); - simplifyAndSetOp(I, 2, DemandedRHS, UndefElts3); + simplifyAndSetOp(I, 1, DemandedLHS, PoisonElts2); + simplifyAndSetOp(I, 2, DemandedRHS, PoisonElts3); // Output elements are undefined if the element from each arm is undefined. // TODO: This can be improved. See comment in select condition handling. - UndefElts = UndefElts2 & UndefElts3; + PoisonElts = PoisonElts2 & PoisonElts3; break; } case Instruction::BitCast: { @@ -1706,7 +1706,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, if (!VTy) break; unsigned InVWidth = cast<FixedVectorType>(VTy)->getNumElements(); APInt InputDemandedElts(InVWidth, 0); - UndefElts2 = APInt(InVWidth, 0); + PoisonElts2 = APInt(InVWidth, 0); unsigned Ratio; if (VWidth == InVWidth) { @@ -1735,25 +1735,25 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, break; } - simplifyAndSetOp(I, 0, InputDemandedElts, UndefElts2); + simplifyAndSetOp(I, 0, InputDemandedElts, PoisonElts2); if (VWidth == InVWidth) { - UndefElts = UndefElts2; + PoisonElts = PoisonElts2; } else if ((VWidth % InVWidth) == 0) { // If the number of elements in the output is a multiple of the number of // elements in the input then an output element is undef if the // corresponding input element is undef. for (unsigned OutIdx = 0; OutIdx != VWidth; ++OutIdx) - if (UndefElts2[OutIdx / Ratio]) - UndefElts.setBit(OutIdx); + if (PoisonElts2[OutIdx / Ratio]) + PoisonElts.setBit(OutIdx); } else if ((InVWidth % VWidth) == 0) { // If the number of elements in the input is a multiple of the number of // elements in the output then an output element is undef if all of the // corresponding input elements are undef. for (unsigned OutIdx = 0; OutIdx != VWidth; ++OutIdx) { - APInt SubUndef = UndefElts2.lshr(OutIdx * Ratio).zextOrTrunc(Ratio); + APInt SubUndef = PoisonElts2.lshr(OutIdx * Ratio).zextOrTrunc(Ratio); if (SubUndef.popcount() == Ratio) - UndefElts.setBit(OutIdx); + PoisonElts.setBit(OutIdx); } } else { llvm_unreachable("Unimp"); @@ -1762,7 +1762,7 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, } case Instruction::FPTrunc: case Instruction::FPExt: - simplifyAndSetOp(I, 0, DemandedElts, UndefElts); + simplifyAndSetOp(I, 0, DemandedElts, PoisonElts); break; case Instruction::Call: { @@ -1785,18 +1785,18 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, DemandedPassThrough.clearBit(i); } if (II->getIntrinsicID() == Intrinsic::masked_gather) - simplifyAndSetOp(II, 0, DemandedPtrs, UndefElts2); - simplifyAndSetOp(II, 3, DemandedPassThrough, UndefElts3); + simplifyAndSetOp(II, 0, DemandedPtrs, PoisonElts2); + simplifyAndSetOp(II, 3, DemandedPassThrough, PoisonElts3); // Output elements are undefined if the element from both sources are. // TODO: can strengthen via mask as well. - UndefElts = UndefElts2 & UndefElts3; + PoisonElts = PoisonElts2 & PoisonElts3; break; } default: { // Handle target specific intrinsics std::optional<Value *> V = targetSimplifyDemandedVectorEltsIntrinsic( - *II, DemandedElts, UndefElts, UndefElts2, UndefElts3, + *II, DemandedElts, PoisonElts, PoisonElts2, PoisonElts3, simplifyAndSetOp); if (V) return *V; @@ -1859,18 +1859,18 @@ Value *InstCombinerImpl::SimplifyDemandedVectorElts(Value *V, return ShufBO; } - simplifyAndSetOp(I, 0, DemandedElts, UndefElts); - simplifyAndSetOp(I, 1, DemandedElts, UndefElts2); + simplifyAndSetOp(I, 0, DemandedElts, PoisonElts); + simplifyAndSetOp(I, 1, DemandedElts, PoisonElts2); // Output elements are undefined if both are undefined. Consider things // like undef & 0. The result is known zero, not undef. - UndefElts &= UndefElts2; + PoisonElts &= PoisonElts2; } - // If we've proven all of the lanes undef, return an undef value. + // If we've proven all of the lanes poison, return a poison value. // TODO: Intersect w/demanded lanes - if (UndefElts.isAllOnes()) - return UndefValue::get(I->getType()); + if (PoisonElts.isAllOnes()) + return PoisonValue::get(I->getType()); return MadeChange ? I : nullptr; } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp index c8b58c51d4e6..18ab510aae7f 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineVectorOps.cpp @@ -388,7 +388,7 @@ static APInt findDemandedEltsByAllUsers(Value *V) { /// arbitrarily pick 64 bit as our canonical type. The actual bitwidth doesn't /// matter, we just want a consistent type to simplify CSE. static ConstantInt *getPreferredVectorIndex(ConstantInt *IndexC) { - const unsigned IndexBW = IndexC->getType()->getBitWidth(); + const unsigned IndexBW = IndexC->getBitWidth(); if (IndexBW == 64 || IndexC->getValue().getActiveBits() > 64) return nullptr; return ConstantInt::get(IndexC->getContext(), @@ -581,20 +581,20 @@ Instruction *InstCombinerImpl::visitExtractElementInst(ExtractElementInst &EI) { // If the input vector has a single use, simplify it based on this use // property. if (SrcVec->hasOneUse()) { - APInt UndefElts(NumElts, 0); + APInt PoisonElts(NumElts, 0); APInt DemandedElts(NumElts, 0); DemandedElts.setBit(IndexC->getZExtValue()); if (Value *V = - SimplifyDemandedVectorElts(SrcVec, DemandedElts, UndefElts)) + SimplifyDemandedVectorElts(SrcVec, DemandedElts, PoisonElts)) return replaceOperand(EI, 0, V); } else { // If the input vector has multiple uses, simplify it based on a union // of all elements used. APInt DemandedElts = findDemandedEltsByAllUsers(SrcVec); if (!DemandedElts.isAllOnes()) { - APInt UndefElts(NumElts, 0); + APInt PoisonElts(NumElts, 0); if (Value *V = SimplifyDemandedVectorElts( - SrcVec, DemandedElts, UndefElts, 0 /* Depth */, + SrcVec, DemandedElts, PoisonElts, 0 /* Depth */, true /* AllowMultipleUsers */)) { if (V != SrcVec) { Worklist.addValue(SrcVec); @@ -777,10 +777,10 @@ static ShuffleOps collectShuffleElements(Value *V, SmallVectorImpl<int> &Mask, assert(V->getType()->isVectorTy() && "Invalid shuffle!"); unsigned NumElts = cast<FixedVectorType>(V->getType())->getNumElements(); - if (match(V, m_Undef())) { + if (match(V, m_Poison())) { Mask.assign(NumElts, -1); return std::make_pair( - PermittedRHS ? UndefValue::get(PermittedRHS->getType()) : V, nullptr); + PermittedRHS ? PoisonValue::get(PermittedRHS->getType()) : V, nullptr); } if (isa<ConstantAggregateZero>(V)) { @@ -1633,7 +1633,8 @@ Instruction *InstCombinerImpl::visitInsertElementInst(InsertElementInst &IE) { // bitcast (inselt undef, ScalarSrc, IdxOp) Type *ScalarTy = ScalarSrc->getType(); Type *VecTy = VectorType::get(ScalarTy, IE.getType()->getElementCount()); - UndefValue *NewUndef = UndefValue::get(VecTy); + Constant *NewUndef = isa<PoisonValue>(VecOp) ? PoisonValue::get(VecTy) + : UndefValue::get(VecTy); Value *NewInsElt = Builder.CreateInsertElement(NewUndef, ScalarSrc, IdxOp); return new BitCastInst(NewInsElt, IE.getType()); } @@ -1713,9 +1714,10 @@ Instruction *InstCombinerImpl::visitInsertElementInst(InsertElementInst &IE) { if (auto VecTy = dyn_cast<FixedVectorType>(VecOp->getType())) { unsigned VWidth = VecTy->getNumElements(); - APInt UndefElts(VWidth, 0); + APInt PoisonElts(VWidth, 0); APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); - if (Value *V = SimplifyDemandedVectorElts(&IE, AllOnesEltMask, UndefElts)) { + if (Value *V = SimplifyDemandedVectorElts(&IE, AllOnesEltMask, + PoisonElts)) { if (V != &IE) return replaceInstUsesWith(IE, V); return &IE; @@ -1918,6 +1920,10 @@ static Value *evaluateInDifferentElementOrder(Value *V, ArrayRef<int> Mask, assert(V->getType()->isVectorTy() && "can't reorder non-vector elements"); Type *EltTy = V->getType()->getScalarType(); + + if (isa<PoisonValue>(V)) + return PoisonValue::get(FixedVectorType::get(EltTy, Mask.size())); + if (match(V, m_Undef())) return UndefValue::get(FixedVectorType::get(EltTy, Mask.size())); @@ -2639,7 +2645,7 @@ static Instruction *foldShuffleWithInsert(ShuffleVectorInst &Shuf, assert(NewInsIndex != -1 && "Did not fold shuffle with unused operand?"); // Index is updated to the potentially translated insertion lane. - IndexC = ConstantInt::get(IndexC->getType(), NewInsIndex); + IndexC = ConstantInt::get(IndexC->getIntegerType(), NewInsIndex); return true; }; @@ -2769,6 +2775,11 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) { if (Instruction *I = simplifyBinOpSplats(SVI)) return I; + // Canonicalize splat shuffle to use poison RHS. Handle this explicitly in + // order to support scalable vectors. + if (match(SVI.getShuffleMask(), m_ZeroMask()) && !isa<PoisonValue>(RHS)) + return replaceOperand(SVI, 1, PoisonValue::get(RHS->getType())); + if (isa<ScalableVectorType>(LHS->getType())) return nullptr; @@ -2855,9 +2866,9 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) { if (Instruction *I = foldCastShuffle(SVI, Builder)) return I; - APInt UndefElts(VWidth, 0); + APInt PoisonElts(VWidth, 0); APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); - if (Value *V = SimplifyDemandedVectorElts(&SVI, AllOnesEltMask, UndefElts)) { + if (Value *V = SimplifyDemandedVectorElts(&SVI, AllOnesEltMask, PoisonElts)) { if (V != &SVI) return replaceInstUsesWith(SVI, V); return &SVI; @@ -3012,10 +3023,11 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) { ShuffleVectorInst* LHSShuffle = dyn_cast<ShuffleVectorInst>(LHS); ShuffleVectorInst* RHSShuffle = dyn_cast<ShuffleVectorInst>(RHS); if (LHSShuffle) - if (!match(LHSShuffle->getOperand(1), m_Undef()) && !match(RHS, m_Undef())) + if (!match(LHSShuffle->getOperand(1), m_Poison()) && + !match(RHS, m_Poison())) LHSShuffle = nullptr; if (RHSShuffle) - if (!match(RHSShuffle->getOperand(1), m_Undef())) + if (!match(RHSShuffle->getOperand(1), m_Poison())) RHSShuffle = nullptr; if (!LHSShuffle && !RHSShuffle) return MadeChange ? &SVI : nullptr; @@ -3038,7 +3050,7 @@ Instruction *InstCombinerImpl::visitShuffleVectorInst(ShuffleVectorInst &SVI) { Value* newRHS = RHS; if (LHSShuffle) { // case 1 - if (match(RHS, m_Undef())) { + if (match(RHS, m_Poison())) { newLHS = LHSOp0; newRHS = LHSOp1; } diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index a7ddadc25de4..7f5a7b666903 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -173,14 +173,14 @@ std::optional<Value *> InstCombiner::targetSimplifyDemandedUseBitsIntrinsic( } std::optional<Value *> InstCombiner::targetSimplifyDemandedVectorEltsIntrinsic( - IntrinsicInst &II, APInt DemandedElts, APInt &UndefElts, APInt &UndefElts2, - APInt &UndefElts3, + IntrinsicInst &II, APInt DemandedElts, APInt &PoisonElts, + APInt &PoisonElts2, APInt &PoisonElts3, std::function<void(Instruction *, unsigned, APInt, APInt &)> SimplifyAndSetOp) { // Handle target specific intrinsics if (II.getCalledFunction()->isTargetIntrinsic()) { return TTI.simplifyDemandedVectorEltsIntrinsic( - *this, II, DemandedElts, UndefElts, UndefElts2, UndefElts3, + *this, II, DemandedElts, PoisonElts, PoisonElts2, PoisonElts3, SimplifyAndSetOp); } return std::nullopt; @@ -1096,6 +1096,54 @@ Value *InstCombinerImpl::foldUsingDistributiveLaws(BinaryOperator &I) { return SimplifySelectsFeedingBinaryOp(I, LHS, RHS); } +std::optional<std::pair<Value *, Value *>> +InstCombinerImpl::matchSymmetricPhiNodesPair(PHINode *LHS, PHINode *RHS) { + if (LHS->getParent() != RHS->getParent()) + return std::nullopt; + + if (LHS->getNumIncomingValues() < 2) + return std::nullopt; + + if (!equal(LHS->blocks(), RHS->blocks())) + return std::nullopt; + + Value *L0 = LHS->getIncomingValue(0); + Value *R0 = RHS->getIncomingValue(0); + + for (unsigned I = 1, E = LHS->getNumIncomingValues(); I != E; ++I) { + Value *L1 = LHS->getIncomingValue(I); + Value *R1 = RHS->getIncomingValue(I); + + if ((L0 == L1 && R0 == R1) || (L0 == R1 && R0 == L1)) + continue; + + return std::nullopt; + } + + return std::optional(std::pair(L0, R0)); +} + +Value *InstCombinerImpl::SimplifyPhiCommutativeBinaryOp(BinaryOperator &I, + Value *Op0, + Value *Op1) { + assert(I.isCommutative() && "Instruction should be commutative"); + + PHINode *LHS = dyn_cast<PHINode>(Op0); + PHINode *RHS = dyn_cast<PHINode>(Op1); + + if (!LHS || !RHS) + return nullptr; + + if (auto P = matchSymmetricPhiNodesPair(LHS, RHS)) { + Value *BI = Builder.CreateBinOp(I.getOpcode(), P->first, P->second); + if (auto *BO = dyn_cast<BinaryOperator>(BI)) + BO->copyIRFlags(&I); + return BI; + } + + return nullptr; +} + Value *InstCombinerImpl::SimplifySelectsFeedingBinaryOp(BinaryOperator &I, Value *LHS, Value *RHS) { @@ -1529,6 +1577,11 @@ Instruction *InstCombinerImpl::foldBinopWithPhiOperands(BinaryOperator &BO) { BO.getParent() != Phi1->getParent()) return nullptr; + if (BO.isCommutative()) { + if (Value *V = SimplifyPhiCommutativeBinaryOp(BO, Phi0, Phi1)) + return replaceInstUsesWith(BO, V); + } + // Fold if there is at least one specific constant value in phi0 or phi1's // incoming values that comes from the same block and this specific constant // value can be used to do optimization for specific binary operator. @@ -1728,8 +1781,8 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { // If both arguments of the binary operation are shuffles that use the same // mask and shuffle within a single vector, move the shuffle after the binop. - if (match(LHS, m_Shuffle(m_Value(V1), m_Undef(), m_Mask(Mask))) && - match(RHS, m_Shuffle(m_Value(V2), m_Undef(), m_SpecificMask(Mask))) && + if (match(LHS, m_Shuffle(m_Value(V1), m_Poison(), m_Mask(Mask))) && + match(RHS, m_Shuffle(m_Value(V2), m_Poison(), m_SpecificMask(Mask))) && V1->getType() == V2->getType() && (LHS->hasOneUse() || RHS->hasOneUse() || LHS == RHS)) { // Op(shuffle(V1, Mask), shuffle(V2, Mask)) -> shuffle(Op(V1, V2), Mask) @@ -1770,9 +1823,9 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { Constant *C; auto *InstVTy = dyn_cast<FixedVectorType>(Inst.getType()); if (InstVTy && - match(&Inst, - m_c_BinOp(m_OneUse(m_Shuffle(m_Value(V1), m_Undef(), m_Mask(Mask))), - m_ImmConstant(C))) && + match(&Inst, m_c_BinOp(m_OneUse(m_Shuffle(m_Value(V1), m_Poison(), + m_Mask(Mask))), + m_ImmConstant(C))) && cast<FixedVectorType>(V1->getType())->getNumElements() <= InstVTy->getNumElements()) { assert(InstVTy->getScalarType() == V1->getType()->getScalarType() && @@ -1787,8 +1840,8 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { ArrayRef<int> ShMask = Mask; unsigned SrcVecNumElts = cast<FixedVectorType>(V1->getType())->getNumElements(); - UndefValue *UndefScalar = UndefValue::get(C->getType()->getScalarType()); - SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, UndefScalar); + PoisonValue *PoisonScalar = PoisonValue::get(C->getType()->getScalarType()); + SmallVector<Constant *, 16> NewVecC(SrcVecNumElts, PoisonScalar); bool MayChange = true; unsigned NumElts = InstVTy->getNumElements(); for (unsigned I = 0; I < NumElts; ++I) { @@ -1801,29 +1854,29 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { // 2. The shuffle needs an element of the constant vector that can't // be mapped to a new constant vector. // 3. This is a widening shuffle that copies elements of V1 into the - // extended elements (extending with undef is allowed). - if (!CElt || (!isa<UndefValue>(NewCElt) && NewCElt != CElt) || + // extended elements (extending with poison is allowed). + if (!CElt || (!isa<PoisonValue>(NewCElt) && NewCElt != CElt) || I >= SrcVecNumElts) { MayChange = false; break; } NewVecC[ShMask[I]] = CElt; } - // If this is a widening shuffle, we must be able to extend with undef - // elements. If the original binop does not produce an undef in the high + // If this is a widening shuffle, we must be able to extend with poison + // elements. If the original binop does not produce a poison in the high // lanes, then this transform is not safe. - // Similarly for undef lanes due to the shuffle mask, we can only - // transform binops that preserve undef. - // TODO: We could shuffle those non-undef constant values into the - // result by using a constant vector (rather than an undef vector) + // Similarly for poison lanes due to the shuffle mask, we can only + // transform binops that preserve poison. + // TODO: We could shuffle those non-poison constant values into the + // result by using a constant vector (rather than an poison vector) // as operand 1 of the new binop, but that might be too aggressive // for target-independent shuffle creation. if (I >= SrcVecNumElts || ShMask[I] < 0) { - Constant *MaybeUndef = + Constant *MaybePoison = ConstOp1 - ? ConstantFoldBinaryOpOperands(Opcode, UndefScalar, CElt, DL) - : ConstantFoldBinaryOpOperands(Opcode, CElt, UndefScalar, DL); - if (!MaybeUndef || !match(MaybeUndef, m_Undef())) { + ? ConstantFoldBinaryOpOperands(Opcode, PoisonScalar, CElt, DL) + : ConstantFoldBinaryOpOperands(Opcode, CElt, PoisonScalar, DL); + if (!MaybePoison || !isa<PoisonValue>(MaybePoison)) { MayChange = false; break; } @@ -1831,9 +1884,10 @@ Instruction *InstCombinerImpl::foldVectorBinop(BinaryOperator &Inst) { } if (MayChange) { Constant *NewC = ConstantVector::get(NewVecC); - // It may not be safe to execute a binop on a vector with undef elements + // It may not be safe to execute a binop on a vector with poison elements // because the entire instruction can be folded to undef or create poison // that did not exist in the original code. + // TODO: The shift case should not be necessary. if (Inst.isIntDivRem() || (Inst.isShift() && ConstOp1)) NewC = getSafeVectorConstantForBinop(Opcode, NewC, ConstOp1); @@ -2241,10 +2295,10 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) { // compile-time. if (auto *GEPFVTy = dyn_cast<FixedVectorType>(GEPType)) { auto VWidth = GEPFVTy->getNumElements(); - APInt UndefElts(VWidth, 0); + APInt PoisonElts(VWidth, 0); APInt AllOnesEltMask(APInt::getAllOnes(VWidth)); if (Value *V = SimplifyDemandedVectorElts(&GEP, AllOnesEltMask, - UndefElts)) { + PoisonElts)) { if (V != &GEP) return replaceInstUsesWith(GEP, V); return &GEP; @@ -2462,7 +2516,7 @@ Instruction *InstCombinerImpl::visitGetElementPtrInst(GetElementPtrInst &GEP) { Idx2); } ConstantInt *C; - if (match(GEP.getOperand(1), m_OneUse(m_SExt(m_OneUse(m_NSWAdd( + if (match(GEP.getOperand(1), m_OneUse(m_SExtLike(m_OneUse(m_NSWAdd( m_Value(Idx1), m_ConstantInt(C))))))) { // %add = add nsw i32 %idx1, idx2 // %sidx = sext i32 %add to i64 diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index 6468d07b4f4f..afb0e6cd1548 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -2737,7 +2737,7 @@ bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) { // the shadow memory. // We cannot just ignore these methods, because they may call other // instrumented functions. - if (F.getName().find(" load]") != std::string::npos) { + if (F.getName().contains(" load]")) { FunctionCallee AsanInitFunction = declareSanitizerInitFunction(*F.getParent(), kAsanInitName, {}); IRBuilder<> IRB(&F.front(), F.front().begin()); diff --git a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp index 539b7441d24b..2236e9cd44c5 100644 --- a/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemProfiler.cpp @@ -535,7 +535,7 @@ bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) { // the shadow memory. // We cannot just ignore these methods, because they may call other // instrumented functions. - if (F.getName().find(" load]") != std::string::npos) { + if (F.getName().contains(" load]")) { FunctionCallee MemProfInitFunction = declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {}); IRBuilder<> IRB(&F.front(), F.front().begin()); diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp index fe672a4377a1..ce570bdfd8b8 100644 --- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp +++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp @@ -603,7 +603,7 @@ void ModuleSanitizerCoverage::instrumentFunction( Function &F, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) { if (F.empty()) return; - if (F.getName().find(".module_ctor") != std::string::npos) + if (F.getName().contains(".module_ctor")) return; // Should not instrument sanitizer init functions. if (F.getName().starts_with("__sanitizer_")) return; // Don't instrument __sanitizer_* callbacks. diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp index 1fb9d7fff32f..9e40d94dd73c 100644 --- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp +++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp @@ -674,8 +674,7 @@ void ConstantHoistingPass::findBaseConstants(GlobalVariable *BaseGV) { llvm::stable_sort(ConstCandVec, [](const ConstantCandidate &LHS, const ConstantCandidate &RHS) { if (LHS.ConstInt->getType() != RHS.ConstInt->getType()) - return LHS.ConstInt->getType()->getBitWidth() < - RHS.ConstInt->getType()->getBitWidth(); + return LHS.ConstInt->getBitWidth() < RHS.ConstInt->getBitWidth(); return LHS.ConstInt->getValue().ult(RHS.ConstInt->getValue()); }); @@ -890,7 +889,7 @@ bool ConstantHoistingPass::emitBaseConstants(GlobalVariable *BaseGV) { Type *Ty = ConstInfo.BaseExpr->getType(); Base = new BitCastInst(ConstInfo.BaseExpr, Ty, "const", IP); } else { - IntegerType *Ty = ConstInfo.BaseInt->getType(); + IntegerType *Ty = ConstInfo.BaseInt->getIntegerType(); Base = new BitCastInst(ConstInfo.BaseInt, Ty, "const", IP); } diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp index 18266ba07898..899d7e0a11e6 100644 --- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp +++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp @@ -273,7 +273,16 @@ class ConstraintInfo { public: ConstraintInfo(const DataLayout &DL, ArrayRef<Value *> FunctionArgs) - : UnsignedCS(FunctionArgs), SignedCS(FunctionArgs), DL(DL) {} + : UnsignedCS(FunctionArgs), SignedCS(FunctionArgs), DL(DL) { + auto &Value2Index = getValue2Index(false); + // Add Arg > -1 constraints to unsigned system for all function arguments. + for (Value *Arg : FunctionArgs) { + ConstraintTy VarPos(SmallVector<int64_t, 8>(Value2Index.size() + 1, 0), + false, false, false); + VarPos.Coefficients[Value2Index[Arg]] = -1; + UnsignedCS.addVariableRow(VarPos.Coefficients); + } + } DenseMap<Value *, unsigned> &getValue2Index(bool Signed) { return Signed ? SignedCS.getValue2Index() : UnsignedCS.getValue2Index(); @@ -1365,18 +1374,34 @@ removeEntryFromStack(const StackEntry &E, ConstraintInfo &Info, ReproducerCondStack.pop_back(); } -/// Check if the first condition for an AND implies the second. -static bool checkAndSecondOpImpliedByFirst( +/// Check if either the first condition of an AND or OR is implied by the +/// (negated in case of OR) second condition or vice versa. +static bool checkOrAndOpImpliedByOther( FactOrCheck &CB, ConstraintInfo &Info, Module *ReproducerModule, SmallVectorImpl<ReproducerEntry> &ReproducerCondStack, SmallVectorImpl<StackEntry> &DFSInStack) { CmpInst::Predicate Pred; Value *A, *B; - Instruction *And = CB.getContextInst(); - if (!match(And->getOperand(0), m_ICmp(Pred, m_Value(A), m_Value(B)))) + Instruction *JoinOp = CB.getContextInst(); + CmpInst *CmpToCheck = cast<CmpInst>(CB.getInstructionToSimplify()); + unsigned OtherOpIdx = JoinOp->getOperand(0) == CmpToCheck ? 1 : 0; + + // Don't try to simplify the first condition of a select by the second, as + // this may make the select more poisonous than the original one. + // TODO: check if the first operand may be poison. + if (OtherOpIdx != 0 && isa<SelectInst>(JoinOp)) return false; + if (!match(JoinOp->getOperand(OtherOpIdx), + m_ICmp(Pred, m_Value(A), m_Value(B)))) + return false; + + // For OR, check if the negated condition implies CmpToCheck. + bool IsOr = match(JoinOp, m_LogicalOr()); + if (IsOr) + Pred = CmpInst::getInversePredicate(Pred); + // Optimistically add fact from first condition. unsigned OldSize = DFSInStack.size(); Info.addFact(Pred, A, B, CB.NumIn, CB.NumOut, DFSInStack); @@ -1385,11 +1410,19 @@ static bool checkAndSecondOpImpliedByFirst( bool Changed = false; // Check if the second condition can be simplified now. - ICmpInst *Cmp = cast<ICmpInst>(And->getOperand(1)); - if (auto ImpliedCondition = checkCondition( - Cmp->getPredicate(), Cmp->getOperand(0), Cmp->getOperand(1), Cmp, - Info, CB.NumIn, CB.NumOut, CB.getContextInst())) { - And->setOperand(1, ConstantInt::getBool(And->getType(), *ImpliedCondition)); + if (auto ImpliedCondition = + checkCondition(CmpToCheck->getPredicate(), CmpToCheck->getOperand(0), + CmpToCheck->getOperand(1), CmpToCheck, Info, CB.NumIn, + CB.NumOut, CB.getContextInst())) { + if (IsOr && isa<SelectInst>(JoinOp)) { + JoinOp->setOperand( + OtherOpIdx == 0 ? 2 : 0, + ConstantInt::getBool(JoinOp->getType(), *ImpliedCondition)); + } else + JoinOp->setOperand( + 1 - OtherOpIdx, + ConstantInt::getBool(JoinOp->getType(), *ImpliedCondition)); + Changed = true; } @@ -1442,6 +1475,17 @@ void ConstraintInfo::addFact(CmpInst::Predicate Pred, Value *A, Value *B, DFSInStack.emplace_back(NumIn, NumOut, R.IsSigned, std::move(ValuesToRelease)); + if (!R.IsSigned) { + for (Value *V : NewVariables) { + ConstraintTy VarPos(SmallVector<int64_t, 8>(Value2Index.size() + 1, 0), + false, false, false); + VarPos.Coefficients[Value2Index[V]] = -1; + CSToUse.addVariableRow(VarPos.Coefficients); + DFSInStack.emplace_back(NumIn, NumOut, R.IsSigned, + SmallVector<Value *, 2>()); + } + } + if (R.isEq()) { // Also add the inverted constraint for equality constraints. for (auto &Coeff : R.Coefficients) @@ -1609,11 +1653,11 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT, LoopInfo &LI, bool Simplified = checkAndReplaceCondition( Cmp, Info, CB.NumIn, CB.NumOut, CB.getContextInst(), ReproducerModule.get(), ReproducerCondStack, S.DT, ToRemove); - if (!Simplified && match(CB.getContextInst(), - m_LogicalAnd(m_Value(), m_Specific(Inst)))) { + if (!Simplified && + match(CB.getContextInst(), m_LogicalOp(m_Value(), m_Value()))) { Simplified = - checkAndSecondOpImpliedByFirst(CB, Info, ReproducerModule.get(), - ReproducerCondStack, DFSInStack); + checkOrAndOpImpliedByOther(CB, Info, ReproducerModule.get(), + ReproducerCondStack, DFSInStack); } Changed |= Simplified; } @@ -1687,7 +1731,8 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT, LoopInfo &LI, #ifndef NDEBUG unsigned SignedEntries = count_if(DFSInStack, [](const StackEntry &E) { return E.IsSigned; }); - assert(Info.getCS(false).size() == DFSInStack.size() - SignedEntries && + assert(Info.getCS(false).size() - FunctionArgs.size() == + DFSInStack.size() - SignedEntries && "updates to CS and DFSInStack are out of sync"); assert(Info.getCS(true).size() == SignedEntries && "updates to CS and DFSInStack are out of sync"); diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp index d2dfc764d042..c44d3748a80d 100644 --- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp +++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp @@ -935,11 +935,13 @@ static bool processSDiv(BinaryOperator *SDI, const ConstantRange &LCR, UDiv->setDebugLoc(SDI->getDebugLoc()); UDiv->setIsExact(SDI->isExact()); - Value *Res = UDiv; + auto *Res = UDiv; // If the operands had two different domains, we need to negate the result. - if (Ops[0].D != Ops[1].D) + if (Ops[0].D != Ops[1].D) { Res = BinaryOperator::CreateNeg(Res, Res->getName() + ".neg", SDI); + Res->setDebugLoc(SDI->getDebugLoc()); + } SDI->replaceAllUsesWith(Res); SDI->eraseFromParent(); diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp index edfeb36f3422..c5bf913cda30 100644 --- a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp +++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp @@ -521,7 +521,7 @@ struct AllSwitchPaths { const BasicBlock *PrevBB = Path.back(); for (const BasicBlock *BB : Path) { - if (StateDef.count(BB) != 0) { + if (StateDef.contains(BB)) { const PHINode *Phi = dyn_cast<PHINode>(StateDef[BB]); assert(Phi && "Expected a state-defining instr to be a phi node."); diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp index 5e58af0edc15..e36578f3de7a 100644 --- a/llvm/lib/Transforms/Scalar/GVN.cpp +++ b/llvm/lib/Transforms/Scalar/GVN.cpp @@ -592,7 +592,7 @@ uint32_t GVNPass::ValueTable::lookupOrAddCall(CallInst *C) { /// Returns true if a value number exists for the specified value. bool GVNPass::ValueTable::exists(Value *V) const { - return valueNumbering.count(V) != 0; + return valueNumbering.contains(V); } /// lookup_or_add - Returns the value number for the specified value, assigning diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp index b1add3c42976..eef94636578d 100644 --- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp +++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp @@ -343,9 +343,8 @@ static bool verifyTripCount(Value *RHS, Loop *L, // If the RHS of the compare is equal to the backedge taken count we need // to add one to get the trip count. if (SCEVRHS == BackedgeTCExt || SCEVRHS == BackedgeTakenCount) { - ConstantInt *One = ConstantInt::get(ConstantRHS->getType(), 1); - Value *NewRHS = ConstantInt::get( - ConstantRHS->getContext(), ConstantRHS->getValue() + One->getValue()); + Value *NewRHS = ConstantInt::get(ConstantRHS->getContext(), + ConstantRHS->getValue() + 1); return setLoopComponents(NewRHS, TripCount, Increment, IterationInstructions); } diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp index 39607464dd00..a58bbe318563 100644 --- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp +++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp @@ -7006,7 +7006,7 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE, LLVM_DEBUG(dbgs() << "Old term-cond:\n" << *OldTermCond << "\n" - << "New term-cond:\b" << *NewTermCond << "\n"); + << "New term-cond:\n" << *NewTermCond << "\n"); BI->setCondition(NewTermCond); diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp index f14541a1a037..7cfeb019af97 100644 --- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -200,6 +200,7 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences( UP.Count = 0; UP.DefaultUnrollRuntimeCount = 8; UP.MaxCount = std::numeric_limits<unsigned>::max(); + UP.MaxUpperBound = UnrollMaxUpperBound; UP.FullUnrollMaxCount = std::numeric_limits<unsigned>::max(); UP.BEInsns = 2; UP.Partial = false; @@ -237,6 +238,8 @@ TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences( UP.MaxPercentThresholdBoost = UnrollMaxPercentThresholdBoost; if (UnrollMaxCount.getNumOccurrences() > 0) UP.MaxCount = UnrollMaxCount; + if (UnrollMaxUpperBound.getNumOccurrences() > 0) + UP.MaxUpperBound = UnrollMaxUpperBound; if (UnrollFullMaxCount.getNumOccurrences() > 0) UP.FullUnrollMaxCount = UnrollFullMaxCount; if (UnrollAllowPartial.getNumOccurrences() > 0) @@ -777,7 +780,7 @@ shouldPragmaUnroll(Loop *L, const PragmaInfo &PInfo, return TripCount; if (PInfo.PragmaEnableUnroll && !TripCount && MaxTripCount && - MaxTripCount <= UnrollMaxUpperBound) + MaxTripCount <= UP.MaxUpperBound) return MaxTripCount; // if didn't return until here, should continue to other priorties @@ -952,7 +955,7 @@ bool llvm::computeUnrollCount( // cost of exact full unrolling. As such, if we have an exact count and // found it unprofitable, we'll never chose to bounded unroll. if (!TripCount && MaxTripCount && (UP.UpperBound || MaxOrZero) && - MaxTripCount <= UnrollMaxUpperBound) { + MaxTripCount <= UP.MaxUpperBound) { UP.Count = MaxTripCount; if (auto UnrollFactor = shouldFullUnroll(L, TTI, DT, SE, EphValues, MaxTripCount, UCE, UP)) { @@ -1026,7 +1029,7 @@ bool llvm::computeUnrollCount( } // Don't unroll a small upper bound loop unless user or TTI asked to do so. - if (MaxTripCount && !UP.Force && MaxTripCount < UnrollMaxUpperBound) { + if (MaxTripCount && !UP.Force && MaxTripCount < UP.MaxUpperBound) { UP.Count = 0; return false; } diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp index 40b4ea92e1ff..3f02441b74ba 100644 --- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp +++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp @@ -2057,7 +2057,7 @@ static void relocationViaAlloca( for (const auto &Info : Records) for (auto RematerializedValuePair : Info.RematerializedValues) { Value *OriginalValue = RematerializedValuePair.second; - if (AllocaMap.count(OriginalValue) != 0) + if (AllocaMap.contains(OriginalValue)) continue; emitAllocaFor(OriginalValue); diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp index 24da26c9f0f2..656abdb0abbf 100644 --- a/llvm/lib/Transforms/Scalar/SROA.cpp +++ b/llvm/lib/Transforms/Scalar/SROA.cpp @@ -3285,6 +3285,7 @@ private: (BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset || SliceSize != DL.getTypeStoreSize(NewAI.getAllocatedType()).getFixedValue() || + !DL.typeSizeEqualsStoreSize(NewAI.getAllocatedType()) || !NewAI.getAllocatedType()->isSingleValueType()); // If we're just going to emit a memcpy, the alloca hasn't changed, and the diff --git a/llvm/lib/Transforms/Utils/DXILUpgrade.cpp b/llvm/lib/Transforms/Utils/DXILUpgrade.cpp index 735686ddce38..09991f628224 100644 --- a/llvm/lib/Transforms/Utils/DXILUpgrade.cpp +++ b/llvm/lib/Transforms/Utils/DXILUpgrade.cpp @@ -7,14 +7,26 @@ //===----------------------------------------------------------------------===// #include "llvm/Transforms/Utils/DXILUpgrade.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/Metadata.h" +#include "llvm/IR/Module.h" +#include "llvm/Support/Debug.h" using namespace llvm; +#define DEBUG_TYPE "dxil-upgrade" + static bool handleValVerMetadata(Module &M) { NamedMDNode *ValVer = M.getNamedMetadata("dx.valver"); if (!ValVer) return false; + LLVM_DEBUG({ + MDNode *N = ValVer->getOperand(0); + auto X = mdconst::extract<ConstantInt>(N->getOperand(0))->getZExtValue(); + auto Y = mdconst::extract<ConstantInt>(N->getOperand(1))->getZExtValue(); + dbgs() << "DXIL: validation version: " << X << "." << Y << "\n"; + }); // We don't need the validation version internally, so we drop it. ValVer->dropAllReferences(); ValVer->eraseFromParent(); diff --git a/llvm/lib/Transforms/Utils/FunctionComparator.cpp b/llvm/lib/Transforms/Utils/FunctionComparator.cpp index 79ca99d1566c..09e19be0d293 100644 --- a/llvm/lib/Transforms/Utils/FunctionComparator.cpp +++ b/llvm/lib/Transforms/Utils/FunctionComparator.cpp @@ -405,6 +405,8 @@ int FunctionComparator::cmpConstants(const Constant *L, case Value::ConstantExprVal: { const ConstantExpr *LE = cast<ConstantExpr>(L); const ConstantExpr *RE = cast<ConstantExpr>(R); + if (int Res = cmpNumbers(LE->getOpcode(), RE->getOpcode())) + return Res; unsigned NumOperandsL = LE->getNumOperands(); unsigned NumOperandsR = RE->getNumOperands(); if (int Res = cmpNumbers(NumOperandsL, NumOperandsR)) @@ -414,6 +416,29 @@ int FunctionComparator::cmpConstants(const Constant *L, cast<Constant>(RE->getOperand(i)))) return Res; } + if (LE->isCompare()) + if (int Res = cmpNumbers(LE->getPredicate(), RE->getPredicate())) + return Res; + if (auto *GEPL = dyn_cast<GEPOperator>(LE)) { + auto *GEPR = cast<GEPOperator>(RE); + if (int Res = cmpTypes(GEPL->getSourceElementType(), + GEPR->getSourceElementType())) + return Res; + if (int Res = cmpNumbers(GEPL->isInBounds(), GEPR->isInBounds())) + return Res; + if (int Res = cmpNumbers(GEPL->getInRangeIndex().value_or(unsigned(-1)), + GEPR->getInRangeIndex().value_or(unsigned(-1)))) + return Res; + } + if (auto *OBOL = dyn_cast<OverflowingBinaryOperator>(LE)) { + auto *OBOR = cast<OverflowingBinaryOperator>(RE); + if (int Res = + cmpNumbers(OBOL->hasNoUnsignedWrap(), OBOR->hasNoUnsignedWrap())) + return Res; + if (int Res = + cmpNumbers(OBOL->hasNoSignedWrap(), OBOR->hasNoSignedWrap())) + return Res; + } return 0; } case Value::BlockAddressVal: { diff --git a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp index 1e42d7491676..f94047633022 100644 --- a/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp +++ b/llvm/lib/Transforms/Utils/MemoryTaggingSupport.cpp @@ -64,7 +64,7 @@ bool forAllReachableExits(const DominatorTree &DT, const PostDominatorTree &PDT, // sure that the return is covered. Otherwise, we can check whether there // is a way to reach the RI from the start of the lifetime without passing // through an end. - if (EndBlocks.count(RI->getParent()) > 0 || + if (EndBlocks.contains(RI->getParent()) || !isPotentiallyReachable(Start, RI, &EndBlocks, &DT, &LI)) { ++NumCoveredExits; } diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 89494a7f6497..55e375670cc6 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -6293,7 +6293,7 @@ Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) { } case BitMapKind: { // Type of the bitmap (e.g. i59). - IntegerType *MapTy = BitMap->getType(); + IntegerType *MapTy = BitMap->getIntegerType(); // Cast Index to the same type as the bitmap. // Note: The Index is <= the number of elements in the table, so @@ -6668,7 +6668,7 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder, Value *TableIndex; ConstantInt *TableIndexOffset; if (UseSwitchConditionAsTableIndex) { - TableIndexOffset = ConstantInt::get(MaxCaseVal->getType(), 0); + TableIndexOffset = ConstantInt::get(MaxCaseVal->getIntegerType(), 0); TableIndex = SI->getCondition(); } else { TableIndexOffset = MinCaseVal; @@ -6752,7 +6752,7 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder, // Get the TableIndex'th bit of the bitmask. // If this bit is 0 (meaning hole) jump to the default destination, // else continue with table lookup. - IntegerType *MapTy = TableMask->getType(); + IntegerType *MapTy = TableMask->getIntegerType(); Value *MaskIndex = Builder.CreateZExtOrTrunc(TableIndex, MapTy, "switch.maskindex"); Value *Shifted = Builder.CreateLShr(TableMask, MaskIndex, "switch.shifted"); @@ -6975,7 +6975,7 @@ static bool simplifySwitchOfPowersOfTwo(SwitchInst *SI, IRBuilder<> &Builder, // Replace each case with its trailing zeros number. for (auto &Case : SI->cases()) { auto *OrigValue = Case.getCaseValue(); - Case.setValue(ConstantInt::get(OrigValue->getType(), + Case.setValue(ConstantInt::get(OrigValue->getIntegerType(), OrigValue->getValue().countr_zero())); } diff --git a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp index 722ed03db3de..42e7c4006b42 100644 --- a/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyIndVar.cpp @@ -27,6 +27,7 @@ #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h" using namespace llvm; +using namespace llvm::PatternMatch; #define DEBUG_TYPE "indvars" @@ -786,8 +787,6 @@ bool SimplifyIndvar::strengthenOverflowingOperation(BinaryOperator *BO, /// otherwise. bool SimplifyIndvar::strengthenRightShift(BinaryOperator *BO, Instruction *IVOperand) { - using namespace llvm::PatternMatch; - if (BO->getOpcode() == Instruction::Shl) { bool Changed = false; ConstantRange IVRange = SE->getUnsignedRange(SE->getSCEV(IVOperand)); @@ -1763,7 +1762,7 @@ Instruction *WidenIV::widenIVUse(WidenIV::NarrowIVDefUse DU, SCEVExpander &Rewri }; // Our raison d'etre! Eliminate sign and zero extension. - if ((isa<SExtInst>(DU.NarrowUse) && canWidenBySExt()) || + if ((match(DU.NarrowUse, m_SExtLike(m_Value())) && canWidenBySExt()) || (isa<ZExtInst>(DU.NarrowUse) && canWidenByZExt())) { Value *NewDef = DU.WideDef; if (DU.NarrowUse->getType() != WideType) { @@ -2011,8 +2010,6 @@ PHINode *WidenIV::createWideIV(SCEVExpander &Rewriter) { /// by looking at dominating conditions inside of the loop void WidenIV::calculatePostIncRange(Instruction *NarrowDef, Instruction *NarrowUser) { - using namespace llvm::PatternMatch; - Value *NarrowDefLHS; const APInt *NarrowDefRHS; if (!match(NarrowDef, m_NSWAdd(m_Value(NarrowDefLHS), diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 9d799124074c..32913b3f5569 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -3760,40 +3760,7 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) { OrdersType CurrentOrder(NumScalars, NumScalars); SmallVector<int> Positions; SmallBitVector UsedPositions(NumScalars); - DenseMap<const TreeEntry *, unsigned> UsedEntries; - DenseMap<Value *, std::pair<const TreeEntry *, unsigned>> ValueToEntryPos; - for (Value *V : TE.Scalars) { - if (!isa<LoadInst, ExtractElementInst, ExtractValueInst>(V)) - continue; - const auto *LocalSTE = getTreeEntry(V); - if (!LocalSTE) - continue; - unsigned Lane = - std::distance(LocalSTE->Scalars.begin(), find(LocalSTE->Scalars, V)); - if (Lane >= NumScalars) - continue; - ++UsedEntries.try_emplace(LocalSTE, 0).first->getSecond(); - ValueToEntryPos.try_emplace(V, LocalSTE, Lane); - } - if (UsedEntries.empty()) - return std::nullopt; - const TreeEntry &BestSTE = - *std::max_element(UsedEntries.begin(), UsedEntries.end(), - [](const std::pair<const TreeEntry *, unsigned> &P1, - const std::pair<const TreeEntry *, unsigned> &P2) { - return P1.second < P2.second; - }) - ->first; - UsedEntries.erase(&BestSTE); - const TreeEntry *SecondBestSTE = nullptr; - if (!UsedEntries.empty()) - SecondBestSTE = - std::max_element(UsedEntries.begin(), UsedEntries.end(), - [](const std::pair<const TreeEntry *, unsigned> &P1, - const std::pair<const TreeEntry *, unsigned> &P2) { - return P1.second < P2.second; - }) - ->first; + const TreeEntry *STE = nullptr; // Try to find all gathered scalars that are gets vectorized in other // vectorize node. Here we can have only one single tree vector node to // correctly identify order of the gathered scalars. @@ -3801,46 +3768,53 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) { Value *V = TE.Scalars[I]; if (!isa<LoadInst, ExtractElementInst, ExtractValueInst>(V)) continue; - const auto [LocalSTE, Lane] = ValueToEntryPos.lookup(V); - if (!LocalSTE || (LocalSTE != &BestSTE && LocalSTE != SecondBestSTE)) - continue; - if (CurrentOrder[Lane] != NumScalars) { - if ((CurrentOrder[Lane] >= BestSTE.Scalars.size() || - BestSTE.Scalars[CurrentOrder[Lane]] == V) && - (Lane != I || LocalSTE == SecondBestSTE)) - continue; - UsedPositions.reset(CurrentOrder[Lane]); + if (const auto *LocalSTE = getTreeEntry(V)) { + if (!STE) + STE = LocalSTE; + else if (STE != LocalSTE) + // Take the order only from the single vector node. + return std::nullopt; + unsigned Lane = + std::distance(STE->Scalars.begin(), find(STE->Scalars, V)); + if (Lane >= NumScalars) + return std::nullopt; + if (CurrentOrder[Lane] != NumScalars) { + if (Lane != I) + continue; + UsedPositions.reset(CurrentOrder[Lane]); + } + // The partial identity (where only some elements of the gather node are + // in the identity order) is good. + CurrentOrder[Lane] = I; + UsedPositions.set(I); } - // The partial identity (where only some elements of the gather node are - // in the identity order) is good. - CurrentOrder[Lane] = I; - UsedPositions.set(I); } // Need to keep the order if we have a vector entry and at least 2 scalars or // the vectorized entry has just 2 scalars. - if (BestSTE.Scalars.size() != 2 && UsedPositions.count() <= 1) - return std::nullopt; - auto IsIdentityOrder = [&](ArrayRef<unsigned> CurrentOrder) { - for (unsigned I = 0; I < NumScalars; ++I) - if (CurrentOrder[I] != I && CurrentOrder[I] != NumScalars) - return false; - return true; - }; - if (IsIdentityOrder(CurrentOrder)) - return OrdersType(); - auto *It = CurrentOrder.begin(); - for (unsigned I = 0; I < NumScalars;) { - if (UsedPositions.test(I)) { - ++I; - continue; - } - if (*It == NumScalars) { - *It = I; - ++I; + if (STE && (UsedPositions.count() > 1 || STE->Scalars.size() == 2)) { + auto &&IsIdentityOrder = [NumScalars](ArrayRef<unsigned> CurrentOrder) { + for (unsigned I = 0; I < NumScalars; ++I) + if (CurrentOrder[I] != I && CurrentOrder[I] != NumScalars) + return false; + return true; + }; + if (IsIdentityOrder(CurrentOrder)) + return OrdersType(); + auto *It = CurrentOrder.begin(); + for (unsigned I = 0; I < NumScalars;) { + if (UsedPositions.test(I)) { + ++I; + continue; + } + if (*It == NumScalars) { + *It = I; + ++I; + } + ++It; } - ++It; + return std::move(CurrentOrder); } - return std::move(CurrentOrder); + return std::nullopt; } namespace { @@ -6469,7 +6443,7 @@ bool BoUpSLP::areAllUsersVectorized( Instruction *I, const SmallDenseSet<Value *> *VectorizedVals) const { return (I->hasOneUse() && (!VectorizedVals || VectorizedVals->contains(I))) || all_of(I->users(), [this](User *U) { - return ScalarToTreeEntry.count(U) > 0 || + return ScalarToTreeEntry.contains(U) || isVectorLikeInstWithConstOps(U) || (isa<ExtractElementInst>(U) && MustGather.contains(U)); }); @@ -11498,7 +11472,7 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E, bool PostponedPHIs) { Value *V = Builder.CreateBinOp( static_cast<Instruction::BinaryOps>(E->getOpcode()), LHS, RHS); - propagateIRFlags(V, E->Scalars, VL0); + propagateIRFlags(V, E->Scalars, VL0, !MinBWs.contains(E)); if (auto *I = dyn_cast<Instruction>(V)) V = propagateMetadata(I, E->Scalars); @@ -15730,6 +15704,8 @@ static bool compareCmp(Value *V, Value *V2, TargetLibraryInfo &TLI, assert(isValidElementType(V->getType()) && isValidElementType(V2->getType()) && "Expected valid element types only."); + if (V == V2) + return IsCompatibility; auto *CI1 = cast<CmpInst>(V); auto *CI2 = cast<CmpInst>(V2); if (CI1->getOperand(0)->getType()->getTypeID() < @@ -15754,6 +15730,8 @@ static bool compareCmp(Value *V, Value *V2, TargetLibraryInfo &TLI, for (int I = 0, E = CI1->getNumOperands(); I < E; ++I) { auto *Op1 = CI1->getOperand(CI1Preds ? I : E - I - 1); auto *Op2 = CI2->getOperand(CI2Preds ? I : E - I - 1); + if (Op1 == Op2) + continue; if (Op1->getValueID() < Op2->getValueID()) return !IsCompatibility; if (Op1->getValueID() > Op2->getValueID()) @@ -15780,7 +15758,10 @@ static bool compareCmp(Value *V, Value *V2, TargetLibraryInfo &TLI, InstructionsState S = getSameOpcode({I1, I2}, TLI); if (S.getOpcode() && (IsCompatibility || !S.isAltShuffle())) continue; - return !IsCompatibility && I1->getOpcode() < I2->getOpcode(); + if (IsCompatibility) + return false; + if (I1->getOpcode() != I2->getOpcode()) + return I1->getOpcode() < I2->getOpcode(); } } return IsCompatibility; |