From ab50317e96e57dee5b3ff4ad3f16f205b2a3359e Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Wed, 24 Jan 2024 20:17:23 +0100 Subject: Merge llvm-project main llvmorg-18-init-18359-g93248729cfae This updates llvm, clang, compiler-rt, libc++, libunwind, lld, lldb and openmp to llvm-project main llvmorg-18-init-18359-g93248729cfae, the last commit before the upstream release/18.x branch was created. PR: 276104 MFC after: 1 month (cherry picked from commit 7a6dacaca14b62ca4b74406814becb87a3fefac0) --- .../lib/Transforms/Vectorize/VPlanTransforms.cpp | 74 ++++++++++++---------- 1 file changed, 41 insertions(+), 33 deletions(-) (limited to 'contrib/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp') diff --git a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 5c430620a2dc..8e6b48cdb2c8 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -74,9 +74,9 @@ void VPlanTransforms::VPInstructionsToVPRecipes( } else if (GetElementPtrInst *GEP = dyn_cast(Inst)) { NewRecipe = new VPWidenGEPRecipe(GEP, Ingredient.operands()); } else if (CallInst *CI = dyn_cast(Inst)) { - NewRecipe = - new VPWidenCallRecipe(*CI, drop_end(Ingredient.operands()), - getVectorIntrinsicIDForCall(CI, &TLI)); + NewRecipe = new VPWidenCallRecipe( + *CI, drop_end(Ingredient.operands()), + getVectorIntrinsicIDForCall(CI, &TLI), CI->getDebugLoc()); } else if (SelectInst *SI = dyn_cast(Inst)) { NewRecipe = new VPWidenSelectRecipe(*SI, Ingredient.operands()); } else if (auto *CI = dyn_cast(Inst)) { @@ -103,7 +103,7 @@ static bool sinkScalarOperands(VPlan &Plan) { bool Changed = false; // First, collect the operands of all recipes in replicate blocks as seeds for // sinking. - SetVector> WorkList; + SetVector> WorkList; for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly(Iter)) { VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock(); if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2) @@ -113,7 +113,8 @@ static bool sinkScalarOperands(VPlan &Plan) { continue; for (auto &Recipe : *VPBB) { for (VPValue *Op : Recipe.operands()) - if (auto *Def = Op->getDefiningRecipe()) + if (auto *Def = + dyn_cast_or_null(Op->getDefiningRecipe())) WorkList.insert(std::make_pair(VPBB, Def)); } } @@ -122,7 +123,7 @@ static bool sinkScalarOperands(VPlan &Plan) { // Try to sink each replicate or scalar IV steps recipe in the worklist. for (unsigned I = 0; I != WorkList.size(); ++I) { VPBasicBlock *SinkTo; - VPRecipeBase *SinkCandidate; + VPSingleDefRecipe *SinkCandidate; std::tie(SinkTo, SinkCandidate) = WorkList[I]; if (SinkCandidate->getParent() == SinkTo || SinkCandidate->mayHaveSideEffects() || @@ -146,12 +147,11 @@ static bool sinkScalarOperands(VPlan &Plan) { return false; if (UI->getParent() == SinkTo) return true; - NeedsDuplicating = - UI->onlyFirstLaneUsed(SinkCandidate->getVPSingleValue()); + NeedsDuplicating = UI->onlyFirstLaneUsed(SinkCandidate); // We only know how to duplicate VPRecipeRecipes for now. return NeedsDuplicating && isa(SinkCandidate); }; - if (!all_of(SinkCandidate->getVPSingleValue()->users(), CanSinkWithUser)) + if (!all_of(SinkCandidate->users(), CanSinkWithUser)) continue; if (NeedsDuplicating) { @@ -163,14 +163,14 @@ static bool sinkScalarOperands(VPlan &Plan) { // TODO: add ".cloned" suffix to name of Clone's VPValue. Clone->insertBefore(SinkCandidate); - SinkCandidate->getVPSingleValue()->replaceUsesWithIf( - Clone, [SinkTo](VPUser &U, unsigned) { - return cast(&U)->getParent() != SinkTo; - }); + SinkCandidate->replaceUsesWithIf(Clone, [SinkTo](VPUser &U, unsigned) { + return cast(&U)->getParent() != SinkTo; + }); } SinkCandidate->moveBefore(*SinkTo, SinkTo->getFirstNonPhi()); for (VPValue *Op : SinkCandidate->operands()) - if (auto *Def = Op->getDefiningRecipe()) + if (auto *Def = + dyn_cast_or_null(Op->getDefiningRecipe())) WorkList.insert(std::make_pair(SinkTo, Def)); Changed = true; } @@ -412,16 +412,15 @@ void VPlanTransforms::removeRedundantInductionCasts(VPlan &Plan) { auto &Casts = IV->getInductionDescriptor().getCastInsts(); VPValue *FindMyCast = IV; for (Instruction *IRCast : reverse(Casts)) { - VPRecipeBase *FoundUserCast = nullptr; + VPSingleDefRecipe *FoundUserCast = nullptr; for (auto *U : FindMyCast->users()) { - auto *UserCast = cast(U); - if (UserCast->getNumDefinedValues() == 1 && - UserCast->getVPSingleValue()->getUnderlyingValue() == IRCast) { + auto *UserCast = dyn_cast(U); + if (UserCast && UserCast->getUnderlyingValue() == IRCast) { FoundUserCast = UserCast; break; } } - FindMyCast = FoundUserCast->getVPSingleValue(); + FindMyCast = FoundUserCast; } FindMyCast->replaceAllUsesWith(IV); } @@ -895,7 +894,10 @@ void VPlanTransforms::truncateToMinimalBitwidths( vp_depth_first_deep(Plan.getVectorLoopRegion()))) { for (VPRecipeBase &R : make_early_inc_range(*VPBB)) { if (!isa(&R)) + VPWidenSelectRecipe, VPWidenMemoryInstructionRecipe>(&R)) + continue; + if (isa(&R) && + cast(&R)->isStore()) continue; VPValue *ResultVPV = R.getVPSingleValue(); @@ -948,6 +950,23 @@ void VPlanTransforms::truncateToMinimalBitwidths( auto *NewResTy = IntegerType::get(Ctx, NewResSizeInBits); + // Any wrapping introduced by shrinking this operation shouldn't be + // considered undefined behavior. So, we can't unconditionally copy + // arithmetic wrapping flags to VPW. + if (auto *VPW = dyn_cast(&R)) + VPW->dropPoisonGeneratingFlags(); + + // Extend result to original width. + auto *Ext = new VPWidenCastRecipe(Instruction::ZExt, ResultVPV, OldResTy); + Ext->insertAfter(&R); + ResultVPV->replaceAllUsesWith(Ext); + Ext->setOperand(0, ResultVPV); + + if (isa(&R)) { + assert(!cast(&R)->isStore() && "stores cannot be narrowed"); + continue; + } + // Shrink operands by introducing truncates as needed. unsigned StartIdx = isa(&R) ? 1 : 0; for (unsigned Idx = StartIdx; Idx != R.getNumOperands(); ++Idx) { @@ -979,17 +998,6 @@ void VPlanTransforms::truncateToMinimalBitwidths( } } - // Any wrapping introduced by shrinking this operation shouldn't be - // considered undefined behavior. So, we can't unconditionally copy - // arithmetic wrapping flags to VPW. - if (auto *VPW = dyn_cast(&R)) - VPW->dropPoisonGeneratingFlags(); - - // Extend result to original width. - auto *Ext = new VPWidenCastRecipe(Instruction::ZExt, ResultVPV, OldResTy); - Ext->insertAfter(&R); - ResultVPV->replaceAllUsesWith(Ext); - Ext->setOperand(0, ResultVPV); } } @@ -1130,7 +1138,7 @@ void VPlanTransforms::addActiveLaneMask( "Must have widened canonical IV when tail folding!"); auto *WideCanonicalIV = cast(*FoundWidenCanonicalIVUser); - VPRecipeBase *LaneMask; + VPSingleDefRecipe *LaneMask; if (UseActiveLaneMaskForControlFlow) { LaneMask = addVPLaneMaskPhiAndUpdateExitBranch( Plan, DataAndControlFlowWithoutRuntimeCheck); @@ -1155,7 +1163,7 @@ void VPlanTransforms::addActiveLaneMask( assert(CompareToReplace->getOperand(0) == WideCanonicalIV && "WidenCanonicalIV must be the first operand of the compare"); - CompareToReplace->replaceAllUsesWith(LaneMask->getVPSingleValue()); + CompareToReplace->replaceAllUsesWith(LaneMask); CompareToReplace->eraseFromParent(); } } -- cgit v1.2.3