aboutsummaryrefslogtreecommitdiff
path: root/contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp')
-rw-r--r--contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp301
1 files changed, 182 insertions, 119 deletions
diff --git a/contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp b/contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp
index 75cddfa16d6d..bdbaf4f55c96 100644
--- a/contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/contrib/llvm-project/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -319,6 +319,29 @@ static DebugVariable getAggregateVariable(DbgVariableIntrinsic *DVI) {
return DebugVariable(DVI->getVariable(), std::nullopt,
DVI->getDebugLoc().getInlinedAt());
}
+static DebugVariable getAggregateVariable(DPValue *DPV) {
+ return DebugVariable(DPV->getVariable(), std::nullopt,
+ DPV->getDebugLoc().getInlinedAt());
+}
+
+static DPValue *createLinkedAssign(DPValue *, DIBuilder &DIB,
+ Instruction *LinkedInstr, Value *NewValue,
+ DILocalVariable *Variable,
+ DIExpression *Expression, Value *Address,
+ DIExpression *AddressExpression,
+ const DILocation *DI) {
+ (void)DIB;
+ return DPValue::createLinkedDPVAssign(LinkedInstr, NewValue, Variable,
+ Expression, Address, AddressExpression,
+ DI);
+}
+static DbgAssignIntrinsic *createLinkedAssign(
+ DbgAssignIntrinsic *, DIBuilder &DIB, Instruction *LinkedInstr,
+ Value *NewValue, DILocalVariable *Variable, DIExpression *Expression,
+ Value *Address, DIExpression *AddressExpression, const DILocation *DI) {
+ return DIB.insertDbgAssign(LinkedInstr, NewValue, Variable, Expression,
+ Address, AddressExpression, DI);
+}
/// Find linked dbg.assign and generate a new one with the correct
/// FragmentInfo. Link Inst to the new dbg.assign. If Value is nullptr the
@@ -340,8 +363,9 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
Instruction *Inst, Value *Dest, Value *Value,
const DataLayout &DL) {
auto MarkerRange = at::getAssignmentMarkers(OldInst);
+ auto DPVAssignMarkerRange = at::getDPVAssignmentMarkers(OldInst);
// Nothing to do if OldInst has no linked dbg.assign intrinsics.
- if (MarkerRange.empty())
+ if (MarkerRange.empty() && DPVAssignMarkerRange.empty())
return;
LLVM_DEBUG(dbgs() << " migrateDebugInfo\n");
@@ -362,6 +386,9 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
for (auto *DAI : at::getAssignmentMarkers(OldAlloca))
BaseFragments[getAggregateVariable(DAI)] =
DAI->getExpression()->getFragmentInfo();
+ for (auto *DPV : at::getDPVAssignmentMarkers(OldAlloca))
+ BaseFragments[getAggregateVariable(DPV)] =
+ DPV->getExpression()->getFragmentInfo();
// The new inst needs a DIAssignID unique metadata tag (if OldInst has
// one). It shouldn't already have one: assert this assumption.
@@ -371,7 +398,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
DIBuilder DIB(*OldInst->getModule(), /*AllowUnresolved*/ false);
assert(OldAlloca->isStaticAlloca());
- for (DbgAssignIntrinsic *DbgAssign : MarkerRange) {
+ auto MigrateDbgAssign = [&](auto DbgAssign) {
LLVM_DEBUG(dbgs() << " existing dbg.assign is: " << *DbgAssign
<< "\n");
auto *Expr = DbgAssign->getExpression();
@@ -382,7 +409,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
{
auto R = BaseFragments.find(getAggregateVariable(DbgAssign));
if (R == BaseFragments.end())
- continue;
+ return;
BaseFragment = R->second;
}
std::optional<DIExpression::FragmentInfo> CurrentFragment =
@@ -393,7 +420,7 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
BaseFragment, CurrentFragment, NewFragment);
if (Result == Skip)
- continue;
+ return;
if (Result == UseFrag && !(NewFragment == CurrentFragment)) {
if (CurrentFragment) {
// Rewrite NewFragment to be relative to the existing one (this is
@@ -425,9 +452,10 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
}
::Value *NewValue = Value ? Value : DbgAssign->getValue();
- auto *NewAssign = DIB.insertDbgAssign(
- Inst, NewValue, DbgAssign->getVariable(), Expr, Dest,
- DIExpression::get(Ctx, std::nullopt), DbgAssign->getDebugLoc());
+ auto *NewAssign = createLinkedAssign(
+ DbgAssign, DIB, Inst, NewValue, DbgAssign->getVariable(), Expr, Dest,
+ DIExpression::get(Expr->getContext(), std::nullopt),
+ DbgAssign->getDebugLoc());
// If we've updated the value but the original dbg.assign has an arglist
// then kill it now - we can't use the requested new value.
@@ -461,9 +489,11 @@ static void migrateDebugInfo(AllocaInst *OldAlloca, bool IsSplit,
NewAssign->moveBefore(DbgAssign);
NewAssign->setDebugLoc(DbgAssign->getDebugLoc());
- LLVM_DEBUG(dbgs() << "Created new assign intrinsic: " << *NewAssign
- << "\n");
- }
+ LLVM_DEBUG(dbgs() << "Created new assign: " << *NewAssign << "\n");
+ };
+
+ for_each(MarkerRange, MigrateDbgAssign);
+ for_each(DPVAssignMarkerRange, MigrateDbgAssign);
}
namespace {
@@ -1903,8 +1933,8 @@ static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr,
APInt Offset, Type *PointerTy,
const Twine &NamePrefix) {
if (Offset != 0)
- Ptr = IRB.CreateInBoundsGEP(IRB.getInt8Ty(), Ptr, IRB.getInt(Offset),
- NamePrefix + "sroa_idx");
+ Ptr = IRB.CreateInBoundsPtrAdd(Ptr, IRB.getInt(Offset),
+ NamePrefix + "sroa_idx");
return IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, PointerTy,
NamePrefix + "sroa_cast");
}
@@ -2108,8 +2138,9 @@ static bool isVectorPromotionViableForSlice(Partition &P, const Slice &S,
/// Test whether a vector type is viable for promotion.
///
-/// This implements the necessary checking for \c isVectorPromotionViable over
-/// all slices of the alloca for the given VectorType.
+/// This implements the necessary checking for \c checkVectorTypesForPromotion
+/// (and thus isVectorPromotionViable) over all slices of the alloca for the
+/// given VectorType.
static bool checkVectorTypeForPromotion(Partition &P, VectorType *VTy,
const DataLayout &DL) {
uint64_t ElementSize =
@@ -2134,6 +2165,98 @@ static bool checkVectorTypeForPromotion(Partition &P, VectorType *VTy,
return true;
}
+/// Test whether any vector type in \p CandidateTys is viable for promotion.
+///
+/// This implements the necessary checking for \c isVectorPromotionViable over
+/// all slices of the alloca for the given VectorType.
+static VectorType *
+checkVectorTypesForPromotion(Partition &P, const DataLayout &DL,
+ SmallVectorImpl<VectorType *> &CandidateTys,
+ bool HaveCommonEltTy, Type *CommonEltTy,
+ bool HaveVecPtrTy, bool HaveCommonVecPtrTy,
+ VectorType *CommonVecPtrTy) {
+ // If we didn't find a vector type, nothing to do here.
+ if (CandidateTys.empty())
+ return nullptr;
+
+ // Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
+ // then we should choose it, not some other alternative.
+ // But, we can't perform a no-op pointer address space change via bitcast,
+ // so if we didn't have a common pointer element type, bail.
+ if (HaveVecPtrTy && !HaveCommonVecPtrTy)
+ return nullptr;
+
+ // Try to pick the "best" element type out of the choices.
+ if (!HaveCommonEltTy && HaveVecPtrTy) {
+ // If there was a pointer element type, there's really only one choice.
+ CandidateTys.clear();
+ CandidateTys.push_back(CommonVecPtrTy);
+ } else if (!HaveCommonEltTy && !HaveVecPtrTy) {
+ // Integer-ify vector types.
+ for (VectorType *&VTy : CandidateTys) {
+ if (!VTy->getElementType()->isIntegerTy())
+ VTy = cast<VectorType>(VTy->getWithNewType(IntegerType::getIntNTy(
+ VTy->getContext(), VTy->getScalarSizeInBits())));
+ }
+
+ // Rank the remaining candidate vector types. This is easy because we know
+ // they're all integer vectors. We sort by ascending number of elements.
+ auto RankVectorTypesComp = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
+ (void)DL;
+ assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
+ DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
+ "Cannot have vector types of different sizes!");
+ assert(RHSTy->getElementType()->isIntegerTy() &&
+ "All non-integer types eliminated!");
+ assert(LHSTy->getElementType()->isIntegerTy() &&
+ "All non-integer types eliminated!");
+ return cast<FixedVectorType>(RHSTy)->getNumElements() <
+ cast<FixedVectorType>(LHSTy)->getNumElements();
+ };
+ auto RankVectorTypesEq = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
+ (void)DL;
+ assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
+ DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
+ "Cannot have vector types of different sizes!");
+ assert(RHSTy->getElementType()->isIntegerTy() &&
+ "All non-integer types eliminated!");
+ assert(LHSTy->getElementType()->isIntegerTy() &&
+ "All non-integer types eliminated!");
+ return cast<FixedVectorType>(RHSTy)->getNumElements() ==
+ cast<FixedVectorType>(LHSTy)->getNumElements();
+ };
+ llvm::sort(CandidateTys, RankVectorTypesComp);
+ CandidateTys.erase(std::unique(CandidateTys.begin(), CandidateTys.end(),
+ RankVectorTypesEq),
+ CandidateTys.end());
+ } else {
+// The only way to have the same element type in every vector type is to
+// have the same vector type. Check that and remove all but one.
+#ifndef NDEBUG
+ for (VectorType *VTy : CandidateTys) {
+ assert(VTy->getElementType() == CommonEltTy &&
+ "Unaccounted for element type!");
+ assert(VTy == CandidateTys[0] &&
+ "Different vector types with the same element type!");
+ }
+#endif
+ CandidateTys.resize(1);
+ }
+
+ // FIXME: hack. Do we have a named constant for this?
+ // SDAG SDNode can't have more than 65535 operands.
+ llvm::erase_if(CandidateTys, [](VectorType *VTy) {
+ return cast<FixedVectorType>(VTy)->getNumElements() >
+ std::numeric_limits<unsigned short>::max();
+ });
+
+ for (VectorType *VTy : CandidateTys)
+ if (checkVectorTypeForPromotion(P, VTy, DL))
+ return VTy;
+
+ return nullptr;
+}
+
/// Test whether the given alloca partitioning and range of slices can be
/// promoted to a vector.
///
@@ -2181,6 +2304,7 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
}
}
};
+
// Put load and store types into a set for de-duplication.
for (const Slice &S : P) {
Type *Ty;
@@ -2195,6 +2319,12 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
if (S.beginOffset() == P.beginOffset() && S.endOffset() == P.endOffset())
CheckCandidateType(Ty);
}
+
+ if (auto *VTy = checkVectorTypesForPromotion(
+ P, DL, CandidateTys, HaveCommonEltTy, CommonEltTy, HaveVecPtrTy,
+ HaveCommonVecPtrTy, CommonVecPtrTy))
+ return VTy;
+
// Consider additional vector types where the element type size is a
// multiple of load/store element size.
for (Type *Ty : LoadStoreTys) {
@@ -2204,6 +2334,7 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
// Make a copy of CandidateTys and iterate through it, because we might
// append to CandidateTys in the loop.
SmallVector<VectorType *, 4> CandidateTysCopy = CandidateTys;
+ CandidateTys.clear();
for (VectorType *&VTy : CandidateTysCopy) {
unsigned VectorSize = DL.getTypeSizeInBits(VTy).getFixedValue();
unsigned ElementSize =
@@ -2216,86 +2347,9 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
}
}
- // If we didn't find a vector type, nothing to do here.
- if (CandidateTys.empty())
- return nullptr;
-
- // Pointer-ness is sticky, if we had a vector-of-pointers candidate type,
- // then we should choose it, not some other alternative.
- // But, we can't perform a no-op pointer address space change via bitcast,
- // so if we didn't have a common pointer element type, bail.
- if (HaveVecPtrTy && !HaveCommonVecPtrTy)
- return nullptr;
-
- // Try to pick the "best" element type out of the choices.
- if (!HaveCommonEltTy && HaveVecPtrTy) {
- // If there was a pointer element type, there's really only one choice.
- CandidateTys.clear();
- CandidateTys.push_back(CommonVecPtrTy);
- } else if (!HaveCommonEltTy && !HaveVecPtrTy) {
- // Integer-ify vector types.
- for (VectorType *&VTy : CandidateTys) {
- if (!VTy->getElementType()->isIntegerTy())
- VTy = cast<VectorType>(VTy->getWithNewType(IntegerType::getIntNTy(
- VTy->getContext(), VTy->getScalarSizeInBits())));
- }
-
- // Rank the remaining candidate vector types. This is easy because we know
- // they're all integer vectors. We sort by ascending number of elements.
- auto RankVectorTypesComp = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
- (void)DL;
- assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
- DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
- "Cannot have vector types of different sizes!");
- assert(RHSTy->getElementType()->isIntegerTy() &&
- "All non-integer types eliminated!");
- assert(LHSTy->getElementType()->isIntegerTy() &&
- "All non-integer types eliminated!");
- return cast<FixedVectorType>(RHSTy)->getNumElements() <
- cast<FixedVectorType>(LHSTy)->getNumElements();
- };
- auto RankVectorTypesEq = [&DL](VectorType *RHSTy, VectorType *LHSTy) {
- (void)DL;
- assert(DL.getTypeSizeInBits(RHSTy).getFixedValue() ==
- DL.getTypeSizeInBits(LHSTy).getFixedValue() &&
- "Cannot have vector types of different sizes!");
- assert(RHSTy->getElementType()->isIntegerTy() &&
- "All non-integer types eliminated!");
- assert(LHSTy->getElementType()->isIntegerTy() &&
- "All non-integer types eliminated!");
- return cast<FixedVectorType>(RHSTy)->getNumElements() ==
- cast<FixedVectorType>(LHSTy)->getNumElements();
- };
- llvm::sort(CandidateTys, RankVectorTypesComp);
- CandidateTys.erase(std::unique(CandidateTys.begin(), CandidateTys.end(),
- RankVectorTypesEq),
- CandidateTys.end());
- } else {
-// The only way to have the same element type in every vector type is to
-// have the same vector type. Check that and remove all but one.
-#ifndef NDEBUG
- for (VectorType *VTy : CandidateTys) {
- assert(VTy->getElementType() == CommonEltTy &&
- "Unaccounted for element type!");
- assert(VTy == CandidateTys[0] &&
- "Different vector types with the same element type!");
- }
-#endif
- CandidateTys.resize(1);
- }
-
- // FIXME: hack. Do we have a named constant for this?
- // SDAG SDNode can't have more than 65535 operands.
- llvm::erase_if(CandidateTys, [](VectorType *VTy) {
- return cast<FixedVectorType>(VTy)->getNumElements() >
- std::numeric_limits<unsigned short>::max();
- });
-
- for (VectorType *VTy : CandidateTys)
- if (checkVectorTypeForPromotion(P, VTy, DL))
- return VTy;
-
- return nullptr;
+ return checkVectorTypesForPromotion(P, DL, CandidateTys, HaveCommonEltTy,
+ CommonEltTy, HaveVecPtrTy,
+ HaveCommonVecPtrTy, CommonVecPtrTy);
}
/// Test whether a slice of an alloca is valid for integer widening.
@@ -3108,6 +3162,7 @@ private:
// emit dbg.assign intrinsics for mem intrinsics storing through non-
// constant geps, or storing a variable number of bytes.
assert(at::getAssignmentMarkers(&II).empty() &&
+ at::getDPVAssignmentMarkers(&II).empty() &&
"AT: Unexpected link to non-const GEP");
deleteIfTriviallyDead(OldPtr);
return false;
@@ -3254,11 +3309,13 @@ private:
Value *AdjustedPtr = getNewAllocaSlicePtr(IRB, OldPtr->getType());
if (IsDest) {
// Update the address component of linked dbg.assigns.
- for (auto *DAI : at::getAssignmentMarkers(&II)) {
- if (llvm::is_contained(DAI->location_ops(), II.getDest()) ||
- DAI->getAddress() == II.getDest())
- DAI->replaceVariableLocationOp(II.getDest(), AdjustedPtr);
- }
+ auto UpdateAssignAddress = [&](auto *DbgAssign) {
+ if (llvm::is_contained(DbgAssign->location_ops(), II.getDest()) ||
+ DbgAssign->getAddress() == II.getDest())
+ DbgAssign->replaceVariableLocationOp(II.getDest(), AdjustedPtr);
+ };
+ for_each(at::getAssignmentMarkers(&II), UpdateAssignAddress);
+ for_each(at::getDPVAssignmentMarkers(&II), UpdateAssignAddress);
II.setDest(AdjustedPtr);
II.setDestAlignment(SliceAlign);
} else {
@@ -3842,6 +3899,7 @@ private:
DL);
} else {
assert(at::getAssignmentMarkers(Store).empty() &&
+ at::getDPVAssignmentMarkers(Store).empty() &&
"AT: unexpected debug.assign linked to store through "
"unbounded GEP");
}
@@ -4861,10 +4919,22 @@ static void insertNewDbgInst(DIBuilder &DIB, DPValue *Orig, AllocaInst *NewAddr,
DIExpression *NewFragmentExpr,
Instruction *BeforeInst) {
(void)DIB;
- DPValue *New = new DPValue(ValueAsMetadata::get(NewAddr), Orig->getVariable(),
- NewFragmentExpr, Orig->getDebugLoc(),
- DPValue::LocationType::Declare);
- BeforeInst->getParent()->insertDPValueBefore(New, BeforeInst->getIterator());
+ if (Orig->isDbgDeclare()) {
+ DPValue *DPV = DPValue::createDPVDeclare(
+ NewAddr, Orig->getVariable(), NewFragmentExpr, Orig->getDebugLoc());
+ BeforeInst->getParent()->insertDPValueBefore(DPV,
+ BeforeInst->getIterator());
+ return;
+ }
+ if (!NewAddr->hasMetadata(LLVMContext::MD_DIAssignID)) {
+ NewAddr->setMetadata(LLVMContext::MD_DIAssignID,
+ DIAssignID::getDistinct(NewAddr->getContext()));
+ }
+ auto *NewAssign = DPValue::createLinkedDPVAssign(
+ NewAddr, Orig->getValue(), Orig->getVariable(), NewFragmentExpr, NewAddr,
+ Orig->getAddressExpression(), Orig->getDebugLoc());
+ LLVM_DEBUG(dbgs() << "Created new DPVAssign: " << *NewAssign << "\n");
+ (void)NewAssign;
}
/// Walks the slices of an alloca and form partitions based on them,
@@ -5021,9 +5091,6 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
// Remove any existing intrinsics on the new alloca describing
// the variable fragment.
- SmallVector<DbgDeclareInst *, 1> FragDbgDeclares;
- SmallVector<DPValue *, 1> FragDPVs;
- findDbgDeclares(FragDbgDeclares, Fragment.Alloca, &FragDPVs);
auto RemoveOne = [DbgVariable](auto *OldDII) {
auto SameVariableFragment = [](const auto *LHS, const auto *RHS) {
return LHS->getVariable() == RHS->getVariable() &&
@@ -5033,8 +5100,8 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
if (SameVariableFragment(OldDII, DbgVariable))
OldDII->eraseFromParent();
};
- for_each(FragDbgDeclares, RemoveOne);
- for_each(FragDPVs, RemoveOne);
+ for_each(findDbgDeclares(Fragment.Alloca), RemoveOne);
+ for_each(findDPVDeclares(Fragment.Alloca), RemoveOne);
insertNewDbgInst(DIB, DbgVariable, Fragment.Alloca, FragmentExpr, &AI);
}
@@ -5042,12 +5109,10 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
// Migrate debug information from the old alloca to the new alloca(s)
// and the individual partitions.
- SmallVector<DbgDeclareInst *, 1> DbgDeclares;
- SmallVector<DPValue *, 1> DPValues;
- findDbgDeclares(DbgDeclares, &AI, &DPValues);
- for_each(DbgDeclares, MigrateOne);
- for_each(DPValues, MigrateOne);
+ for_each(findDbgDeclares(&AI), MigrateOne);
+ for_each(findDPVDeclares(&AI), MigrateOne);
for_each(at::getAssignmentMarkers(&AI), MigrateOne);
+ for_each(at::getDPVAssignmentMarkers(&AI), MigrateOne);
return Changed;
}
@@ -5169,12 +5234,9 @@ bool SROA::deleteDeadInstructions(
// not be able to find it.
if (AllocaInst *AI = dyn_cast<AllocaInst>(I)) {
DeletedAllocas.insert(AI);
- SmallVector<DbgDeclareInst *, 1> DbgDeclares;
- SmallVector<DPValue *, 1> DPValues;
- findDbgDeclares(DbgDeclares, AI, &DPValues);
- for (DbgDeclareInst *OldDII : DbgDeclares)
+ for (DbgDeclareInst *OldDII : findDbgDeclares(AI))
OldDII->eraseFromParent();
- for (DPValue *OldDII : DPValues)
+ for (DPValue *OldDII : findDPVDeclares(AI))
OldDII->eraseFromParent();
}
@@ -5271,8 +5333,9 @@ std::pair<bool /*Changed*/, bool /*CFGChanged*/> SROA::runSROA(Function &F) {
"Should not have modified the CFG when told to preserve it.");
if (Changed && isAssignmentTrackingEnabled(*F.getParent())) {
- for (auto &BB : F)
+ for (auto &BB : F) {
RemoveRedundantDbgInstrs(&BB);
+ }
}
return {Changed, CFGChanged};
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-16 12:10:02 +0000