aboutsummaryrefslogtreecommitdiff
path: root/lib/Transforms/Scalar/SROA.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Transforms/Scalar/SROA.cpp')
-rw-r--r--lib/Transforms/Scalar/SROA.cpp214
1 files changed, 104 insertions, 110 deletions
diff --git a/lib/Transforms/Scalar/SROA.cpp b/lib/Transforms/Scalar/SROA.cpp
index 7d33259c030b..bfcb15530ef5 100644
--- a/lib/Transforms/Scalar/SROA.cpp
+++ b/lib/Transforms/Scalar/SROA.cpp
@@ -44,12 +44,12 @@
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Operator.h"
#include "llvm/Pass.h"
+#include "llvm/Support/Chrono.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/TimeValue.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
#include "llvm/Transforms/Utils/Local.h"
@@ -432,19 +432,18 @@ class AllocaSlices::partition_iterator
// cannot change the max split slice end because we just checked that
// the prior partition ended prior to that max.
P.SplitTails.erase(
- std::remove_if(
- P.SplitTails.begin(), P.SplitTails.end(),
- [&](Slice *S) { return S->endOffset() <= P.EndOffset; }),
+ remove_if(P.SplitTails,
+ [&](Slice *S) { return S->endOffset() <= P.EndOffset; }),
P.SplitTails.end());
- assert(std::any_of(P.SplitTails.begin(), P.SplitTails.end(),
- [&](Slice *S) {
- return S->endOffset() == MaxSplitSliceEndOffset;
- }) &&
+ assert(any_of(P.SplitTails,
+ [&](Slice *S) {
+ return S->endOffset() == MaxSplitSliceEndOffset;
+ }) &&
"Could not find the current max split slice offset!");
- assert(std::all_of(P.SplitTails.begin(), P.SplitTails.end(),
- [&](Slice *S) {
- return S->endOffset() <= MaxSplitSliceEndOffset;
- }) &&
+ assert(all_of(P.SplitTails,
+ [&](Slice *S) {
+ return S->endOffset() <= MaxSplitSliceEndOffset;
+ }) &&
"Max split slice end offset is not actually the max!");
}
}
@@ -693,7 +692,7 @@ private:
break;
// Handle a struct index, which adds its field offset to the pointer.
- if (StructType *STy = dyn_cast<StructType>(*GTI)) {
+ if (StructType *STy = GTI.getStructTypeOrNull()) {
unsigned ElementIdx = OpC->getZExtValue();
const StructLayout *SL = DL.getStructLayout(STy);
GEPOffset +=
@@ -996,15 +995,13 @@ AllocaSlices::AllocaSlices(const DataLayout &DL, AllocaInst &AI)
return;
}
- Slices.erase(std::remove_if(Slices.begin(), Slices.end(),
- [](const Slice &S) {
- return S.isDead();
- }),
+ Slices.erase(remove_if(Slices, [](const Slice &S) { return S.isDead(); }),
Slices.end());
#ifndef NDEBUG
if (SROARandomShuffleSlices) {
- std::mt19937 MT(static_cast<unsigned>(sys::TimeValue::now().msec()));
+ std::mt19937 MT(static_cast<unsigned>(
+ std::chrono::system_clock::now().time_since_epoch().count()));
std::shuffle(Slices.begin(), Slices.end(), MT);
}
#endif
@@ -1815,10 +1812,10 @@ static VectorType *isVectorPromotionViable(Partition &P, const DataLayout &DL) {
// do that until all the backends are known to produce good code for all
// integer vector types.
if (!HaveCommonEltTy) {
- CandidateTys.erase(std::remove_if(CandidateTys.begin(), CandidateTys.end(),
- [](VectorType *VTy) {
- return !VTy->getElementType()->isIntegerTy();
- }),
+ CandidateTys.erase(remove_if(CandidateTys,
+ [](VectorType *VTy) {
+ return !VTy->getElementType()->isIntegerTy();
+ }),
CandidateTys.end());
// If there were no integer vector types, give up.
@@ -2486,8 +2483,8 @@ private:
}
V = convertValue(DL, IRB, V, NewAllocaTy);
StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment());
+ Store->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access);
Pass.DeadInsts.insert(&SI);
- (void)Store;
DEBUG(dbgs() << " to: " << *Store << "\n");
return true;
}
@@ -2549,6 +2546,7 @@ private:
NewSI = IRB.CreateAlignedStore(V, NewPtr, getSliceAlign(V->getType()),
SI.isVolatile());
}
+ NewSI->copyMetadata(SI, LLVMContext::MD_mem_parallel_loop_access);
if (SI.isVolatile())
NewSI->setAtomic(SI.getOrdering(), SI.getSynchScope());
Pass.DeadInsts.insert(&SI);
@@ -2878,6 +2876,17 @@ private:
// Record this instruction for deletion.
Pass.DeadInsts.insert(&II);
+ // Lifetime intrinsics are only promotable if they cover the whole alloca.
+ // Therefore, we drop lifetime intrinsics which don't cover the whole
+ // alloca.
+ // (In theory, intrinsics which partially cover an alloca could be
+ // promoted, but PromoteMemToReg doesn't handle that case.)
+ // FIXME: Check whether the alloca is promotable before dropping the
+ // lifetime intrinsics?
+ if (NewBeginOffset != NewAllocaBeginOffset ||
+ NewEndOffset != NewAllocaEndOffset)
+ return true;
+
ConstantInt *Size =
ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()),
NewEndOffset - NewBeginOffset);
@@ -2890,6 +2899,7 @@ private:
(void)New;
DEBUG(dbgs() << " to: " << *New << "\n");
+
return true;
}
@@ -3209,20 +3219,11 @@ static Type *getTypePartition(const DataLayout &DL, Type *Ty, uint64_t Offset,
return nullptr;
if (SequentialType *SeqTy = dyn_cast<SequentialType>(Ty)) {
- // We can't partition pointers...
- if (SeqTy->isPointerTy())
- return nullptr;
-
Type *ElementTy = SeqTy->getElementType();
uint64_t ElementSize = DL.getTypeAllocSize(ElementTy);
uint64_t NumSkippedElements = Offset / ElementSize;
- if (ArrayType *ArrTy = dyn_cast<ArrayType>(SeqTy)) {
- if (NumSkippedElements >= ArrTy->getNumElements())
- return nullptr;
- } else if (VectorType *VecTy = dyn_cast<VectorType>(SeqTy)) {
- if (NumSkippedElements >= VecTy->getNumElements())
- return nullptr;
- }
+ if (NumSkippedElements >= SeqTy->getNumElements())
+ return nullptr;
Offset -= NumSkippedElements * ElementSize;
// First check if we need to recurse.
@@ -3456,63 +3457,60 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
// match relative to their starting offset. We have to verify this prior to
// any rewriting.
Stores.erase(
- std::remove_if(Stores.begin(), Stores.end(),
- [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
- // Lookup the load we are storing in our map of split
- // offsets.
- auto *LI = cast<LoadInst>(SI->getValueOperand());
- // If it was completely unsplittable, then we're done,
- // and this store can't be pre-split.
- if (UnsplittableLoads.count(LI))
- return true;
+ remove_if(Stores,
+ [&UnsplittableLoads, &SplitOffsetsMap](StoreInst *SI) {
+ // Lookup the load we are storing in our map of split
+ // offsets.
+ auto *LI = cast<LoadInst>(SI->getValueOperand());
+ // If it was completely unsplittable, then we're done,
+ // and this store can't be pre-split.
+ if (UnsplittableLoads.count(LI))
+ return true;
- auto LoadOffsetsI = SplitOffsetsMap.find(LI);
- if (LoadOffsetsI == SplitOffsetsMap.end())
- return false; // Unrelated loads are definitely safe.
- auto &LoadOffsets = LoadOffsetsI->second;
+ auto LoadOffsetsI = SplitOffsetsMap.find(LI);
+ if (LoadOffsetsI == SplitOffsetsMap.end())
+ return false; // Unrelated loads are definitely safe.
+ auto &LoadOffsets = LoadOffsetsI->second;
- // Now lookup the store's offsets.
- auto &StoreOffsets = SplitOffsetsMap[SI];
+ // Now lookup the store's offsets.
+ auto &StoreOffsets = SplitOffsetsMap[SI];
- // If the relative offsets of each split in the load and
- // store match exactly, then we can split them and we
- // don't need to remove them here.
- if (LoadOffsets.Splits == StoreOffsets.Splits)
- return false;
+ // If the relative offsets of each split in the load and
+ // store match exactly, then we can split them and we
+ // don't need to remove them here.
+ if (LoadOffsets.Splits == StoreOffsets.Splits)
+ return false;
- DEBUG(dbgs()
- << " Mismatched splits for load and store:\n"
- << " " << *LI << "\n"
- << " " << *SI << "\n");
+ DEBUG(dbgs() << " Mismatched splits for load and store:\n"
+ << " " << *LI << "\n"
+ << " " << *SI << "\n");
- // We've found a store and load that we need to split
- // with mismatched relative splits. Just give up on them
- // and remove both instructions from our list of
- // candidates.
- UnsplittableLoads.insert(LI);
- return true;
- }),
+ // We've found a store and load that we need to split
+ // with mismatched relative splits. Just give up on them
+ // and remove both instructions from our list of
+ // candidates.
+ UnsplittableLoads.insert(LI);
+ return true;
+ }),
Stores.end());
// Now we have to go *back* through all the stores, because a later store may
// have caused an earlier store's load to become unsplittable and if it is
// unsplittable for the later store, then we can't rely on it being split in
// the earlier store either.
- Stores.erase(std::remove_if(Stores.begin(), Stores.end(),
- [&UnsplittableLoads](StoreInst *SI) {
- auto *LI =
- cast<LoadInst>(SI->getValueOperand());
- return UnsplittableLoads.count(LI);
- }),
+ Stores.erase(remove_if(Stores,
+ [&UnsplittableLoads](StoreInst *SI) {
+ auto *LI = cast<LoadInst>(SI->getValueOperand());
+ return UnsplittableLoads.count(LI);
+ }),
Stores.end());
// Once we've established all the loads that can't be split for some reason,
// filter any that made it into our list out.
- Loads.erase(std::remove_if(Loads.begin(), Loads.end(),
- [&UnsplittableLoads](LoadInst *LI) {
- return UnsplittableLoads.count(LI);
- }),
+ Loads.erase(remove_if(Loads,
+ [&UnsplittableLoads](LoadInst *LI) {
+ return UnsplittableLoads.count(LI);
+ }),
Loads.end());
-
// If no loads or stores are left, there is no pre-splitting to be done for
// this alloca.
if (Loads.empty() && Stores.empty())
@@ -3570,6 +3568,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
PartPtrTy, BasePtr->getName() + "."),
getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false,
LI->getName());
+ PLoad->copyMetadata(*LI, LLVMContext::MD_mem_parallel_loop_access);
// Append this load onto the list of split loads so we can find it later
// to rewrite the stores.
@@ -3622,7 +3621,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
APInt(DL.getPointerSizeInBits(), PartOffset),
PartPtrTy, StoreBasePtr->getName() + "."),
getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false);
- (void)PStore;
+ PStore->copyMetadata(*LI, LLVMContext::MD_mem_parallel_loop_access);
DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
}
@@ -3770,9 +3769,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
}
// Remove the killed slices that have ben pre-split.
- AS.erase(std::remove_if(AS.begin(), AS.end(), [](const Slice &S) {
- return S.isDead();
- }), AS.end());
+ AS.erase(remove_if(AS, [](const Slice &S) { return S.isDead(); }), AS.end());
// Insert our new slices. This will sort and merge them into the sorted
// sequence.
@@ -3787,8 +3784,8 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
// Finally, don't try to promote any allocas that new require re-splitting.
// They have already been added to the worklist above.
PromotableAllocas.erase(
- std::remove_if(
- PromotableAllocas.begin(), PromotableAllocas.end(),
+ remove_if(
+ PromotableAllocas,
[&](AllocaInst *AI) { return ResplitPromotableAllocas.count(AI); }),
PromotableAllocas.end());
@@ -3985,16 +3982,16 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
if (!IsSorted)
std::sort(AS.begin(), AS.end());
- /// \brief Describes the allocas introduced by rewritePartition
- /// in order to migrate the debug info.
- struct Piece {
+ /// Describes the allocas introduced by rewritePartition in order to migrate
+ /// the debug info.
+ struct Fragment {
AllocaInst *Alloca;
uint64_t Offset;
uint64_t Size;
- Piece(AllocaInst *AI, uint64_t O, uint64_t S)
+ Fragment(AllocaInst *AI, uint64_t O, uint64_t S)
: Alloca(AI), Offset(O), Size(S) {}
};
- SmallVector<Piece, 4> Pieces;
+ SmallVector<Fragment, 4> Fragments;
// Rewrite each partition.
for (auto &P : AS.partitions()) {
@@ -4005,7 +4002,7 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
uint64_t AllocaSize = DL.getTypeSizeInBits(NewAI->getAllocatedType());
// Don't include any padding.
uint64_t Size = std::min(AllocaSize, P.size() * SizeOfByte);
- Pieces.push_back(Piece(NewAI, P.beginOffset() * SizeOfByte, Size));
+ Fragments.push_back(Fragment(NewAI, P.beginOffset() * SizeOfByte, Size));
}
}
++NumPartitions;
@@ -4022,35 +4019,34 @@ bool SROA::splitAlloca(AllocaInst &AI, AllocaSlices &AS) {
auto *Expr = DbgDecl->getExpression();
DIBuilder DIB(*AI.getModule(), /*AllowUnresolved*/ false);
uint64_t AllocaSize = DL.getTypeSizeInBits(AI.getAllocatedType());
- for (auto Piece : Pieces) {
- // Create a piece expression describing the new partition or reuse AI's
+ for (auto Fragment : Fragments) {
+ // Create a fragment expression describing the new partition or reuse AI's
// expression if there is only one partition.
- auto *PieceExpr = Expr;
- if (Piece.Size < AllocaSize || Expr->isBitPiece()) {
+ auto *FragmentExpr = Expr;
+ if (Fragment.Size < AllocaSize || Expr->isFragment()) {
// If this alloca is already a scalar replacement of a larger aggregate,
- // Piece.Offset describes the offset inside the scalar.
- uint64_t Offset = Expr->isBitPiece() ? Expr->getBitPieceOffset() : 0;
- uint64_t Start = Offset + Piece.Offset;
- uint64_t Size = Piece.Size;
- if (Expr->isBitPiece()) {
- uint64_t AbsEnd = Expr->getBitPieceOffset() + Expr->getBitPieceSize();
+ // Fragment.Offset describes the offset inside the scalar.
+ auto ExprFragment = Expr->getFragmentInfo();
+ uint64_t Offset = ExprFragment ? ExprFragment->OffsetInBits : 0;
+ uint64_t Start = Offset + Fragment.Offset;
+ uint64_t Size = Fragment.Size;
+ if (ExprFragment) {
+ uint64_t AbsEnd =
+ ExprFragment->OffsetInBits + ExprFragment->SizeInBits;
if (Start >= AbsEnd)
// No need to describe a SROAed padding.
continue;
Size = std::min(Size, AbsEnd - Start);
}
- PieceExpr = DIB.createBitPieceExpression(Start, Size);
- } else {
- assert(Pieces.size() == 1 &&
- "partition is as large as original alloca");
+ FragmentExpr = DIB.createFragmentExpression(Start, Size);
}
// Remove any existing dbg.declare intrinsic describing the same alloca.
- if (DbgDeclareInst *OldDDI = FindAllocaDbgDeclare(Piece.Alloca))
+ if (DbgDeclareInst *OldDDI = FindAllocaDbgDeclare(Fragment.Alloca))
OldDDI->eraseFromParent();
- DIB.insertDeclare(Piece.Alloca, Var, PieceExpr, DbgDecl->getDebugLoc(),
- &AI);
+ DIB.insertDeclare(Fragment.Alloca, Var, FragmentExpr,
+ DbgDecl->getDebugLoc(), &AI);
}
}
return Changed;
@@ -4223,9 +4219,7 @@ PreservedAnalyses SROA::runImpl(Function &F, DominatorTree &RunDT,
auto IsInSet = [&](AllocaInst *AI) { return DeletedAllocas.count(AI); };
Worklist.remove_if(IsInSet);
PostPromotionWorklist.remove_if(IsInSet);
- PromotableAllocas.erase(std::remove_if(PromotableAllocas.begin(),
- PromotableAllocas.end(),
- IsInSet),
+ PromotableAllocas.erase(remove_if(PromotableAllocas, IsInSet),
PromotableAllocas.end());
DeletedAllocas.clear();
}
@@ -4247,7 +4241,7 @@ PreservedAnalyses SROA::runImpl(Function &F, DominatorTree &RunDT,
return PA;
}
-PreservedAnalyses SROA::run(Function &F, AnalysisManager<Function> &AM) {
+PreservedAnalyses SROA::run(Function &F, FunctionAnalysisManager &AM) {
return runImpl(F, AM.getResult<DominatorTreeAnalysis>(F),
AM.getResult<AssumptionAnalysis>(F));
}
@@ -4280,7 +4274,7 @@ public:
AU.setPreservesCFG();
}
- const char *getPassName() const override { return "SROA"; }
+ StringRef getPassName() const override { return "SROA"; }
static char ID;
};
generated by cgit v1.3 (git 2.53.0) at 2026-05-06 11:36:23 +0000