summaryrefslogtreecommitdiff
path: root/llvm/lib/Transforms/Scalar/SROA.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms/Scalar/SROA.cpp')
-rw-r--r--llvm/lib/Transforms/Scalar/SROA.cpp134
1 files changed, 66 insertions, 68 deletions
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index 74b8ff913050..89916e43fce2 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -41,7 +41,6 @@
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/PtrUseVisitor.h"
-#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
@@ -71,6 +70,7 @@
#include "llvm/IR/Use.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
+#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
@@ -80,6 +80,7 @@
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
#include <algorithm>
#include <cassert>
@@ -361,7 +362,7 @@ private:
/// The beginning and ending offsets of the alloca for this
/// partition.
- uint64_t BeginOffset, EndOffset;
+ uint64_t BeginOffset = 0, EndOffset = 0;
/// The start and end iterators of this partition.
iterator SI, SJ;
@@ -1680,24 +1681,20 @@ static Value *getAdjustedPtr(IRBuilderTy &IRB, const DataLayout &DL, Value *Ptr,
}
/// Compute the adjusted alignment for a load or store from an offset.
-static unsigned getAdjustedAlignment(Instruction *I, uint64_t Offset,
- const DataLayout &DL) {
- unsigned Alignment;
+static Align getAdjustedAlignment(Instruction *I, uint64_t Offset,
+ const DataLayout &DL) {
+ MaybeAlign Alignment;
Type *Ty;
if (auto *LI = dyn_cast<LoadInst>(I)) {
- Alignment = LI->getAlignment();
+ Alignment = MaybeAlign(LI->getAlignment());
Ty = LI->getType();
} else if (auto *SI = dyn_cast<StoreInst>(I)) {
- Alignment = SI->getAlignment();
+ Alignment = MaybeAlign(SI->getAlignment());
Ty = SI->getValueOperand()->getType();
} else {
llvm_unreachable("Only loads and stores are allowed!");
}
-
- if (!Alignment)
- Alignment = DL.getABITypeAlignment(Ty);
-
- return MinAlign(Alignment, Offset);
+ return commonAlignment(DL.getValueOrABITypeAlignment(Alignment, Ty), Offset);
}
/// Test whether we can convert a value from the old to the new type.
@@ -2300,9 +2297,9 @@ class llvm::sroa::AllocaSliceRewriter
// The new offsets of the slice currently being rewritten relative to the
// original alloca.
- uint64_t NewBeginOffset, NewEndOffset;
+ uint64_t NewBeginOffset = 0, NewEndOffset = 0;
- uint64_t SliceSize;
+ uint64_t SliceSize = 0;
bool IsSplittable = false;
bool IsSplit = false;
Use *OldUse = nullptr;
@@ -2432,13 +2429,14 @@ private:
///
/// You can optionally pass a type to this routine and if that type's ABI
/// alignment is itself suitable, this will return zero.
- unsigned getSliceAlign(Type *Ty = nullptr) {
- unsigned NewAIAlign = NewAI.getAlignment();
- if (!NewAIAlign)
- NewAIAlign = DL.getABITypeAlignment(NewAI.getAllocatedType());
- unsigned Align =
- MinAlign(NewAIAlign, NewBeginOffset - NewAllocaBeginOffset);
- return (Ty && Align == DL.getABITypeAlignment(Ty)) ? 0 : Align;
+ MaybeAlign getSliceAlign(Type *Ty = nullptr) {
+ const MaybeAlign NewAIAlign = DL.getValueOrABITypeAlignment(
+ MaybeAlign(NewAI.getAlignment()), NewAI.getAllocatedType());
+ const MaybeAlign Align =
+ commonAlignment(NewAIAlign, NewBeginOffset - NewAllocaBeginOffset);
+ return (Ty && Align && Align->value() == DL.getABITypeAlignment(Ty))
+ ? None
+ : Align;
}
unsigned getIndex(uint64_t Offset) {
@@ -2800,7 +2798,7 @@ private:
Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset);
CallInst *New = IRB.CreateMemSet(
getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size,
- getSliceAlign(), II.isVolatile());
+ MaybeAlign(getSliceAlign()), II.isVolatile());
if (AATags)
New->setAAMetadata(AATags);
LLVM_DEBUG(dbgs() << " to: " << *New << "\n");
@@ -2886,7 +2884,7 @@ private:
assert((IsDest && II.getRawDest() == OldPtr) ||
(!IsDest && II.getRawSource() == OldPtr));
- unsigned SliceAlign = getSliceAlign();
+ MaybeAlign SliceAlign = getSliceAlign();
// For unsplit intrinsics, we simply modify the source and destination
// pointers in place. This isn't just an optimization, it is a matter of
@@ -2956,10 +2954,10 @@ private:
// Compute the relative offset for the other pointer within the transfer.
unsigned OffsetWidth = DL.getIndexSizeInBits(OtherAS);
APInt OtherOffset(OffsetWidth, NewBeginOffset - BeginOffset);
- unsigned OtherAlign =
- IsDest ? II.getSourceAlignment() : II.getDestAlignment();
- OtherAlign = MinAlign(OtherAlign ? OtherAlign : 1,
- OtherOffset.zextOrTrunc(64).getZExtValue());
+ Align OtherAlign =
+ assumeAligned(IsDest ? II.getSourceAlignment() : II.getDestAlignment());
+ OtherAlign =
+ commonAlignment(OtherAlign, OtherOffset.zextOrTrunc(64).getZExtValue());
if (EmitMemCpy) {
// Compute the other pointer, folding as much as possible to produce
@@ -2972,7 +2970,7 @@ private:
Constant *Size = ConstantInt::get(SizeTy, NewEndOffset - NewBeginOffset);
Value *DestPtr, *SrcPtr;
- unsigned DestAlign, SrcAlign;
+ MaybeAlign DestAlign, SrcAlign;
// Note: IsDest is true iff we're copying into the new alloca slice
if (IsDest) {
DestPtr = OurPtr;
@@ -3019,9 +3017,9 @@ private:
Value *SrcPtr = getAdjustedPtr(IRB, DL, OtherPtr, OtherOffset, OtherPtrTy,
OtherPtr->getName() + ".");
- unsigned SrcAlign = OtherAlign;
+ MaybeAlign SrcAlign = OtherAlign;
Value *DstPtr = &NewAI;
- unsigned DstAlign = SliceAlign;
+ MaybeAlign DstAlign = SliceAlign;
if (!IsDest) {
std::swap(SrcPtr, DstPtr);
std::swap(SrcAlign, DstAlign);
@@ -3117,20 +3115,17 @@ private:
Instruction *I = Uses.pop_back_val();
if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
- unsigned LoadAlign = LI->getAlignment();
- if (!LoadAlign)
- LoadAlign = DL.getABITypeAlignment(LI->getType());
- LI->setAlignment(MaybeAlign(std::min(LoadAlign, getSliceAlign())));
+ MaybeAlign LoadAlign = DL.getValueOrABITypeAlignment(
+ MaybeAlign(LI->getAlignment()), LI->getType());
+ LI->setAlignment(std::min(LoadAlign, getSliceAlign()));
continue;
}
if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
- unsigned StoreAlign = SI->getAlignment();
- if (!StoreAlign) {
Value *Op = SI->getOperand(0);
- StoreAlign = DL.getABITypeAlignment(Op->getType());
- }
- SI->setAlignment(MaybeAlign(std::min(StoreAlign, getSliceAlign())));
- continue;
+ MaybeAlign StoreAlign = DL.getValueOrABITypeAlignment(
+ MaybeAlign(SI->getAlignment()), Op->getType());
+ SI->setAlignment(std::min(StoreAlign, getSliceAlign()));
+ continue;
}
assert(isa<BitCastInst>(I) || isa<AddrSpaceCastInst>(I) ||
@@ -3222,7 +3217,7 @@ class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
/// The current pointer use being rewritten. This is used to dig up the used
/// value (as opposed to the user).
- Use *U;
+ Use *U = nullptr;
/// Used to calculate offsets, and hence alignment, of subobjects.
const DataLayout &DL;
@@ -3277,7 +3272,7 @@ private:
Type *BaseTy;
/// Known alignment of the base pointer.
- unsigned BaseAlign;
+ Align BaseAlign;
/// To calculate offset of each component so we can correctly deduce
/// alignments.
@@ -3286,7 +3281,7 @@ private:
/// Initialize the splitter with an insertion point, Ptr and start with a
/// single zero GEP index.
OpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
- unsigned BaseAlign, const DataLayout &DL)
+ Align BaseAlign, const DataLayout &DL)
: IRB(InsertionPoint), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr),
BaseTy(BaseTy), BaseAlign(BaseAlign), DL(DL) {}
@@ -3308,7 +3303,7 @@ private:
if (Ty->isSingleValueType()) {
unsigned Offset = DL.getIndexedOffsetInType(BaseTy, GEPIndices);
return static_cast<Derived *>(this)->emitFunc(
- Ty, Agg, MinAlign(BaseAlign, Offset), Name);
+ Ty, Agg, commonAlignment(BaseAlign, Offset), Name);
}
if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
@@ -3349,18 +3344,20 @@ private:
AAMDNodes AATags;
LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
- AAMDNodes AATags, unsigned BaseAlign, const DataLayout &DL)
+ AAMDNodes AATags, Align BaseAlign, const DataLayout &DL)
: OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
- DL), AATags(AATags) {}
+ DL),
+ AATags(AATags) {}
/// Emit a leaf load of a single value. This is called at the leaves of the
/// recursive emission to actually load values.
- void emitFunc(Type *Ty, Value *&Agg, unsigned Align, const Twine &Name) {
+ void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
assert(Ty->isSingleValueType());
// Load the single value and insert it using the indices.
Value *GEP =
IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
- LoadInst *Load = IRB.CreateAlignedLoad(Ty, GEP, Align, Name + ".load");
+ LoadInst *Load =
+ IRB.CreateAlignedLoad(Ty, GEP, Alignment.value(), Name + ".load");
if (AATags)
Load->setAAMetadata(AATags);
Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert");
@@ -3388,14 +3385,14 @@ private:
struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr, Type *BaseTy,
- AAMDNodes AATags, unsigned BaseAlign, const DataLayout &DL)
+ AAMDNodes AATags, Align BaseAlign, const DataLayout &DL)
: OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr, BaseTy, BaseAlign,
DL),
AATags(AATags) {}
AAMDNodes AATags;
/// Emit a leaf store of a single value. This is called at the leaves of the
/// recursive emission to actually produce stores.
- void emitFunc(Type *Ty, Value *&Agg, unsigned Align, const Twine &Name) {
+ void emitFunc(Type *Ty, Value *&Agg, Align Alignment, const Twine &Name) {
assert(Ty->isSingleValueType());
// Extract the single value and store it using the indices.
//
@@ -3406,7 +3403,7 @@ private:
Value *InBoundsGEP =
IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
StoreInst *Store =
- IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Align);
+ IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Alignment.value());
if (AATags)
Store->setAAMetadata(AATags);
LLVM_DEBUG(dbgs() << " to: " << *Store << "\n");
@@ -3863,8 +3860,8 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
getAdjustedPtr(IRB, DL, BasePtr,
APInt(DL.getIndexSizeInBits(AS), PartOffset),
PartPtrTy, BasePtr->getName() + "."),
- getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false,
- LI->getName());
+ getAdjustedAlignment(LI, PartOffset, DL).value(),
+ /*IsVolatile*/ false, LI->getName());
PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
@@ -3921,7 +3918,8 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
getAdjustedPtr(IRB, DL, StoreBasePtr,
APInt(DL.getIndexSizeInBits(AS), PartOffset),
PartPtrTy, StoreBasePtr->getName() + "."),
- getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false);
+ getAdjustedAlignment(SI, PartOffset, DL).value(),
+ /*IsVolatile*/ false);
PStore->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
LLVMContext::MD_access_group});
LLVM_DEBUG(dbgs() << " +" << PartOffset << ":" << *PStore << "\n");
@@ -4005,8 +4003,8 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
getAdjustedPtr(IRB, DL, LoadBasePtr,
APInt(DL.getIndexSizeInBits(AS), PartOffset),
LoadPartPtrTy, LoadBasePtr->getName() + "."),
- getAdjustedAlignment(LI, PartOffset, DL), /*IsVolatile*/ false,
- LI->getName());
+ getAdjustedAlignment(LI, PartOffset, DL).value(),
+ /*IsVolatile*/ false, LI->getName());
}
// And store this partition.
@@ -4017,7 +4015,8 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
getAdjustedPtr(IRB, DL, StoreBasePtr,
APInt(DL.getIndexSizeInBits(AS), PartOffset),
StorePartPtrTy, StoreBasePtr->getName() + "."),
- getAdjustedAlignment(SI, PartOffset, DL), /*IsVolatile*/ false);
+ getAdjustedAlignment(SI, PartOffset, DL).value(),
+ /*IsVolatile*/ false);
// Now build a new slice for the alloca.
NewSlices.push_back(
@@ -4152,20 +4151,19 @@ AllocaInst *SROA::rewritePartition(AllocaInst &AI, AllocaSlices &AS,
// FIXME: We might want to defer PHI speculation until after here.
// FIXME: return nullptr;
} else {
- unsigned Alignment = AI.getAlignment();
- if (!Alignment) {
- // The minimum alignment which users can rely on when the explicit
- // alignment is omitted or zero is that required by the ABI for this
- // type.
- Alignment = DL.getABITypeAlignment(AI.getAllocatedType());
- }
- Alignment = MinAlign(Alignment, P.beginOffset());
+ // If alignment is unspecified we fallback on the one required by the ABI
+ // for this type. We also make sure the alignment is compatible with
+ // P.beginOffset().
+ const Align Alignment = commonAlignment(
+ DL.getValueOrABITypeAlignment(MaybeAlign(AI.getAlignment()),
+ AI.getAllocatedType()),
+ P.beginOffset());
// If we will get at least this much alignment from the type alone, leave
// the alloca's alignment unconstrained.
- if (Alignment <= DL.getABITypeAlignment(SliceTy))
- Alignment = 0;
+ const bool IsUnconstrained = Alignment <= DL.getABITypeAlignment(SliceTy);
NewAI = new AllocaInst(
- SliceTy, AI.getType()->getAddressSpace(), nullptr, Alignment,
+ SliceTy, AI.getType()->getAddressSpace(), nullptr,
+ IsUnconstrained ? MaybeAlign() : Alignment,
AI.getName() + ".sroa." + Twine(P.begin() - AS.begin()), &AI);
// Copy the old AI debug location over to the new one.
NewAI->setDebugLoc(AI.getDebugLoc());
generated by cgit v1.3 (git 2.53.0) at 2026-07-28 04:12:31 +0000