diff options
Diffstat (limited to 'llvm/lib/Transforms/Utils/LoopUnroll.cpp')
| -rw-r--r-- | llvm/lib/Transforms/Utils/LoopUnroll.cpp | 42 |
1 files changed, 40 insertions, 2 deletions
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp index e8f585b4a94d..511dd61308f9 100644 --- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp +++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp @@ -45,6 +45,7 @@ #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" +#include "llvm/IR/PatternMatch.h" #include "llvm/IR/Use.h" #include "llvm/IR/User.h" #include "llvm/IR/ValueHandle.h" @@ -216,6 +217,8 @@ void llvm::simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, AssumptionCache *AC, const TargetTransformInfo *TTI) { + using namespace llvm::PatternMatch; + // Simplify any new induction variables in the partially unrolled loop. if (SE && SimplifyIVs) { SmallVector<WeakTrackingVH, 16> DeadInsts; @@ -241,6 +244,30 @@ void llvm::simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI, Inst.replaceAllUsesWith(V); if (isInstructionTriviallyDead(&Inst)) DeadInsts.emplace_back(&Inst); + + // Fold ((add X, C1), C2) to (add X, C1+C2). This is very common in + // unrolled loops, and handling this early allows following code to + // identify the IV as a "simple recurrence" without first folding away + // a long chain of adds. + { + Value *X; + const APInt *C1, *C2; + if (match(&Inst, m_Add(m_Add(m_Value(X), m_APInt(C1)), m_APInt(C2)))) { + auto *InnerI = dyn_cast<Instruction>(Inst.getOperand(0)); + auto *InnerOBO = cast<OverflowingBinaryOperator>(Inst.getOperand(0)); + bool SignedOverflow; + APInt NewC = C1->sadd_ov(*C2, SignedOverflow); + Inst.setOperand(0, X); + Inst.setOperand(1, ConstantInt::get(Inst.getType(), NewC)); + Inst.setHasNoUnsignedWrap(Inst.hasNoUnsignedWrap() && + InnerOBO->hasNoUnsignedWrap()); + Inst.setHasNoSignedWrap(Inst.hasNoSignedWrap() && + InnerOBO->hasNoSignedWrap() && + !SignedOverflow); + if (InnerI && isInstructionTriviallyDead(InnerI)) + DeadInsts.emplace_back(InnerI); + } + } } // We can't do recursive deletion until we're done iterating, as we might // have a phi which (potentially indirectly) uses instructions later in @@ -310,6 +337,9 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, const unsigned MaxTripCount = SE->getSmallConstantMaxTripCount(L); const bool MaxOrZero = SE->isBackedgeTakenCountMaxOrZero(L); + unsigned EstimatedLoopInvocationWeight = 0; + std::optional<unsigned> OriginalTripCount = + llvm::getLoopEstimatedTripCount(L, &EstimatedLoopInvocationWeight); // Effectively "DCE" unrolled iterations that are beyond the max tripcount // and will never be executed. @@ -513,7 +543,7 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, !EnableFSDiscriminator) for (BasicBlock *BB : L->getBlocks()) for (Instruction &I : *BB) - if (!isa<DbgInfoIntrinsic>(&I)) + if (!I.isDebugOrPseudoInst()) if (const DILocation *DIL = I.getDebugLoc()) { auto NewDIL = DIL->cloneByMultiplyingDuplicationFactor(ULO.Count); if (NewDIL) @@ -830,8 +860,16 @@ LoopUnrollResult llvm::UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, Loop *OuterL = L->getParentLoop(); // Update LoopInfo if the loop is completely removed. - if (CompletelyUnroll) + if (CompletelyUnroll) { LI->erase(L); + // We shouldn't try to use `L` anymore. + L = nullptr; + } else if (OriginalTripCount) { + // Update the trip count. Note that the remainder has already logic + // computing it in `UnrollRuntimeLoopRemainder`. + setLoopEstimatedTripCount(L, *OriginalTripCount / ULO.Count, + EstimatedLoopInvocationWeight); + } // LoopInfo should not be valid, confirm that. if (UnrollVerifyLoopInfo) |