diff options
Diffstat (limited to 'contrib/llvm-project/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp')
| -rw-r--r-- | contrib/llvm-project/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp | 47 |
1 files changed, 33 insertions, 14 deletions
diff --git a/contrib/llvm-project/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp b/contrib/llvm-project/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp index bb719a499a4c..cd3b6c1a095a 100644 --- a/contrib/llvm-project/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp +++ b/contrib/llvm-project/llvm/lib/Transforms/Utils/LoopUnrollRuntime.cpp @@ -20,20 +20,19 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/LoopIterator.h" #include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/MDBuilder.h" -#include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/Utils.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/Local.h" @@ -74,7 +73,8 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count, BasicBlock *OriginalLoopLatchExit, BasicBlock *PreHeader, BasicBlock *NewPreHeader, ValueToValueMapTy &VMap, DominatorTree *DT, - LoopInfo *LI, bool PreserveLCSSA) { + LoopInfo *LI, bool PreserveLCSSA, + ScalarEvolution &SE) { // Loop structure should be the following: // Preheader // PrologHeader @@ -134,6 +134,7 @@ static void ConnectProlog(Loop *L, Value *BECount, unsigned Count, PN.setIncomingValueForBlock(NewPreHeader, NewPN); else PN.addIncoming(NewPN, PrologExit); + SE.forgetValue(&PN); } } @@ -192,7 +193,8 @@ static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, BasicBlock *Exit, BasicBlock *PreHeader, BasicBlock *EpilogPreHeader, BasicBlock *NewPreHeader, ValueToValueMapTy &VMap, DominatorTree *DT, - LoopInfo *LI, bool PreserveLCSSA) { + LoopInfo *LI, bool PreserveLCSSA, + ScalarEvolution &SE) { BasicBlock *Latch = L->getLoopLatch(); assert(Latch && "Loop must have a latch"); BasicBlock *EpilogLatch = cast<BasicBlock>(VMap[Latch]); @@ -233,6 +235,7 @@ static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, // Add incoming PreHeader from branch around the Loop PN.addIncoming(UndefValue::get(PN.getType()), PreHeader); + SE.forgetValue(&PN); Value *V = PN.getIncomingValueForBlock(Latch); Instruction *I = dyn_cast<Instruction>(V); @@ -398,7 +401,7 @@ CloneLoopBlocks(Loop *L, Value *NewIter, const bool UseEpilogRemainder, Optional<MDNode *> NewLoopID = makeFollowupLoopID( LoopID, {LLVMLoopUnrollFollowupAll, LLVMLoopUnrollFollowupRemainder}); - if (NewLoopID.hasValue()) { + if (NewLoopID) { NewLoop->setLoopID(NewLoopID.getValue()); // Do not setLoopAlreadyUnrolled if loop attributes have been defined @@ -739,11 +742,28 @@ bool llvm::UnrollRuntimeLoopRemainder( // Compute the number of extra iterations required, which is: // extra iterations = run-time trip count % loop unroll factor PreHeaderBR = cast<BranchInst>(PreHeader->getTerminator()); + IRBuilder<> B(PreHeaderBR); Value *TripCount = Expander.expandCodeFor(TripCountSC, TripCountSC->getType(), PreHeaderBR); - Value *BECount = Expander.expandCodeFor(BECountSC, BECountSC->getType(), - PreHeaderBR); - IRBuilder<> B(PreHeaderBR); + Value *BECount; + // If there are other exits before the latch, that may cause the latch exit + // branch to never be executed, and the latch exit count may be poison. + // In this case, freeze the TripCount and base BECount on the frozen + // TripCount. We will introduce two branches using these values, and it's + // important that they see a consistent value (which would not be guaranteed + // if were frozen independently.) + if ((!OtherExits.empty() || !SE->loopHasNoAbnormalExits(L)) && + !isGuaranteedNotToBeUndefOrPoison(TripCount, AC, PreHeaderBR, DT)) { + TripCount = B.CreateFreeze(TripCount); + BECount = + B.CreateAdd(TripCount, ConstantInt::get(TripCount->getType(), -1)); + } else { + // If we don't need to freeze, use SCEVExpander for BECount as well, to + // allow slightly better value reuse. + BECount = + Expander.expandCodeFor(BECountSC, BECountSC->getType(), PreHeaderBR); + } + Value * const ModVal = CreateTripRemainder(B, BECount, TripCount, Count); Value *BranchVal = @@ -884,9 +904,8 @@ bool llvm::UnrollRuntimeLoopRemainder( if (UseEpilogRemainder) { // Connect the epilog code to the original loop and update the // PHI functions. - ConnectEpilog(L, ModVal, NewExit, LatchExit, PreHeader, - EpilogPreHeader, NewPreHeader, VMap, DT, LI, - PreserveLCSSA); + ConnectEpilog(L, ModVal, NewExit, LatchExit, PreHeader, EpilogPreHeader, + NewPreHeader, VMap, DT, LI, PreserveLCSSA, *SE); // Update counter in loop for unrolling. // Use an incrementing IV. Pre-incr/post-incr is backedge/trip count. @@ -910,7 +929,7 @@ bool llvm::UnrollRuntimeLoopRemainder( // Connect the prolog code to the original loop and update the // PHI functions. ConnectProlog(L, BECount, Count, PrologExit, LatchExit, PreHeader, - NewPreHeader, VMap, DT, LI, PreserveLCSSA); + NewPreHeader, VMap, DT, LI, PreserveLCSSA, *SE); } // If this loop is nested, then the loop unroller changes the code in the any @@ -941,7 +960,7 @@ bool llvm::UnrollRuntimeLoopRemainder( SmallVector<WeakTrackingVH, 16> DeadInsts; for (BasicBlock *BB : RemainderBlocks) { for (Instruction &Inst : llvm::make_early_inc_range(*BB)) { - if (Value *V = SimplifyInstruction(&Inst, {DL, nullptr, DT, AC})) + if (Value *V = simplifyInstruction(&Inst, {DL, nullptr, DT, AC})) if (LI->replacementPreservesLCSSAForm(&Inst, V)) Inst.replaceAllUsesWith(V); if (isInstructionTriviallyDead(&Inst)) |