diff options
author | Dimitry Andric <dim@FreeBSD.org> | 2019-08-20 20:50:12 +0000 |
---|---|---|
committer | Dimitry Andric <dim@FreeBSD.org> | 2019-08-20 20:50:12 +0000 |
commit | e6d1592492a3a379186bfb02bd0f4eda0669c0d5 (patch) | |
tree | 599ab169a01f1c86eda9adc774edaedde2f2db5b /lib/Transforms/Utils/SimplifyCFG.cpp | |
parent | 1a56a5ead7a2e84bee8240f5f6b033b5f1707154 (diff) |
Diffstat (limited to 'lib/Transforms/Utils/SimplifyCFG.cpp')
-rw-r--r-- | lib/Transforms/Utils/SimplifyCFG.cpp | 203 |
1 files changed, 97 insertions, 106 deletions
diff --git a/lib/Transforms/Utils/SimplifyCFG.cpp b/lib/Transforms/Utils/SimplifyCFG.cpp index 03b73954321d..11651d040dc0 100644 --- a/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/lib/Transforms/Utils/SimplifyCFG.cpp @@ -1,9 +1,8 @@ //===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,8 +25,9 @@ #include "llvm/Analysis/ConstantFolding.h" #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/InstructionSimplify.h" +#include "llvm/Analysis/MemorySSA.h" +#include "llvm/Analysis/MemorySSAUpdater.h" #include "llvm/Analysis/TargetTransformInfo.h" -#include "llvm/Transforms/Utils/Local.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" @@ -66,6 +66,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/ValueMapper.h" #include <algorithm> #include <cassert> @@ -292,9 +293,13 @@ isProfitableToFoldUnconditional(BranchInst *SI1, BranchInst *SI2, /// will be the same as those coming in from ExistPred, an existing predecessor /// of Succ. static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred, - BasicBlock *ExistPred) { + BasicBlock *ExistPred, + MemorySSAUpdater *MSSAU = nullptr) { for (PHINode &PN : Succ->phis()) PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred); + if (MSSAU) + if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ)) + MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred); } /// Compute an abstract "cost" of speculating the given instruction, @@ -670,7 +675,8 @@ private: } // end anonymous namespace -static void EraseTerminatorAndDCECond(Instruction *TI) { +static void EraseTerminatorAndDCECond(Instruction *TI, + MemorySSAUpdater *MSSAU = nullptr) { Instruction *Cond = nullptr; if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) { Cond = dyn_cast<Instruction>(SI->getCondition()); @@ -683,7 +689,7 @@ static void EraseTerminatorAndDCECond(Instruction *TI) { TI->eraseFromParent(); if (Cond) - RecursivelyDeleteTriviallyDeadInstructions(Cond); + RecursivelyDeleteTriviallyDeadInstructions(Cond, nullptr, MSSAU); } /// Return true if the specified terminator checks @@ -858,7 +864,7 @@ bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor( return true; } - SwitchInst *SI = cast<SwitchInst>(TI); + SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI); // Okay, TI has cases that are statically dead, prune them away. SmallPtrSet<Constant *, 16> DeadCases; for (unsigned i = 0, e = PredCases.size(); i != e; ++i) @@ -867,30 +873,13 @@ bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor( LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator() << "Through successor TI: " << *TI); - // Collect branch weights into a vector. - SmallVector<uint32_t, 8> Weights; - MDNode *MD = SI->getMetadata(LLVMContext::MD_prof); - bool HasWeight = MD && (MD->getNumOperands() == 2 + SI->getNumCases()); - if (HasWeight) - for (unsigned MD_i = 1, MD_e = MD->getNumOperands(); MD_i < MD_e; - ++MD_i) { - ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(MD_i)); - Weights.push_back(CI->getValue().getZExtValue()); - } for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) { --i; if (DeadCases.count(i->getCaseValue())) { - if (HasWeight) { - std::swap(Weights[i->getCaseIndex() + 1], Weights.back()); - Weights.pop_back(); - } i->getCaseSuccessor()->removePredecessor(TI->getParent()); - SI->removeCase(i); + SI.removeCase(i); } } - if (HasWeight && Weights.size() >= 2) - setBranchWeights(SI, Weights); - LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n"); return true; } @@ -1266,8 +1255,10 @@ static bool HoistThenElseCodeToIf(BranchInst *BI, while (isa<DbgInfoIntrinsic>(I2)) I2 = &*BB2_Itr++; } + // FIXME: Can we define a safety predicate for CallBr? if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) || - (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))) + (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) || + isa<CallBrInst>(I1)) return false; BasicBlock *BIParent = BI->getParent(); @@ -1350,9 +1341,14 @@ static bool HoistThenElseCodeToIf(BranchInst *BI, HoistTerminator: // It may not be possible to hoist an invoke. + // FIXME: Can we define a safety predicate for CallBr? if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) return Changed; + // TODO: callbr hoisting currently disabled pending further study. + if (isa<CallBrInst>(I1)) + return Changed; + for (BasicBlock *Succ : successors(BB1)) { for (PHINode &PN : Succ->phis()) { Value *BB1V = PN.getIncomingValueForBlock(BB1); @@ -1432,9 +1428,10 @@ HoistTerminator: static bool canSinkInstructions( ArrayRef<Instruction *> Insts, DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) { - // Prune out obviously bad instructions to move. Any non-store instruction - // must have exactly one use, and we check later that use is by a single, - // common PHI instruction in the successor. + // Prune out obviously bad instructions to move. Each instruction must have + // exactly zero or one use, and we check later that use is by a single, common + // PHI instruction in the successor. + bool HasUse = !Insts.front()->user_empty(); for (auto *I : Insts) { // These instructions may change or break semantics if moved. if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) || @@ -1444,13 +1441,14 @@ static bool canSinkInstructions( // Conservatively return false if I is an inline-asm instruction. Sinking // and merging inline-asm instructions can potentially create arguments // that cannot satisfy the inline-asm constraints. - if (const auto *C = dyn_cast<CallInst>(I)) + if (const auto *C = dyn_cast<CallBase>(I)) if (C->isInlineAsm()) return false; - // Everything must have only one use too, apart from stores which - // have no uses. - if (!isa<StoreInst>(I) && !I->hasOneUse()) + // Each instruction must have zero or one use. + if (HasUse && !I->hasOneUse()) + return false; + if (!HasUse && !I->user_empty()) return false; } @@ -1459,11 +1457,11 @@ static bool canSinkInstructions( if (!I->isSameOperationAs(I0)) return false; - // All instructions in Insts are known to be the same opcode. If they aren't - // stores, check the only user of each is a PHI or in the same block as the - // instruction, because if a user is in the same block as an instruction - // we're contemplating sinking, it must already be determined to be sinkable. - if (!isa<StoreInst>(I0)) { + // All instructions in Insts are known to be the same opcode. If they have a + // use, check that the only user is a PHI or in the same block as the + // instruction, because if a user is in the same block as an instruction we're + // contemplating sinking, it must already be determined to be sinkable. + if (HasUse) { auto *PNUse = dyn_cast<PHINode>(*I0->user_begin()); auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0); if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool { @@ -1507,7 +1505,7 @@ static bool canSinkInstructions( // We can't create a PHI from this GEP. return false; // Don't create indirect calls! The called value is the final operand. - if ((isa<CallInst>(I0) || isa<InvokeInst>(I0)) && OI == OE - 1) { + if (isa<CallBase>(I0) && OI == OE - 1) { // FIXME: if the call was *already* indirect, we should do this. return false; } @@ -1541,7 +1539,7 @@ static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) { // it is slightly over-aggressive - it gets confused by commutative instructions // so double-check it here. Instruction *I0 = Insts.front(); - if (!isa<StoreInst>(I0)) { + if (!I0->user_empty()) { auto *PNUse = dyn_cast<PHINode>(*I0->user_begin()); if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool { auto *U = cast<Instruction>(*I->user_begin()); @@ -1599,11 +1597,10 @@ static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) { I0->andIRFlags(I); } - if (!isa<StoreInst>(I0)) { + if (!I0->user_empty()) { // canSinkLastInstruction checked that all instructions were used by // one and only one PHI node. Find that now, RAUW it to our common // instruction and nuke it. - assert(I0->hasOneUse()); auto *PN = cast<PHINode>(*I0->user_begin()); PN->replaceAllUsesWith(I0); PN->eraseFromParent(); @@ -2203,7 +2200,8 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout &DL, BasicBlock *EdgeBB = BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge", RealDest->getParent(), RealDest); - BranchInst::Create(RealDest, EdgeBB); + BranchInst *CritEdgeBranch = BranchInst::Create(RealDest, EdgeBB); + CritEdgeBranch->setDebugLoc(BI->getDebugLoc()); // Update PHI nodes. AddPredecessorToBlock(RealDest, EdgeBB, BB); @@ -2539,7 +2537,8 @@ static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI, /// If this basic block is simple enough, and if a predecessor branches to us /// and one of our successors, fold the block into the predecessor and use /// logical operations to pick the right destination. -bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) { +bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU, + unsigned BonusInstThreshold) { BasicBlock *BB = BI->getParent(); const unsigned PredCount = pred_size(BB); @@ -2594,7 +2593,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) { // unconditionally. We denote all involved instructions except the condition // as "bonus instructions", and only allow this transformation when the // number of the bonus instructions we'll need to create when cloning into - // each predecessor does not exceed a certain threshold. + // each predecessor does not exceed a certain threshold. unsigned NumBonusInsts = 0; for (auto I = BB->begin(); Cond != &*I; ++I) { // Ignore dbg intrinsics. @@ -2611,7 +2610,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) { // and Cond. // Account for the cost of duplicating this instruction into each - // predecessor. + // predecessor. NumBonusInsts += PredCount; // Early exits once we reach the limit. if (NumBonusInsts > BonusInstThreshold) @@ -2750,7 +2749,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) { (SuccFalseWeight + SuccTrueWeight) + PredTrueWeight * SuccFalseWeight); } - AddPredecessorToBlock(TrueDest, PredBlock, BB); + AddPredecessorToBlock(TrueDest, PredBlock, BB, MSSAU); PBI->setSuccessor(0, TrueDest); } if (PBI->getSuccessor(1) == BB) { @@ -2765,7 +2764,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) { // FalseWeight is FalseWeight for PBI * FalseWeight for BI. NewWeights.push_back(PredFalseWeight * SuccFalseWeight); } - AddPredecessorToBlock(FalseDest, PredBlock, BB); + AddPredecessorToBlock(FalseDest, PredBlock, BB, MSSAU); PBI->setSuccessor(1, FalseDest); } if (NewWeights.size() == 2) { @@ -2810,12 +2809,17 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) { } } // Update PHI Node. - PHIs[i]->setIncomingValue(PHIs[i]->getBasicBlockIndex(PBI->getParent()), - MergedCond); + PHIs[i]->setIncomingValueForBlock(PBI->getParent(), MergedCond); } + + // PBI is changed to branch to TrueDest below. Remove itself from + // potential phis from all other successors. + if (MSSAU) + MSSAU->changeCondBranchToUnconditionalTo(PBI, TrueDest); + // Change PBI from Conditional to Unconditional. BranchInst *New_PBI = BranchInst::Create(TrueDest, PBI); - EraseTerminatorAndDCECond(PBI); + EraseTerminatorAndDCECond(PBI, MSSAU); PBI = New_PBI; } @@ -3430,7 +3434,7 @@ static bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond, KeepEdge2 = nullptr; else Succ->removePredecessor(OldTerm->getParent(), - /*DontDeleteUselessPHIs=*/true); + /*KeepOneInputPHIs=*/true); } IRBuilder<> Builder(OldTerm); @@ -3622,20 +3626,16 @@ bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt( // the switch to the merge point on the compared value. BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB); - SmallVector<uint64_t, 8> Weights; - bool HasWeights = HasBranchWeights(SI); - if (HasWeights) { - GetBranchWeights(SI, Weights); - if (Weights.size() == 1 + SI->getNumCases()) { - // Split weight for default case to case for "Cst". - Weights[0] = (Weights[0] + 1) >> 1; - Weights.push_back(Weights[0]); - - SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end()); - setBranchWeights(SI, MDWeights); + { + SwitchInstProfUpdateWrapper SIW(*SI); + auto W0 = SIW.getSuccessorWeight(0); + SwitchInstProfUpdateWrapper::CaseWeightOpt NewW; + if (W0) { + NewW = ((uint64_t(*W0) + 1) >> 1); + SIW.setSuccessorWeight(0, *NewW); } + SIW.addCase(Cst, NewBB, NewW); } - SI->addCase(Cst, NewBB); // NewBB branches to the phi block, add the uncond branch and the phi entry. Builder.SetInsertPoint(NewBB); @@ -4184,24 +4184,28 @@ bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) { Changed = true; } } else { + Value* Cond = BI->getCondition(); if (BI->getSuccessor(0) == BB) { + Builder.CreateAssumption(Builder.CreateNot(Cond)); Builder.CreateBr(BI->getSuccessor(1)); EraseTerminatorAndDCECond(BI); } else if (BI->getSuccessor(1) == BB) { + Builder.CreateAssumption(Cond); Builder.CreateBr(BI->getSuccessor(0)); EraseTerminatorAndDCECond(BI); Changed = true; } } } else if (auto *SI = dyn_cast<SwitchInst>(TI)) { - for (auto i = SI->case_begin(), e = SI->case_end(); i != e;) { + SwitchInstProfUpdateWrapper SU(*SI); + for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) { if (i->getCaseSuccessor() != BB) { ++i; continue; } - BB->removePredecessor(SI->getParent()); - i = SI->removeCase(i); - e = SI->case_end(); + BB->removePredecessor(SU->getParent()); + i = SU.removeCase(i); + e = SU->case_end(); Changed = true; } } else if (auto *II = dyn_cast<InvokeInst>(TI)) { @@ -4435,33 +4439,20 @@ static bool eliminateDeadSwitchCases(SwitchInst *SI, AssumptionCache *AC, return true; } - SmallVector<uint64_t, 8> Weights; - bool HasWeight = HasBranchWeights(SI); - if (HasWeight) { - GetBranchWeights(SI, Weights); - HasWeight = (Weights.size() == 1 + SI->getNumCases()); - } + if (DeadCases.empty()) + return false; - // Remove dead cases from the switch. + SwitchInstProfUpdateWrapper SIW(*SI); for (ConstantInt *DeadCase : DeadCases) { SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase); assert(CaseI != SI->case_default() && "Case was not found. Probably mistake in DeadCases forming."); - if (HasWeight) { - std::swap(Weights[CaseI->getCaseIndex() + 1], Weights.back()); - Weights.pop_back(); - } - // Prune unused values from PHI nodes. CaseI->getCaseSuccessor()->removePredecessor(SI->getParent()); - SI->removeCase(CaseI); - } - if (HasWeight && Weights.size() >= 2) { - SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end()); - setBranchWeights(SI, MDWeights); + SIW.removeCase(CaseI); } - return !DeadCases.empty(); + return true; } /// If BB would be eligible for simplification by @@ -5034,7 +5025,7 @@ SwitchLookupTable::SwitchLookupTable( ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize); Constant *Initializer = ConstantArray::get(ArrayTy, TableContents); - Array = new GlobalVariable(M, ArrayTy, /*constant=*/true, + Array = new GlobalVariable(M, ArrayTy, /*isConstant=*/true, GlobalVariable::PrivateLinkage, Initializer, "switch.table." + FuncName); Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global); @@ -5091,7 +5082,9 @@ Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) { Value *GEPIndices[] = {Builder.getInt32(0), Index}; Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array, GEPIndices, "switch.gep"); - return Builder.CreateLoad(GEP, "switch.load"); + return Builder.CreateLoad( + cast<ArrayType>(Array->getValueType())->getElementType(), GEP, + "switch.load"); } } llvm_unreachable("Unknown lookup table kind!"); @@ -5425,7 +5418,7 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder, // We cached PHINodes in PHIs. To avoid accessing deleted PHINodes later, // do not delete PHINodes here. SI->getDefaultDest()->removePredecessor(SI->getParent(), - /*DontDeleteUselessPHIs=*/true); + /*KeepOneInputPHIs=*/true); } bool ReturnedEarly = false; @@ -5533,25 +5526,23 @@ static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder, // Now we have signed numbers that have been shifted so that, given enough // precision, there are no negative values. Since the rest of the transform // is bitwise only, we switch now to an unsigned representation. - uint64_t GCD = 0; - for (auto &V : Values) - GCD = GreatestCommonDivisor64(GCD, (uint64_t)V); - // This transform can be done speculatively because it is so cheap - it results - // in a single rotate operation being inserted. This can only happen if the - // factor extracted is a power of 2. - // FIXME: If the GCD is an odd number we can multiply by the multiplicative - // inverse of GCD and then perform this transform. + // This transform can be done speculatively because it is so cheap - it + // results in a single rotate operation being inserted. // FIXME: It's possible that optimizing a switch on powers of two might also // be beneficial - flag values are often powers of two and we could use a CLZ // as the key function. - if (GCD <= 1 || !isPowerOf2_64(GCD)) - // No common divisor found or too expensive to compute key function. - return false; - unsigned Shift = Log2_64(GCD); + // countTrailingZeros(0) returns 64. As Values is guaranteed to have more than + // one element and LLVM disallows duplicate cases, Shift is guaranteed to be + // less than 64. + unsigned Shift = 64; for (auto &V : Values) - V = (int64_t)((uint64_t)V >> Shift); + Shift = std::min(Shift, countTrailingZeros((uint64_t)V)); + assert(Shift < 64); + if (Shift > 0) + for (auto &V : Values) + V = (int64_t)((uint64_t)V >> Shift); if (!isSwitchDense(Values)) // Transform didn't create a dense switch. @@ -5796,7 +5787,7 @@ bool SimplifyCFGOpt::SimplifyUncondBranch(BranchInst *BI, // branches to us and our successor, fold the comparison into the // predecessor and use logical operations to update the incoming value // for PHI nodes in common successor. - if (FoldBranchToCommonDest(BI, Options.BonusInstThreshold)) + if (FoldBranchToCommonDest(BI, nullptr, Options.BonusInstThreshold)) return requestResimplify(); return false; } @@ -5860,7 +5851,7 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) { // If this basic block is ONLY a compare and a branch, and if a predecessor // branches to us and one of our successors, fold the comparison into the // predecessor and use logical operations to pick the right destination. - if (FoldBranchToCommonDest(BI, Options.BonusInstThreshold)) + if (FoldBranchToCommonDest(BI, nullptr, Options.BonusInstThreshold)) return requestResimplify(); // We have a conditional branch to two blocks that are only reachable |