aboutsummaryrefslogtreecommitdiff
path: root/lib/Transforms/Utils/SimplifyCFG.cpp
diff options
context:
space:
mode:
authorDimitry Andric <dim@FreeBSD.org>2019-08-20 20:50:12 +0000
committerDimitry Andric <dim@FreeBSD.org>2019-08-20 20:50:12 +0000
commite6d1592492a3a379186bfb02bd0f4eda0669c0d5 (patch)
tree599ab169a01f1c86eda9adc774edaedde2f2db5b /lib/Transforms/Utils/SimplifyCFG.cpp
parent1a56a5ead7a2e84bee8240f5f6b033b5f1707154 (diff)
Diffstat (limited to 'lib/Transforms/Utils/SimplifyCFG.cpp')
-rw-r--r--lib/Transforms/Utils/SimplifyCFG.cpp203
1 files changed, 97 insertions, 106 deletions
diff --git a/lib/Transforms/Utils/SimplifyCFG.cpp b/lib/Transforms/Utils/SimplifyCFG.cpp
index 03b73954321d..11651d040dc0 100644
--- a/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -1,9 +1,8 @@
//===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
@@ -26,8 +25,9 @@
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/EHPersonalities.h"
#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
@@ -66,6 +66,7 @@
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/ValueMapper.h"
#include <algorithm>
#include <cassert>
@@ -292,9 +293,13 @@ isProfitableToFoldUnconditional(BranchInst *SI1, BranchInst *SI2,
/// will be the same as those coming in from ExistPred, an existing predecessor
/// of Succ.
static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
- BasicBlock *ExistPred) {
+ BasicBlock *ExistPred,
+ MemorySSAUpdater *MSSAU = nullptr) {
for (PHINode &PN : Succ->phis())
PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred);
+ if (MSSAU)
+ if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ))
+ MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred);
}
/// Compute an abstract "cost" of speculating the given instruction,
@@ -670,7 +675,8 @@ private:
} // end anonymous namespace
-static void EraseTerminatorAndDCECond(Instruction *TI) {
+static void EraseTerminatorAndDCECond(Instruction *TI,
+ MemorySSAUpdater *MSSAU = nullptr) {
Instruction *Cond = nullptr;
if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
Cond = dyn_cast<Instruction>(SI->getCondition());
@@ -683,7 +689,7 @@ static void EraseTerminatorAndDCECond(Instruction *TI) {
TI->eraseFromParent();
if (Cond)
- RecursivelyDeleteTriviallyDeadInstructions(Cond);
+ RecursivelyDeleteTriviallyDeadInstructions(Cond, nullptr, MSSAU);
}
/// Return true if the specified terminator checks
@@ -858,7 +864,7 @@ bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
return true;
}
- SwitchInst *SI = cast<SwitchInst>(TI);
+ SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI);
// Okay, TI has cases that are statically dead, prune them away.
SmallPtrSet<Constant *, 16> DeadCases;
for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
@@ -867,30 +873,13 @@ bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
<< "Through successor TI: " << *TI);
- // Collect branch weights into a vector.
- SmallVector<uint32_t, 8> Weights;
- MDNode *MD = SI->getMetadata(LLVMContext::MD_prof);
- bool HasWeight = MD && (MD->getNumOperands() == 2 + SI->getNumCases());
- if (HasWeight)
- for (unsigned MD_i = 1, MD_e = MD->getNumOperands(); MD_i < MD_e;
- ++MD_i) {
- ConstantInt *CI = mdconst::extract<ConstantInt>(MD->getOperand(MD_i));
- Weights.push_back(CI->getValue().getZExtValue());
- }
for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
--i;
if (DeadCases.count(i->getCaseValue())) {
- if (HasWeight) {
- std::swap(Weights[i->getCaseIndex() + 1], Weights.back());
- Weights.pop_back();
- }
i->getCaseSuccessor()->removePredecessor(TI->getParent());
- SI->removeCase(i);
+ SI.removeCase(i);
}
}
- if (HasWeight && Weights.size() >= 2)
- setBranchWeights(SI, Weights);
-
LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n");
return true;
}
@@ -1266,8 +1255,10 @@ static bool HoistThenElseCodeToIf(BranchInst *BI,
while (isa<DbgInfoIntrinsic>(I2))
I2 = &*BB2_Itr++;
}
+ // FIXME: Can we define a safety predicate for CallBr?
if (isa<PHINode>(I1) || !I1->isIdenticalToWhenDefined(I2) ||
- (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)))
+ (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2)) ||
+ isa<CallBrInst>(I1))
return false;
BasicBlock *BIParent = BI->getParent();
@@ -1350,9 +1341,14 @@ static bool HoistThenElseCodeToIf(BranchInst *BI,
HoistTerminator:
// It may not be possible to hoist an invoke.
+ // FIXME: Can we define a safety predicate for CallBr?
if (isa<InvokeInst>(I1) && !isSafeToHoistInvoke(BB1, BB2, I1, I2))
return Changed;
+ // TODO: callbr hoisting currently disabled pending further study.
+ if (isa<CallBrInst>(I1))
+ return Changed;
+
for (BasicBlock *Succ : successors(BB1)) {
for (PHINode &PN : Succ->phis()) {
Value *BB1V = PN.getIncomingValueForBlock(BB1);
@@ -1432,9 +1428,10 @@ HoistTerminator:
static bool canSinkInstructions(
ArrayRef<Instruction *> Insts,
DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) {
- // Prune out obviously bad instructions to move. Any non-store instruction
- // must have exactly one use, and we check later that use is by a single,
- // common PHI instruction in the successor.
+ // Prune out obviously bad instructions to move. Each instruction must have
+ // exactly zero or one use, and we check later that use is by a single, common
+ // PHI instruction in the successor.
+ bool HasUse = !Insts.front()->user_empty();
for (auto *I : Insts) {
// These instructions may change or break semantics if moved.
if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) ||
@@ -1444,13 +1441,14 @@ static bool canSinkInstructions(
// Conservatively return false if I is an inline-asm instruction. Sinking
// and merging inline-asm instructions can potentially create arguments
// that cannot satisfy the inline-asm constraints.
- if (const auto *C = dyn_cast<CallInst>(I))
+ if (const auto *C = dyn_cast<CallBase>(I))
if (C->isInlineAsm())
return false;
- // Everything must have only one use too, apart from stores which
- // have no uses.
- if (!isa<StoreInst>(I) && !I->hasOneUse())
+ // Each instruction must have zero or one use.
+ if (HasUse && !I->hasOneUse())
+ return false;
+ if (!HasUse && !I->user_empty())
return false;
}
@@ -1459,11 +1457,11 @@ static bool canSinkInstructions(
if (!I->isSameOperationAs(I0))
return false;
- // All instructions in Insts are known to be the same opcode. If they aren't
- // stores, check the only user of each is a PHI or in the same block as the
- // instruction, because if a user is in the same block as an instruction
- // we're contemplating sinking, it must already be determined to be sinkable.
- if (!isa<StoreInst>(I0)) {
+ // All instructions in Insts are known to be the same opcode. If they have a
+ // use, check that the only user is a PHI or in the same block as the
+ // instruction, because if a user is in the same block as an instruction we're
+ // contemplating sinking, it must already be determined to be sinkable.
+ if (HasUse) {
auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0);
if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool {
@@ -1507,7 +1505,7 @@ static bool canSinkInstructions(
// We can't create a PHI from this GEP.
return false;
// Don't create indirect calls! The called value is the final operand.
- if ((isa<CallInst>(I0) || isa<InvokeInst>(I0)) && OI == OE - 1) {
+ if (isa<CallBase>(I0) && OI == OE - 1) {
// FIXME: if the call was *already* indirect, we should do this.
return false;
}
@@ -1541,7 +1539,7 @@ static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
// it is slightly over-aggressive - it gets confused by commutative instructions
// so double-check it here.
Instruction *I0 = Insts.front();
- if (!isa<StoreInst>(I0)) {
+ if (!I0->user_empty()) {
auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool {
auto *U = cast<Instruction>(*I->user_begin());
@@ -1599,11 +1597,10 @@ static bool sinkLastInstruction(ArrayRef<BasicBlock*> Blocks) {
I0->andIRFlags(I);
}
- if (!isa<StoreInst>(I0)) {
+ if (!I0->user_empty()) {
// canSinkLastInstruction checked that all instructions were used by
// one and only one PHI node. Find that now, RAUW it to our common
// instruction and nuke it.
- assert(I0->hasOneUse());
auto *PN = cast<PHINode>(*I0->user_begin());
PN->replaceAllUsesWith(I0);
PN->eraseFromParent();
@@ -2203,7 +2200,8 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout &DL,
BasicBlock *EdgeBB =
BasicBlock::Create(BB->getContext(), RealDest->getName() + ".critedge",
RealDest->getParent(), RealDest);
- BranchInst::Create(RealDest, EdgeBB);
+ BranchInst *CritEdgeBranch = BranchInst::Create(RealDest, EdgeBB);
+ CritEdgeBranch->setDebugLoc(BI->getDebugLoc());
// Update PHI nodes.
AddPredecessorToBlock(RealDest, EdgeBB, BB);
@@ -2539,7 +2537,8 @@ static bool extractPredSuccWeights(BranchInst *PBI, BranchInst *BI,
/// If this basic block is simple enough, and if a predecessor branches to us
/// and one of our successors, fold the block into the predecessor and use
/// logical operations to pick the right destination.
-bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
+bool llvm::FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU,
+ unsigned BonusInstThreshold) {
BasicBlock *BB = BI->getParent();
const unsigned PredCount = pred_size(BB);
@@ -2594,7 +2593,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
// unconditionally. We denote all involved instructions except the condition
// as "bonus instructions", and only allow this transformation when the
// number of the bonus instructions we'll need to create when cloning into
- // each predecessor does not exceed a certain threshold.
+ // each predecessor does not exceed a certain threshold.
unsigned NumBonusInsts = 0;
for (auto I = BB->begin(); Cond != &*I; ++I) {
// Ignore dbg intrinsics.
@@ -2611,7 +2610,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
// and Cond.
// Account for the cost of duplicating this instruction into each
- // predecessor.
+ // predecessor.
NumBonusInsts += PredCount;
// Early exits once we reach the limit.
if (NumBonusInsts > BonusInstThreshold)
@@ -2750,7 +2749,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
(SuccFalseWeight + SuccTrueWeight) +
PredTrueWeight * SuccFalseWeight);
}
- AddPredecessorToBlock(TrueDest, PredBlock, BB);
+ AddPredecessorToBlock(TrueDest, PredBlock, BB, MSSAU);
PBI->setSuccessor(0, TrueDest);
}
if (PBI->getSuccessor(1) == BB) {
@@ -2765,7 +2764,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
// FalseWeight is FalseWeight for PBI * FalseWeight for BI.
NewWeights.push_back(PredFalseWeight * SuccFalseWeight);
}
- AddPredecessorToBlock(FalseDest, PredBlock, BB);
+ AddPredecessorToBlock(FalseDest, PredBlock, BB, MSSAU);
PBI->setSuccessor(1, FalseDest);
}
if (NewWeights.size() == 2) {
@@ -2810,12 +2809,17 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold) {
}
}
// Update PHI Node.
- PHIs[i]->setIncomingValue(PHIs[i]->getBasicBlockIndex(PBI->getParent()),
- MergedCond);
+ PHIs[i]->setIncomingValueForBlock(PBI->getParent(), MergedCond);
}
+
+ // PBI is changed to branch to TrueDest below. Remove itself from
+ // potential phis from all other successors.
+ if (MSSAU)
+ MSSAU->changeCondBranchToUnconditionalTo(PBI, TrueDest);
+
// Change PBI from Conditional to Unconditional.
BranchInst *New_PBI = BranchInst::Create(TrueDest, PBI);
- EraseTerminatorAndDCECond(PBI);
+ EraseTerminatorAndDCECond(PBI, MSSAU);
PBI = New_PBI;
}
@@ -3430,7 +3434,7 @@ static bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
KeepEdge2 = nullptr;
else
Succ->removePredecessor(OldTerm->getParent(),
- /*DontDeleteUselessPHIs=*/true);
+ /*KeepOneInputPHIs=*/true);
}
IRBuilder<> Builder(OldTerm);
@@ -3622,20 +3626,16 @@ bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt(
// the switch to the merge point on the compared value.
BasicBlock *NewBB =
BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB);
- SmallVector<uint64_t, 8> Weights;
- bool HasWeights = HasBranchWeights(SI);
- if (HasWeights) {
- GetBranchWeights(SI, Weights);
- if (Weights.size() == 1 + SI->getNumCases()) {
- // Split weight for default case to case for "Cst".
- Weights[0] = (Weights[0] + 1) >> 1;
- Weights.push_back(Weights[0]);
-
- SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
- setBranchWeights(SI, MDWeights);
+ {
+ SwitchInstProfUpdateWrapper SIW(*SI);
+ auto W0 = SIW.getSuccessorWeight(0);
+ SwitchInstProfUpdateWrapper::CaseWeightOpt NewW;
+ if (W0) {
+ NewW = ((uint64_t(*W0) + 1) >> 1);
+ SIW.setSuccessorWeight(0, *NewW);
}
+ SIW.addCase(Cst, NewBB, NewW);
}
- SI->addCase(Cst, NewBB);
// NewBB branches to the phi block, add the uncond branch and the phi entry.
Builder.SetInsertPoint(NewBB);
@@ -4184,24 +4184,28 @@ bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) {
Changed = true;
}
} else {
+ Value* Cond = BI->getCondition();
if (BI->getSuccessor(0) == BB) {
+ Builder.CreateAssumption(Builder.CreateNot(Cond));
Builder.CreateBr(BI->getSuccessor(1));
EraseTerminatorAndDCECond(BI);
} else if (BI->getSuccessor(1) == BB) {
+ Builder.CreateAssumption(Cond);
Builder.CreateBr(BI->getSuccessor(0));
EraseTerminatorAndDCECond(BI);
Changed = true;
}
}
} else if (auto *SI = dyn_cast<SwitchInst>(TI)) {
- for (auto i = SI->case_begin(), e = SI->case_end(); i != e;) {
+ SwitchInstProfUpdateWrapper SU(*SI);
+ for (auto i = SU->case_begin(), e = SU->case_end(); i != e;) {
if (i->getCaseSuccessor() != BB) {
++i;
continue;
}
- BB->removePredecessor(SI->getParent());
- i = SI->removeCase(i);
- e = SI->case_end();
+ BB->removePredecessor(SU->getParent());
+ i = SU.removeCase(i);
+ e = SU->case_end();
Changed = true;
}
} else if (auto *II = dyn_cast<InvokeInst>(TI)) {
@@ -4435,33 +4439,20 @@ static bool eliminateDeadSwitchCases(SwitchInst *SI, AssumptionCache *AC,
return true;
}
- SmallVector<uint64_t, 8> Weights;
- bool HasWeight = HasBranchWeights(SI);
- if (HasWeight) {
- GetBranchWeights(SI, Weights);
- HasWeight = (Weights.size() == 1 + SI->getNumCases());
- }
+ if (DeadCases.empty())
+ return false;
- // Remove dead cases from the switch.
+ SwitchInstProfUpdateWrapper SIW(*SI);
for (ConstantInt *DeadCase : DeadCases) {
SwitchInst::CaseIt CaseI = SI->findCaseValue(DeadCase);
assert(CaseI != SI->case_default() &&
"Case was not found. Probably mistake in DeadCases forming.");
- if (HasWeight) {
- std::swap(Weights[CaseI->getCaseIndex() + 1], Weights.back());
- Weights.pop_back();
- }
-
// Prune unused values from PHI nodes.
CaseI->getCaseSuccessor()->removePredecessor(SI->getParent());
- SI->removeCase(CaseI);
- }
- if (HasWeight && Weights.size() >= 2) {
- SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
- setBranchWeights(SI, MDWeights);
+ SIW.removeCase(CaseI);
}
- return !DeadCases.empty();
+ return true;
}
/// If BB would be eligible for simplification by
@@ -5034,7 +5025,7 @@ SwitchLookupTable::SwitchLookupTable(
ArrayType *ArrayTy = ArrayType::get(ValueType, TableSize);
Constant *Initializer = ConstantArray::get(ArrayTy, TableContents);
- Array = new GlobalVariable(M, ArrayTy, /*constant=*/true,
+ Array = new GlobalVariable(M, ArrayTy, /*isConstant=*/true,
GlobalVariable::PrivateLinkage, Initializer,
"switch.table." + FuncName);
Array->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
@@ -5091,7 +5082,9 @@ Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) {
Value *GEPIndices[] = {Builder.getInt32(0), Index};
Value *GEP = Builder.CreateInBoundsGEP(Array->getValueType(), Array,
GEPIndices, "switch.gep");
- return Builder.CreateLoad(GEP, "switch.load");
+ return Builder.CreateLoad(
+ cast<ArrayType>(Array->getValueType())->getElementType(), GEP,
+ "switch.load");
}
}
llvm_unreachable("Unknown lookup table kind!");
@@ -5425,7 +5418,7 @@ static bool SwitchToLookupTable(SwitchInst *SI, IRBuilder<> &Builder,
// We cached PHINodes in PHIs. To avoid accessing deleted PHINodes later,
// do not delete PHINodes here.
SI->getDefaultDest()->removePredecessor(SI->getParent(),
- /*DontDeleteUselessPHIs=*/true);
+ /*KeepOneInputPHIs=*/true);
}
bool ReturnedEarly = false;
@@ -5533,25 +5526,23 @@ static bool ReduceSwitchRange(SwitchInst *SI, IRBuilder<> &Builder,
// Now we have signed numbers that have been shifted so that, given enough
// precision, there are no negative values. Since the rest of the transform
// is bitwise only, we switch now to an unsigned representation.
- uint64_t GCD = 0;
- for (auto &V : Values)
- GCD = GreatestCommonDivisor64(GCD, (uint64_t)V);
- // This transform can be done speculatively because it is so cheap - it results
- // in a single rotate operation being inserted. This can only happen if the
- // factor extracted is a power of 2.
- // FIXME: If the GCD is an odd number we can multiply by the multiplicative
- // inverse of GCD and then perform this transform.
+ // This transform can be done speculatively because it is so cheap - it
+ // results in a single rotate operation being inserted.
// FIXME: It's possible that optimizing a switch on powers of two might also
// be beneficial - flag values are often powers of two and we could use a CLZ
// as the key function.
- if (GCD <= 1 || !isPowerOf2_64(GCD))
- // No common divisor found or too expensive to compute key function.
- return false;
- unsigned Shift = Log2_64(GCD);
+ // countTrailingZeros(0) returns 64. As Values is guaranteed to have more than
+ // one element and LLVM disallows duplicate cases, Shift is guaranteed to be
+ // less than 64.
+ unsigned Shift = 64;
for (auto &V : Values)
- V = (int64_t)((uint64_t)V >> Shift);
+ Shift = std::min(Shift, countTrailingZeros((uint64_t)V));
+ assert(Shift < 64);
+ if (Shift > 0)
+ for (auto &V : Values)
+ V = (int64_t)((uint64_t)V >> Shift);
if (!isSwitchDense(Values))
// Transform didn't create a dense switch.
@@ -5796,7 +5787,7 @@ bool SimplifyCFGOpt::SimplifyUncondBranch(BranchInst *BI,
// branches to us and our successor, fold the comparison into the
// predecessor and use logical operations to update the incoming value
// for PHI nodes in common successor.
- if (FoldBranchToCommonDest(BI, Options.BonusInstThreshold))
+ if (FoldBranchToCommonDest(BI, nullptr, Options.BonusInstThreshold))
return requestResimplify();
return false;
}
@@ -5860,7 +5851,7 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
// If this basic block is ONLY a compare and a branch, and if a predecessor
// branches to us and one of our successors, fold the comparison into the
// predecessor and use logical operations to pick the right destination.
- if (FoldBranchToCommonDest(BI, Options.BonusInstThreshold))
+ if (FoldBranchToCommonDest(BI, nullptr, Options.BonusInstThreshold))
return requestResimplify();
// We have a conditional branch to two blocks that are only reachable