aboutsummaryrefslogtreecommitdiff
path: root/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'lib/Transforms/Scalar/CorrelatedValuePropagation.cpp')
-rw-r--r--lib/Transforms/Scalar/CorrelatedValuePropagation.cpp305
1 files changed, 161 insertions, 144 deletions
diff --git a/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index d0105701c73f..89497177524f 100644
--- a/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -1,9 +1,8 @@
//===- CorrelatedValuePropagation.cpp - Propagate CFG-derived info --------===//
//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
@@ -16,6 +15,7 @@
#include "llvm/ADT/Optional.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/LazyValueInfo.h"
@@ -27,7 +27,6 @@
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
-#include "llvm/IR/DomTreeUpdater.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstrTypes.h"
@@ -64,8 +63,10 @@ STATISTIC(NumUDivs, "Number of udivs whose width was decreased");
STATISTIC(NumAShrs, "Number of ashr converted to lshr");
STATISTIC(NumSRems, "Number of srem converted to urem");
STATISTIC(NumOverflows, "Number of overflow checks removed");
+STATISTIC(NumSaturating,
+ "Number of saturating arithmetics converted to normal arithmetics");
-static cl::opt<bool> DontProcessAdds("cvp-dont-process-adds", cl::init(true));
+static cl::opt<bool> DontAddNoWrapFlags("cvp-dont-add-nowrap-flags", cl::init(false));
namespace {
@@ -307,11 +308,11 @@ static bool processCmp(CmpInst *Cmp, LazyValueInfo *LVI) {
/// that cannot fire no matter what the incoming edge can safely be removed. If
/// a case fires on every incoming edge then the entire switch can be removed
/// and replaced with a branch to the case destination.
-static bool processSwitch(SwitchInst *SI, LazyValueInfo *LVI,
+static bool processSwitch(SwitchInst *I, LazyValueInfo *LVI,
DominatorTree *DT) {
DomTreeUpdater DTU(*DT, DomTreeUpdater::UpdateStrategy::Lazy);
- Value *Cond = SI->getCondition();
- BasicBlock *BB = SI->getParent();
+ Value *Cond = I->getCondition();
+ BasicBlock *BB = I->getParent();
// If the condition was defined in same block as the switch then LazyValueInfo
// currently won't say anything useful about it, though in theory it could.
@@ -328,67 +329,72 @@ static bool processSwitch(SwitchInst *SI, LazyValueInfo *LVI,
for (auto *Succ : successors(BB))
SuccessorsCount[Succ]++;
- for (auto CI = SI->case_begin(), CE = SI->case_end(); CI != CE;) {
- ConstantInt *Case = CI->getCaseValue();
-
- // Check to see if the switch condition is equal to/not equal to the case
- // value on every incoming edge, equal/not equal being the same each time.
- LazyValueInfo::Tristate State = LazyValueInfo::Unknown;
- for (pred_iterator PI = PB; PI != PE; ++PI) {
- // Is the switch condition equal to the case value?
- LazyValueInfo::Tristate Value = LVI->getPredicateOnEdge(CmpInst::ICMP_EQ,
- Cond, Case, *PI,
- BB, SI);
- // Give up on this case if nothing is known.
- if (Value == LazyValueInfo::Unknown) {
- State = LazyValueInfo::Unknown;
- break;
+ { // Scope for SwitchInstProfUpdateWrapper. It must not live during
+ // ConstantFoldTerminator() as the underlying SwitchInst can be changed.
+ SwitchInstProfUpdateWrapper SI(*I);
+
+ for (auto CI = SI->case_begin(), CE = SI->case_end(); CI != CE;) {
+ ConstantInt *Case = CI->getCaseValue();
+
+ // Check to see if the switch condition is equal to/not equal to the case
+ // value on every incoming edge, equal/not equal being the same each time.
+ LazyValueInfo::Tristate State = LazyValueInfo::Unknown;
+ for (pred_iterator PI = PB; PI != PE; ++PI) {
+ // Is the switch condition equal to the case value?
+ LazyValueInfo::Tristate Value = LVI->getPredicateOnEdge(CmpInst::ICMP_EQ,
+ Cond, Case, *PI,
+ BB, SI);
+ // Give up on this case if nothing is known.
+ if (Value == LazyValueInfo::Unknown) {
+ State = LazyValueInfo::Unknown;
+ break;
+ }
+
+ // If this was the first edge to be visited, record that all other edges
+ // need to give the same result.
+ if (PI == PB) {
+ State = Value;
+ continue;
+ }
+
+ // If this case is known to fire for some edges and known not to fire for
+ // others then there is nothing we can do - give up.
+ if (Value != State) {
+ State = LazyValueInfo::Unknown;
+ break;
+ }
}
- // If this was the first edge to be visited, record that all other edges
- // need to give the same result.
- if (PI == PB) {
- State = Value;
+ if (State == LazyValueInfo::False) {
+ // This case never fires - remove it.
+ BasicBlock *Succ = CI->getCaseSuccessor();
+ Succ->removePredecessor(BB);
+ CI = SI.removeCase(CI);
+ CE = SI->case_end();
+
+ // The condition can be modified by removePredecessor's PHI simplification
+ // logic.
+ Cond = SI->getCondition();
+
+ ++NumDeadCases;
+ Changed = true;
+ if (--SuccessorsCount[Succ] == 0)
+ DTU.applyUpdatesPermissive({{DominatorTree::Delete, BB, Succ}});
continue;
}
-
- // If this case is known to fire for some edges and known not to fire for
- // others then there is nothing we can do - give up.
- if (Value != State) {
- State = LazyValueInfo::Unknown;
+ if (State == LazyValueInfo::True) {
+ // This case always fires. Arrange for the switch to be turned into an
+ // unconditional branch by replacing the switch condition with the case
+ // value.
+ SI->setCondition(Case);
+ NumDeadCases += SI->getNumCases();
+ Changed = true;
break;
}
- }
- if (State == LazyValueInfo::False) {
- // This case never fires - remove it.
- BasicBlock *Succ = CI->getCaseSuccessor();
- Succ->removePredecessor(BB);
- CI = SI->removeCase(CI);
- CE = SI->case_end();
-
- // The condition can be modified by removePredecessor's PHI simplification
- // logic.
- Cond = SI->getCondition();
-
- ++NumDeadCases;
- Changed = true;
- if (--SuccessorsCount[Succ] == 0)
- DTU.deleteEdge(BB, Succ);
- continue;
- }
- if (State == LazyValueInfo::True) {
- // This case always fires. Arrange for the switch to be turned into an
- // unconditional branch by replacing the switch condition with the case
- // value.
- SI->setCondition(Case);
- NumDeadCases += SI->getNumCases();
- Changed = true;
- break;
+ // Increment the case iterator since we didn't delete it.
+ ++CI;
}
-
- // Increment the case iterator since we didn't delete it.
- ++CI;
}
if (Changed)
@@ -399,56 +405,48 @@ static bool processSwitch(SwitchInst *SI, LazyValueInfo *LVI,
return Changed;
}
-// See if we can prove that the given overflow intrinsic will not overflow.
-static bool willNotOverflow(IntrinsicInst *II, LazyValueInfo *LVI) {
- using OBO = OverflowingBinaryOperator;
- auto NoWrap = [&] (Instruction::BinaryOps BinOp, unsigned NoWrapKind) {
- Value *RHS = II->getOperand(1);
- ConstantRange RRange = LVI->getConstantRange(RHS, II->getParent(), II);
- ConstantRange NWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
- BinOp, RRange, NoWrapKind);
- // As an optimization, do not compute LRange if we do not need it.
- if (NWRegion.isEmptySet())
- return false;
- Value *LHS = II->getOperand(0);
- ConstantRange LRange = LVI->getConstantRange(LHS, II->getParent(), II);
- return NWRegion.contains(LRange);
- };
- switch (II->getIntrinsicID()) {
- default:
- break;
- case Intrinsic::uadd_with_overflow:
- return NoWrap(Instruction::Add, OBO::NoUnsignedWrap);
- case Intrinsic::sadd_with_overflow:
- return NoWrap(Instruction::Add, OBO::NoSignedWrap);
- case Intrinsic::usub_with_overflow:
- return NoWrap(Instruction::Sub, OBO::NoUnsignedWrap);
- case Intrinsic::ssub_with_overflow:
- return NoWrap(Instruction::Sub, OBO::NoSignedWrap);
- }
- return false;
+// See if we can prove that the given binary op intrinsic will not overflow.
+static bool willNotOverflow(BinaryOpIntrinsic *BO, LazyValueInfo *LVI) {
+ ConstantRange LRange = LVI->getConstantRange(
+ BO->getLHS(), BO->getParent(), BO);
+ ConstantRange RRange = LVI->getConstantRange(
+ BO->getRHS(), BO->getParent(), BO);
+ ConstantRange NWRegion = ConstantRange::makeGuaranteedNoWrapRegion(
+ BO->getBinaryOp(), RRange, BO->getNoWrapKind());
+ return NWRegion.contains(LRange);
}
-static void processOverflowIntrinsic(IntrinsicInst *II) {
- IRBuilder<> B(II);
- Value *NewOp = nullptr;
- switch (II->getIntrinsicID()) {
- default:
- llvm_unreachable("Unexpected instruction.");
- case Intrinsic::uadd_with_overflow:
- case Intrinsic::sadd_with_overflow:
- NewOp = B.CreateAdd(II->getOperand(0), II->getOperand(1), II->getName());
- break;
- case Intrinsic::usub_with_overflow:
- case Intrinsic::ssub_with_overflow:
- NewOp = B.CreateSub(II->getOperand(0), II->getOperand(1), II->getName());
- break;
+static void processOverflowIntrinsic(WithOverflowInst *WO) {
+ IRBuilder<> B(WO);
+ Value *NewOp = B.CreateBinOp(
+ WO->getBinaryOp(), WO->getLHS(), WO->getRHS(), WO->getName());
+ // Constant-folding could have happened.
+ if (auto *Inst = dyn_cast<Instruction>(NewOp)) {
+ if (WO->isSigned())
+ Inst->setHasNoSignedWrap();
+ else
+ Inst->setHasNoUnsignedWrap();
}
+
+ Value *NewI = B.CreateInsertValue(UndefValue::get(WO->getType()), NewOp, 0);
+ NewI = B.CreateInsertValue(NewI, ConstantInt::getFalse(WO->getContext()), 1);
+ WO->replaceAllUsesWith(NewI);
+ WO->eraseFromParent();
++NumOverflows;
- Value *NewI = B.CreateInsertValue(UndefValue::get(II->getType()), NewOp, 0);
- NewI = B.CreateInsertValue(NewI, ConstantInt::getFalse(II->getContext()), 1);
- II->replaceAllUsesWith(NewI);
- II->eraseFromParent();
+}
+
+static void processSaturatingInst(SaturatingInst *SI) {
+ BinaryOperator *BinOp = BinaryOperator::Create(
+ SI->getBinaryOp(), SI->getLHS(), SI->getRHS(), SI->getName(), SI);
+ BinOp->setDebugLoc(SI->getDebugLoc());
+ if (SI->isSigned())
+ BinOp->setHasNoSignedWrap();
+ else
+ BinOp->setHasNoUnsignedWrap();
+
+ SI->replaceAllUsesWith(BinOp);
+ SI->eraseFromParent();
+ ++NumSaturating;
}
/// Infer nonnull attributes for the arguments at the specified callsite.
@@ -456,13 +454,44 @@ static bool processCallSite(CallSite CS, LazyValueInfo *LVI) {
SmallVector<unsigned, 4> ArgNos;
unsigned ArgNo = 0;
- if (auto *II = dyn_cast<IntrinsicInst>(CS.getInstruction())) {
- if (willNotOverflow(II, LVI)) {
- processOverflowIntrinsic(II);
+ if (auto *WO = dyn_cast<WithOverflowInst>(CS.getInstruction())) {
+ if (WO->getLHS()->getType()->isIntegerTy() && willNotOverflow(WO, LVI)) {
+ processOverflowIntrinsic(WO);
+ return true;
+ }
+ }
+
+ if (auto *SI = dyn_cast<SaturatingInst>(CS.getInstruction())) {
+ if (SI->getType()->isIntegerTy() && willNotOverflow(SI, LVI)) {
+ processSaturatingInst(SI);
return true;
}
}
+ // Deopt bundle operands are intended to capture state with minimal
+ // perturbance of the code otherwise. If we can find a constant value for
+ // any such operand and remove a use of the original value, that's
+ // desireable since it may allow further optimization of that value (e.g. via
+ // single use rules in instcombine). Since deopt uses tend to,
+ // idiomatically, appear along rare conditional paths, it's reasonable likely
+ // we may have a conditional fact with which LVI can fold.
+ if (auto DeoptBundle = CS.getOperandBundle(LLVMContext::OB_deopt)) {
+ bool Progress = false;
+ for (const Use &ConstU : DeoptBundle->Inputs) {
+ Use &U = const_cast<Use&>(ConstU);
+ Value *V = U.get();
+ if (V->getType()->isVectorTy()) continue;
+ if (isa<Constant>(V)) continue;
+
+ Constant *C = LVI->getConstant(V, CS.getParent(), CS.getInstruction());
+ if (!C) continue;
+ U.set(C);
+ Progress = true;
+ }
+ if (Progress)
+ return true;
+ }
+
for (Value *V : CS.args()) {
PointerType *Type = dyn_cast<PointerType>(V->getType());
// Try to mark pointer typed parameters as non-null. We skip the
@@ -512,7 +541,7 @@ static bool processUDivOrURem(BinaryOperator *Instr, LazyValueInfo *LVI) {
// Find the smallest power of two bitwidth that's sufficient to hold Instr's
// operands.
auto OrigWidth = Instr->getType()->getIntegerBitWidth();
- ConstantRange OperandRange(OrigWidth, /*isFullset=*/false);
+ ConstantRange OperandRange(OrigWidth, /*isFullSet=*/false);
for (Value *Operand : Instr->operands()) {
OperandRange = OperandRange.unionWith(
LVI->getConstantRange(Operand, Instr->getParent()));
@@ -603,55 +632,42 @@ static bool processAShr(BinaryOperator *SDI, LazyValueInfo *LVI) {
return true;
}
-static bool processAdd(BinaryOperator *AddOp, LazyValueInfo *LVI) {
+static bool processBinOp(BinaryOperator *BinOp, LazyValueInfo *LVI) {
using OBO = OverflowingBinaryOperator;
- if (DontProcessAdds)
+ if (DontAddNoWrapFlags)
return false;
- if (AddOp->getType()->isVectorTy())
+ if (BinOp->getType()->isVectorTy())
return false;
- bool NSW = AddOp->hasNoSignedWrap();
- bool NUW = AddOp->hasNoUnsignedWrap();
+ bool NSW = BinOp->hasNoSignedWrap();
+ bool NUW = BinOp->hasNoUnsignedWrap();
if (NSW && NUW)
return false;
- BasicBlock *BB = AddOp->getParent();
+ BasicBlock *BB = BinOp->getParent();
- Value *LHS = AddOp->getOperand(0);
- Value *RHS = AddOp->getOperand(1);
+ Value *LHS = BinOp->getOperand(0);
+ Value *RHS = BinOp->getOperand(1);
- ConstantRange LRange = LVI->getConstantRange(LHS, BB, AddOp);
-
- // Initialize RRange only if we need it. If we know that guaranteed no wrap
- // range for the given LHS range is empty don't spend time calculating the
- // range for the RHS.
- Optional<ConstantRange> RRange;
- auto LazyRRange = [&] () {
- if (!RRange)
- RRange = LVI->getConstantRange(RHS, BB, AddOp);
- return RRange.getValue();
- };
+ ConstantRange LRange = LVI->getConstantRange(LHS, BB, BinOp);
+ ConstantRange RRange = LVI->getConstantRange(RHS, BB, BinOp);
bool Changed = false;
if (!NUW) {
ConstantRange NUWRange = ConstantRange::makeGuaranteedNoWrapRegion(
- BinaryOperator::Add, LRange, OBO::NoUnsignedWrap);
- if (!NUWRange.isEmptySet()) {
- bool NewNUW = NUWRange.contains(LazyRRange());
- AddOp->setHasNoUnsignedWrap(NewNUW);
- Changed |= NewNUW;
- }
+ BinOp->getOpcode(), RRange, OBO::NoUnsignedWrap);
+ bool NewNUW = NUWRange.contains(LRange);
+ BinOp->setHasNoUnsignedWrap(NewNUW);
+ Changed |= NewNUW;
}
if (!NSW) {
ConstantRange NSWRange = ConstantRange::makeGuaranteedNoWrapRegion(
- BinaryOperator::Add, LRange, OBO::NoSignedWrap);
- if (!NSWRange.isEmptySet()) {
- bool NewNSW = NSWRange.contains(LazyRRange());
- AddOp->setHasNoSignedWrap(NewNSW);
- Changed |= NewNSW;
- }
+ BinOp->getOpcode(), RRange, OBO::NoSignedWrap);
+ bool NewNSW = NSWRange.contains(LRange);
+ BinOp->setHasNoSignedWrap(NewNSW);
+ Changed |= NewNSW;
}
return Changed;
@@ -725,7 +741,8 @@ static bool runImpl(Function &F, LazyValueInfo *LVI, DominatorTree *DT,
BBChanged |= processAShr(cast<BinaryOperator>(II), LVI);
break;
case Instruction::Add:
- BBChanged |= processAdd(cast<BinaryOperator>(II), LVI);
+ case Instruction::Sub:
+ BBChanged |= processBinOp(cast<BinaryOperator>(II), LVI);
break;
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-08 15:40:15 +0000