vendor/llvm-project/llvmorg-13-init-16847-g88e66fa60ae5vendor/llvm-project/llvmorg-12.0.1-rc2-0-ge7dac564cd0evendor/llvm-project/llvmorg-12.0.1-0-gfed41342a82f

63 files changed, 7245 insertions, 5588 deletions

diff --git a/llvm/lib/Transforms/Scalar/ADCE.cpp b/llvm/lib/Transforms/Scalar/ADCE.cpp
index 2b649732a799..6f3fdb88eda5 100644
--- a/llvm/lib/Transforms/Scalar/ADCE.cpp
+++ b/llvm/lib/Transforms/Scalar/ADCE.cpp
@@ -50,6 +50,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/Local.h"
 #include <cassert>
 #include <cstddef>
 #include <utility>
@@ -521,10 +522,14 @@ bool AggressiveDeadCodeElimination::removeDeadInstructions() {
         // If intrinsic is pointing at a live SSA value, there may be an
         // earlier optimization bug: if we know the location of the variable,
         // why isn't the scope of the location alive?
-        if (Value *V = DII->getVariableLocation())
-          if (Instruction *II = dyn_cast<Instruction>(V))
-            if (isLive(II))
+        for (Value *V : DII->location_ops()) {
+          if (Instruction *II = dyn_cast<Instruction>(V)) {
+            if (isLive(II)) {
               dbgs() << "Dropping debug info for " << *DII << "\n";
+              break;
+            }
+          }
+        }
       }
     }
   });
@@ -548,6 +553,7 @@ bool AggressiveDeadCodeElimination::removeDeadInstructions() {
 
     // Prepare to delete.
     Worklist.push_back(&I);
+    salvageDebugInfo(I);
     I.dropAllReferences();
   }
 
@@ -696,7 +702,6 @@ PreservedAnalyses ADCEPass::run(Function &F, FunctionAnalysisManager &FAM) {
     PA.preserve<DominatorTreeAnalysis>();
     PA.preserve<PostDominatorTreeAnalysis>();
   }
-  PA.preserve<GlobalsAA>();
   return PA;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
index bccf94fc217f..be21db9087d2 100644
--- a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
+++ b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp
@@ -17,8 +17,6 @@
 
 #include "llvm/IR/Instructions.h"
 #include "llvm/InitializePasses.h"
-#define AA_NAME "alignment-from-assumptions"
-#define DEBUG_TYPE AA_NAME
 #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
@@ -37,6 +35,9 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
+
+#define AA_NAME "alignment-from-assumptions"
+#define DEBUG_TYPE AA_NAME
 using namespace llvm;
 
 STATISTIC(NumLoadAlignChanged,
@@ -134,6 +135,8 @@ static Align getNewAlignment(const SCEV *AASCEV, const SCEV *AlignSCEV,
   PtrSCEV = SE->getTruncateOrZeroExtend(
       PtrSCEV, SE->getEffectiveSCEVType(AASCEV->getType()));
   const SCEV *DiffSCEV = SE->getMinusSCEV(PtrSCEV, AASCEV);
+  if (isa<SCEVCouldNotCompute>(DiffSCEV))
+    return Align(1);
 
   // On 32-bit platforms, DiffSCEV might now have type i32 -- we've always
   // sign-extended OffSCEV to i64, so make sure they agree again.
@@ -141,7 +144,7 @@ static Align getNewAlignment(const SCEV *AASCEV, const SCEV *AlignSCEV,
 
   // What we really want to know is the overall offset to the aligned
   // address. This address is displaced by the provided offset.
-  DiffSCEV = SE->getMinusSCEV(DiffSCEV, OffSCEV);
+  DiffSCEV = SE->getAddExpr(DiffSCEV, OffSCEV);
 
   LLVM_DEBUG(dbgs() << "AFI: alignment of " << *Ptr << " relative to "
                     << *AlignSCEV << " and offset " << *OffSCEV
@@ -352,8 +355,6 @@ AlignmentFromAssumptionsPass::run(Function &F, FunctionAnalysisManager &AM) {
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
-  PA.preserve<AAManager>();
   PA.preserve<ScalarEvolutionAnalysis>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
diff --git a/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp b/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
index a02d88fe066f..a5e65ffc45fe 100644
--- a/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
+++ b/llvm/lib/Transforms/Scalar/AnnotationRemarks.cpp
@@ -16,10 +16,13 @@
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/InstIterator.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/MemoryOpRemark.h"
 
 using namespace llvm;
 using namespace llvm::ore;
@@ -27,16 +30,37 @@ using namespace llvm::ore;
 #define DEBUG_TYPE "annotation-remarks"
 #define REMARK_PASS DEBUG_TYPE
 
-static void runImpl(Function &F) {
+static void tryEmitAutoInitRemark(ArrayRef<Instruction *> Instructions,
+                                  OptimizationRemarkEmitter &ORE,
+                                  const TargetLibraryInfo &TLI) {
+  // For every auto-init annotation generate a separate remark.
+  for (Instruction *I : Instructions) {
+    if (!AutoInitRemark::canHandle(I))
+      continue;
+
+    Function &F = *I->getParent()->getParent();
+    const DataLayout &DL = F.getParent()->getDataLayout();
+    AutoInitRemark Remark(ORE, REMARK_PASS, DL, TLI);
+    Remark.visit(I);
+  }
+}
+
+static void runImpl(Function &F, const TargetLibraryInfo &TLI) {
   if (!OptimizationRemarkEmitter::allowExtraAnalysis(F, REMARK_PASS))
     return;
 
+  // Track all annotated instructions aggregated based on their debug location.
+  DenseMap<MDNode *, SmallVector<Instruction *, 4>> DebugLoc2Annotated;
+
   OptimizationRemarkEmitter ORE(&F);
-  // For now, just generate a summary of the annotated instructions.
+  // First, generate a summary of the annotated instructions.
   MapVector<StringRef, unsigned> Mapping;
   for (Instruction &I : instructions(F)) {
     if (!I.hasMetadata(LLVMContext::MD_annotation))
       continue;
+    auto Iter = DebugLoc2Annotated.insert({I.getDebugLoc().getAsMDNode(), {}});
+    Iter.first->second.push_back(&I);
+
     for (const MDOperand &Op :
          I.getMetadata(LLVMContext::MD_annotation)->operands()) {
       auto Iter = Mapping.insert({cast<MDString>(Op.get())->getString(), 0});
@@ -44,11 +68,21 @@ static void runImpl(Function &F) {
     }
   }
 
-  Instruction *IP = &*F.begin()->begin();
   for (const auto &KV : Mapping)
-    ORE.emit(OptimizationRemarkAnalysis(REMARK_PASS, "AnnotationSummary", IP)
+    ORE.emit(OptimizationRemarkAnalysis(REMARK_PASS, "AnnotationSummary",
+                                        F.getSubprogram(), &F.front())
              << "Annotated " << NV("count", KV.second) << " instructions with "
              << NV("type", KV.first));
+
+  // For each debug location, look for all the instructions with annotations and
+  // generate more detailed remarks to be displayed at that location.
+  for (auto &KV : DebugLoc2Annotated) {
+    // Don't generate remarks with no debug location.
+    if (!KV.first)
+      continue;
+
+    tryEmitAutoInitRemark(KV.second, ORE, TLI);
+  }
 }
 
 namespace {
@@ -61,12 +95,15 @@ struct AnnotationRemarksLegacy : public FunctionPass {
   }
 
   bool runOnFunction(Function &F) override {
-    runImpl(F);
+    const TargetLibraryInfo &TLI =
+        getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+    runImpl(F, TLI);
     return false;
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesAll();
+    AU.addRequired<TargetLibraryInfoWrapperPass>();
   }
 };
 
@@ -76,6 +113,7 @@ char AnnotationRemarksLegacy::ID = 0;
 
 INITIALIZE_PASS_BEGIN(AnnotationRemarksLegacy, "annotation-remarks",
                       "Annotation Remarks", false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_END(AnnotationRemarksLegacy, "annotation-remarks",
                     "Annotation Remarks", false, false)
 
@@ -85,6 +123,7 @@ FunctionPass *llvm::createAnnotationRemarksLegacyPass() {
 
 PreservedAnalyses AnnotationRemarksPass::run(Function &F,
                                              FunctionAnalysisManager &AM) {
-  runImpl(F);
+  auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
+  runImpl(F, TLI);
   return PreservedAnalyses::all();
 }
diff --git a/llvm/lib/Transforms/Scalar/BDCE.cpp b/llvm/lib/Transforms/Scalar/BDCE.cpp
index 767c7656dcfa..c06125788f37 100644
--- a/llvm/lib/Transforms/Scalar/BDCE.cpp
+++ b/llvm/lib/Transforms/Scalar/BDCE.cpp
@@ -170,7 +170,6 @@ PreservedAnalyses BDCEPass::run(Function &F, FunctionAnalysisManager &AM) {
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
index fdab74fc94c5..535f50d4f904 100644
--- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -185,13 +185,20 @@ Instruction *ConstantHoistingPass::findMatInsertPt(Instruction *Inst,
   // We can't insert directly before a phi node or an eh pad. Insert before
   // the terminator of the incoming or dominating block.
   assert(Entry != Inst->getParent() && "PHI or landing pad in entry block!");
-  if (Idx != ~0U && isa<PHINode>(Inst))
-    return cast<PHINode>(Inst)->getIncomingBlock(Idx)->getTerminator();
+  BasicBlock *InsertionBlock = nullptr;
+  if (Idx != ~0U && isa<PHINode>(Inst)) {
+    InsertionBlock = cast<PHINode>(Inst)->getIncomingBlock(Idx);
+    if (!InsertionBlock->isEHPad()) {
+      return InsertionBlock->getTerminator();
+    }
+  } else {
+    InsertionBlock = Inst->getParent();
+  }
 
   // This must be an EH pad. Iterate over immediate dominators until we find a
   // non-EH pad. We need to skip over catchswitch blocks, which are both EH pads
   // and terminators.
-  auto IDom = DT->getNode(Inst->getParent())->getIDom();
+  auto *IDom = DT->getNode(InsertionBlock)->getIDom();
   while (IDom->getBlock()->isEHPad()) {
     assert(Entry != IDom->getBlock() && "eh pad in entry block");
     IDom = IDom->getIDom();
@@ -358,7 +365,7 @@ SetVector<Instruction *> ConstantHoistingPass::findConstantInsertionPoint(
 void ConstantHoistingPass::collectConstantCandidates(
     ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx,
     ConstantInt *ConstInt) {
-  unsigned Cost;
+  InstructionCost Cost;
   // Ask the target about the cost of materializing the constant for the given
   // instruction and operand index.
   if (auto IntrInst = dyn_cast<IntrinsicInst>(Inst))
@@ -380,7 +387,7 @@ void ConstantHoistingPass::collectConstantCandidates(
       ConstIntCandVec.push_back(ConstantCandidate(ConstInt));
       Itr->second = ConstIntCandVec.size() - 1;
     }
-    ConstIntCandVec[Itr->second].addUser(Inst, Idx, Cost);
+    ConstIntCandVec[Itr->second].addUser(Inst, Idx, *Cost.getValue());
     LLVM_DEBUG(if (isa<ConstantInt>(Inst->getOperand(Idx))) dbgs()
                    << "Collect constant " << *ConstInt << " from " << *Inst
                    << " with cost " << Cost << '\n';
@@ -418,7 +425,7 @@ void ConstantHoistingPass::collectConstantCandidates(
   // usually lowered to a load from constant pool. Such operation is unlikely
   // to be cheaper than compute it by <Base + Offset>, which can be lowered to
   // an ADD instruction or folded into Load/Store instruction.
-  int Cost =
+  InstructionCost Cost =
       TTI->getIntImmCostInst(Instruction::Add, 1, Offset, PtrIntTy,
                              TargetTransformInfo::TCK_SizeAndLatency, Inst);
   ConstCandVecType &ExprCandVec = ConstGEPCandMap[BaseGV];
@@ -432,7 +439,7 @@ void ConstantHoistingPass::collectConstantCandidates(
         ConstExpr));
     Itr->second = ExprCandVec.size() - 1;
   }
-  ExprCandVec[Itr->second].addUser(Inst, Idx, Cost);
+  ExprCandVec[Itr->second].addUser(Inst, Idx, *Cost.getValue());
 }
 
 /// Check the operand for instruction Inst at index Idx.
@@ -574,11 +581,11 @@ ConstantHoistingPass::maximizeConstantsInRange(ConstCandVecType::iterator S,
   }
 
   LLVM_DEBUG(dbgs() << "== Maximize constants in range ==\n");
-  int MaxCost = -1;
+  InstructionCost MaxCost = -1;
   for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
     auto Value = ConstCand->ConstInt->getValue();
     Type *Ty = ConstCand->ConstInt->getType();
-    int Cost = 0;
+    InstructionCost Cost = 0;
     NumUses += ConstCand->Uses.size();
     LLVM_DEBUG(dbgs() << "= Constant: " << ConstCand->ConstInt->getValue()
                       << "\n");
@@ -595,8 +602,8 @@ ConstantHoistingPass::maximizeConstantsInRange(ConstCandVecType::iterator S,
                                    C2->ConstInt->getValue(),
                                    ConstCand->ConstInt->getValue());
         if (Diff) {
-          const int ImmCosts =
-            TTI->getIntImmCodeSizeCost(Opcode, OpndIdx, Diff.getValue(), Ty);
+          const InstructionCost ImmCosts =
+              TTI->getIntImmCodeSizeCost(Opcode, OpndIdx, Diff.getValue(), Ty);
           Cost -= ImmCosts;
           LLVM_DEBUG(dbgs() << "Offset " << Diff.getValue() << " "
                             << "has penalty: " << ImmCosts << "\n"
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index 3b8af6f21ce5..efd1c025d0cd 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/Transforms/Scalar/ConstraintElimination.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/ScopeExit.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/ConstraintSystem.h"
@@ -28,6 +29,8 @@
 #include "llvm/Support/DebugCounter.h"
 #include "llvm/Transforms/Scalar.h"
 
+#include <string>
+
 using namespace llvm;
 using namespace PatternMatch;
 
@@ -50,32 +53,50 @@ static SmallVector<std::pair<int64_t, Value *>, 4> decompose(Value *V) {
     return {{CI->getSExtValue(), nullptr}};
   }
   auto *GEP = dyn_cast<GetElementPtrInst>(V);
-  if (GEP && GEP->getNumOperands() == 2) {
-    if (isa<ConstantInt>(GEP->getOperand(GEP->getNumOperands() - 1))) {
-      return {{cast<ConstantInt>(GEP->getOperand(GEP->getNumOperands() - 1))
-                   ->getSExtValue(),
-               nullptr},
-              {1, GEP->getPointerOperand()}};
-    }
-    Value *Op0;
+  if (GEP && GEP->getNumOperands() == 2 && GEP->isInBounds()) {
+    Value *Op0, *Op1;
     ConstantInt *CI;
+
+    // If the index is zero-extended, it is guaranteed to be positive.
     if (match(GEP->getOperand(GEP->getNumOperands() - 1),
-              m_NUWShl(m_Value(Op0), m_ConstantInt(CI))))
-      return {{0, nullptr},
-              {1, GEP->getPointerOperand()},
-              {std::pow(int64_t(2), CI->getSExtValue()), Op0}};
-    if (match(GEP->getOperand(GEP->getNumOperands() - 1),
-              m_ZExt(m_NUWShl(m_Value(Op0), m_ConstantInt(CI)))))
-      return {{0, nullptr},
-              {1, GEP->getPointerOperand()},
-              {std::pow(int64_t(2), CI->getSExtValue()), Op0}};
+              m_ZExt(m_Value(Op0)))) {
+      if (match(Op0, m_NUWShl(m_Value(Op1), m_ConstantInt(CI))))
+        return {{0, nullptr},
+                {1, GEP->getPointerOperand()},
+                {std::pow(int64_t(2), CI->getSExtValue()), Op1}};
+      if (match(Op0, m_NSWAdd(m_Value(Op1), m_ConstantInt(CI))))
+        return {{CI->getSExtValue(), nullptr},
+                {1, GEP->getPointerOperand()},
+                {1, Op1}};
+      return {{0, nullptr}, {1, GEP->getPointerOperand()}, {1, Op0}};
+    }
+
+    if (match(GEP->getOperand(GEP->getNumOperands() - 1), m_ConstantInt(CI)) &&
+        !CI->isNegative())
+      return {{CI->getSExtValue(), nullptr}, {1, GEP->getPointerOperand()}};
 
-    return {{0, nullptr},
-            {1, GEP->getPointerOperand()},
-            {1, GEP->getOperand(GEP->getNumOperands() - 1)}};
+    SmallVector<std::pair<int64_t, Value *>, 4> Result;
+    if (match(GEP->getOperand(GEP->getNumOperands() - 1),
+              m_NUWShl(m_Value(Op0), m_ConstantInt(CI))))
+      Result = {{0, nullptr},
+                {1, GEP->getPointerOperand()},
+                {std::pow(int64_t(2), CI->getSExtValue()), Op0}};
+    else if (match(GEP->getOperand(GEP->getNumOperands() - 1),
+                   m_NSWAdd(m_Value(Op0), m_ConstantInt(CI))))
+      Result = {{CI->getSExtValue(), nullptr},
+                {1, GEP->getPointerOperand()},
+                {1, Op0}};
+    else {
+      Op0 = GEP->getOperand(GEP->getNumOperands() - 1);
+      Result = {{0, nullptr}, {1, GEP->getPointerOperand()}, {1, Op0}};
+    }
+    return Result;
   }
 
   Value *Op0;
+  if (match(V, m_ZExt(m_Value(Op0))))
+    V = Op0;
+
   Value *Op1;
   ConstantInt *CI;
   if (match(V, m_NUWAdd(m_Value(Op0), m_ConstantInt(CI))))
@@ -91,37 +112,62 @@ static SmallVector<std::pair<int64_t, Value *>, 4> decompose(Value *V) {
   return {{0, nullptr}, {1, V}};
 }
 
-/// Turn a condition \p CmpI into a constraint vector, using indices from \p
-/// Value2Index. If \p ShouldAdd is true, new indices are added for values not
-/// yet in \p Value2Index.
-static SmallVector<int64_t, 8>
+struct ConstraintTy {
+  SmallVector<int64_t, 8> Coefficients;
+
+  ConstraintTy(SmallVector<int64_t, 8> Coefficients)
+      : Coefficients(Coefficients) {}
+
+  unsigned size() const { return Coefficients.size(); }
+};
+
+/// Turn a condition \p CmpI into a vector of constraints, using indices from \p
+/// Value2Index. Additional indices for newly discovered values are added to \p
+/// NewIndices.
+static SmallVector<ConstraintTy, 4>
 getConstraint(CmpInst::Predicate Pred, Value *Op0, Value *Op1,
-              DenseMap<Value *, unsigned> &Value2Index, bool ShouldAdd) {
+              const DenseMap<Value *, unsigned> &Value2Index,
+              DenseMap<Value *, unsigned> &NewIndices) {
   int64_t Offset1 = 0;
   int64_t Offset2 = 0;
 
-  auto TryToGetIndex = [ShouldAdd,
-                        &Value2Index](Value *V) -> Optional<unsigned> {
-    if (ShouldAdd) {
-      Value2Index.insert({V, Value2Index.size() + 1});
-      return Value2Index[V];
-    }
-    auto I = Value2Index.find(V);
-    if (I == Value2Index.end())
-      return None;
-    return I->second;
+  // First try to look up \p V in Value2Index and NewIndices. Otherwise add a
+  // new entry to NewIndices.
+  auto GetOrAddIndex = [&Value2Index, &NewIndices](Value *V) -> unsigned {
+    auto V2I = Value2Index.find(V);
+    if (V2I != Value2Index.end())
+      return V2I->second;
+    auto NewI = NewIndices.find(V);
+    if (NewI != NewIndices.end())
+      return NewI->second;
+    auto Insert =
+        NewIndices.insert({V, Value2Index.size() + NewIndices.size() + 1});
+    return Insert.first->second;
   };
 
   if (Pred == CmpInst::ICMP_UGT || Pred == CmpInst::ICMP_UGE)
     return getConstraint(CmpInst::getSwappedPredicate(Pred), Op1, Op0,
-                         Value2Index, ShouldAdd);
+                         Value2Index, NewIndices);
+
+  if (Pred == CmpInst::ICMP_EQ) {
+    auto A =
+        getConstraint(CmpInst::ICMP_UGE, Op0, Op1, Value2Index, NewIndices);
+    auto B =
+        getConstraint(CmpInst::ICMP_ULE, Op0, Op1, Value2Index, NewIndices);
+    append_range(A, B);
+    return A;
+  }
+
+  if (Pred == CmpInst::ICMP_NE && match(Op1, m_Zero())) {
+    return getConstraint(CmpInst::ICMP_UGT, Op0, Op1, Value2Index, NewIndices);
+  }
 
   // Only ULE and ULT predicates are supported at the moment.
   if (Pred != CmpInst::ICMP_ULE && Pred != CmpInst::ICMP_ULT)
     return {};
 
-  auto ADec = decompose(Op0);
-  auto BDec = decompose(Op1);
+  auto ADec = decompose(Op0->stripPointerCastsSameRepresentation());
+  auto BDec = decompose(Op1->stripPointerCastsSameRepresentation());
   // Skip if decomposing either of the values failed.
   if (ADec.empty() || BDec.empty())
     return {};
@@ -135,34 +181,32 @@ getConstraint(CmpInst::Predicate Pred, Value *Op0, Value *Op1,
   Offset1 *= -1;
 
   // Create iterator ranges that skip the constant-factor.
-  auto VariablesA = make_range(std::next(ADec.begin()), ADec.end());
-  auto VariablesB = make_range(std::next(BDec.begin()), BDec.end());
+  auto VariablesA = llvm::drop_begin(ADec);
+  auto VariablesB = llvm::drop_begin(BDec);
 
-  // Check if each referenced value in the constraint is already in the system
-  // or can be added (if ShouldAdd is true).
+  // Make sure all variables have entries in Value2Index or NewIndices.
   for (const auto &KV :
        concat<std::pair<int64_t, Value *>>(VariablesA, VariablesB))
-    if (!TryToGetIndex(KV.second))
-      return {};
+    GetOrAddIndex(KV.second);
 
   // Build result constraint, by first adding all coefficients from A and then
   // subtracting all coefficients from B.
-  SmallVector<int64_t, 8> R(Value2Index.size() + 1, 0);
+  SmallVector<int64_t, 8> R(Value2Index.size() + NewIndices.size() + 1, 0);
   for (const auto &KV : VariablesA)
-    R[Value2Index[KV.second]] += KV.first;
+    R[GetOrAddIndex(KV.second)] += KV.first;
 
   for (const auto &KV : VariablesB)
-    R[Value2Index[KV.second]] -= KV.first;
+    R[GetOrAddIndex(KV.second)] -= KV.first;
 
   R[0] = Offset1 + Offset2 + (Pred == CmpInst::ICMP_ULT ? -1 : 0);
-  return R;
+  return {R};
 }
 
-static SmallVector<int64_t, 8>
-getConstraint(CmpInst *Cmp, DenseMap<Value *, unsigned> &Value2Index,
-              bool ShouldAdd) {
+static SmallVector<ConstraintTy, 4>
+getConstraint(CmpInst *Cmp, const DenseMap<Value *, unsigned> &Value2Index,
+              DenseMap<Value *, unsigned> &NewIndices) {
   return getConstraint(Cmp->getPredicate(), Cmp->getOperand(0),
-                       Cmp->getOperand(1), Value2Index, ShouldAdd);
+                       Cmp->getOperand(1), Value2Index, NewIndices);
 }
 
 namespace {
@@ -197,6 +241,19 @@ struct StackEntry {
 };
 } // namespace
 
+#ifndef NDEBUG
+static void dumpWithNames(ConstraintTy &C,
+                          DenseMap<Value *, unsigned> &Value2Index) {
+  SmallVector<std::string> Names(Value2Index.size(), "");
+  for (auto &KV : Value2Index) {
+    Names[KV.second - 1] = std::string("%") + KV.first->getName().str();
+  }
+  ConstraintSystem CS;
+  CS.addVariableRowFill(C.Coefficients);
+  CS.dump(Names);
+}
+#endif
+
 static bool eliminateConstraints(Function &F, DominatorTree &DT) {
   bool Changed = false;
   DT.updateDFSNumbers();
@@ -215,6 +272,15 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT) {
     if (!Br || !Br->isConditional())
       continue;
 
+    // Returns true if we can add a known condition from BB to its successor
+    // block Succ. Each predecessor of Succ can either be BB or be dominated by
+    // Succ (e.g. the case when adding a condition from a pre-header to a loop
+    // header).
+    auto CanAdd = [&BB, &DT](BasicBlock *Succ) {
+      return all_of(predecessors(Succ), [&BB, &DT, Succ](BasicBlock *Pred) {
+        return Pred == &BB || DT.dominates(Succ, Pred);
+      });
+    };
     // If the condition is an OR of 2 compares and the false successor only has
     // the current block as predecessor, queue both negated conditions for the
     // false successor.
@@ -222,7 +288,7 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT) {
     if (match(Br->getCondition(), m_LogicalOr(m_Value(Op0), m_Value(Op1))) &&
         match(Op0, m_Cmp()) && match(Op1, m_Cmp())) {
       BasicBlock *FalseSuccessor = Br->getSuccessor(1);
-      if (FalseSuccessor->getSinglePredecessor()) {
+      if (CanAdd(FalseSuccessor)) {
         WorkList.emplace_back(DT.getNode(FalseSuccessor), cast<CmpInst>(Op0),
                               true);
         WorkList.emplace_back(DT.getNode(FalseSuccessor), cast<CmpInst>(Op1),
@@ -237,7 +303,7 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT) {
     if (match(Br->getCondition(), m_LogicalAnd(m_Value(Op0), m_Value(Op1))) &&
         match(Op0, m_Cmp()) && match(Op1, m_Cmp())) {
       BasicBlock *TrueSuccessor = Br->getSuccessor(0);
-      if (TrueSuccessor->getSinglePredecessor()) {
+      if (CanAdd(TrueSuccessor)) {
         WorkList.emplace_back(DT.getNode(TrueSuccessor), cast<CmpInst>(Op0),
                               false);
         WorkList.emplace_back(DT.getNode(TrueSuccessor), cast<CmpInst>(Op1),
@@ -249,9 +315,9 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT) {
     auto *CmpI = dyn_cast<CmpInst>(Br->getCondition());
     if (!CmpI)
       continue;
-    if (Br->getSuccessor(0)->getSinglePredecessor())
+    if (CanAdd(Br->getSuccessor(0)))
       WorkList.emplace_back(DT.getNode(Br->getSuccessor(0)), CmpI, false);
-    if (Br->getSuccessor(1)->getSinglePredecessor())
+    if (CanAdd(Br->getSuccessor(1)))
       WorkList.emplace_back(DT.getNode(Br->getSuccessor(1)), CmpI, true);
   }
 
@@ -299,10 +365,27 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT) {
         auto *Cmp = dyn_cast<CmpInst>(&I);
         if (!Cmp)
           continue;
-        auto R = getConstraint(Cmp, Value2Index, false);
-        if (R.empty() || R.size() == 1)
+
+        DenseMap<Value *, unsigned> NewIndices;
+        auto R = getConstraint(Cmp, Value2Index, NewIndices);
+        if (R.size() != 1)
           continue;
-        if (CS.isConditionImplied(R)) {
+
+        // Check if all coefficients of new indices are 0 after building the
+        // constraint. Skip if any of the new indices has a non-null
+        // coefficient.
+        bool HasNewIndex = false;
+        for (unsigned I = 0; I < NewIndices.size(); ++I) {
+          int64_t Last = R[0].Coefficients.pop_back_val();
+          if (Last != 0) {
+            HasNewIndex = true;
+            break;
+          }
+        }
+        if (HasNewIndex || R[0].size() == 1)
+          continue;
+
+        if (CS.isConditionImplied(R[0].Coefficients)) {
           if (!DebugCounter::shouldExecute(EliminatedCounter))
             continue;
 
@@ -317,7 +400,8 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT) {
           NumCondsRemoved++;
           Changed = true;
         }
-        if (CS.isConditionImplied(ConstraintSystem::negate(R))) {
+        if (CS.isConditionImplied(
+                ConstraintSystem::negate(R[0].Coefficients))) {
           if (!DebugCounter::shouldExecute(EliminatedCounter))
             continue;
 
@@ -336,22 +420,50 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT) {
       continue;
     }
 
+    // Set up a function to restore the predicate at the end of the scope if it
+    // has been negated. Negate the predicate in-place, if required.
+    auto *CI = dyn_cast<CmpInst>(CB.Condition);
+    auto PredicateRestorer = make_scope_exit([CI, &CB]() {
+      if (CB.Not && CI)
+        CI->setPredicate(CI->getInversePredicate());
+    });
+    if (CB.Not) {
+      if (CI) {
+        CI->setPredicate(CI->getInversePredicate());
+      } else {
+        LLVM_DEBUG(dbgs() << "Can only negate compares so far.\n");
+        continue;
+      }
+    }
+
     // Otherwise, add the condition to the system and stack, if we can transform
     // it into a constraint.
-    auto R = getConstraint(CB.Condition, Value2Index, true);
+    DenseMap<Value *, unsigned> NewIndices;
+    auto R = getConstraint(CB.Condition, Value2Index, NewIndices);
     if (R.empty())
       continue;
 
-    LLVM_DEBUG(dbgs() << "Adding " << *CB.Condition << " " << CB.Not << "\n");
-    if (CB.Not)
-      R = ConstraintSystem::negate(R);
+    for (auto &KV : NewIndices)
+      Value2Index.insert(KV);
 
-    // If R has been added to the system, queue it for removal once it goes
-    // out-of-scope.
-    if (CS.addVariableRowFill(R))
-      DFSInStack.emplace_back(CB.NumIn, CB.NumOut, CB.Condition, CB.Not);
+    LLVM_DEBUG(dbgs() << "Adding " << *CB.Condition << " " << CB.Not << "\n");
+    bool Added = false;
+    for (auto &C : R) {
+      auto Coeffs = C.Coefficients;
+      LLVM_DEBUG({
+        dbgs() << "  constraint: ";
+        dumpWithNames(C, Value2Index);
+      });
+      Added |= CS.addVariableRowFill(Coeffs);
+      // If R has been added to the system, queue it for removal once it goes
+      // out-of-scope.
+      if (Added)
+        DFSInStack.emplace_back(CB.NumIn, CB.NumOut, CB.Condition, CB.Not);
+    }
   }
 
+  assert(CS.size() == DFSInStack.size() &&
+         "updates to CS and DFSInStack are out of sync");
   return Changed;
 }
 
@@ -363,7 +475,6 @@ PreservedAnalyses ConstraintEliminationPass::run(Function &F,
 
   PreservedAnalyses PA;
   PA.preserve<DominatorTreeAnalysis>();
-  PA.preserve<GlobalsAA>();
   PA.preserveSet<CFGAnalyses>();
   return PA;
 }
diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index b671d68031a8..36cbd42a5fdd 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LazyValueInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/Attributes.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/CFG.h"
@@ -82,11 +83,12 @@ STATISTIC(NumMulNUW,    "Number of no-unsigned-wrap deductions for mul");
 STATISTIC(NumShlNW,     "Number of no-wrap deductions for shl");
 STATISTIC(NumShlNSW,    "Number of no-signed-wrap deductions for shl");
 STATISTIC(NumShlNUW,    "Number of no-unsigned-wrap deductions for shl");
+STATISTIC(NumAbs,       "Number of llvm.abs intrinsics removed");
 STATISTIC(NumOverflows, "Number of overflow checks removed");
 STATISTIC(NumSaturating,
     "Number of saturating arithmetics converted to normal arithmetics");
-
-static cl::opt<bool> DontAddNoWrapFlags("cvp-dont-add-nowrap-flags", cl::init(false));
+STATISTIC(NumNonNull, "Number of function pointer arguments marked non-null");
+STATISTIC(NumMinMax, "Number of llvm.[us]{min,max} intrinsics removed");
 
 namespace {
 
@@ -192,15 +194,14 @@ static bool simplifyCommonValuePhi(PHINode *P, LazyValueInfo *LVI,
       return false;
   }
 
+  // LVI only guarantees that the value matches a certain constant if the value
+  // is not poison. Make sure we don't replace a well-defined value with poison.
+  // This is usually satisfied due to a prior branch on the value.
+  if (!isGuaranteedNotToBePoison(CommonValue, nullptr, P, DT))
+    return false;
+
   // All constant incoming values map to the same variable along the incoming
-  // edges of the phi. The phi is unnecessary. However, we must drop all
-  // poison-generating flags to ensure that no poison is propagated to the phi
-  // location by performing this substitution.
-  // Warning: If the underlying analysis changes, this may not be enough to
-  //          guarantee that poison is not propagated.
-  // TODO: We may be able to re-infer flags by re-analyzing the instruction.
-  if (auto *CommonInst = dyn_cast<Instruction>(CommonValue))
-    CommonInst->dropPoisonGeneratingFlags();
+  // edges of the phi. The phi is unnecessary.
   P->replaceAllUsesWith(CommonValue);
   P->eraseFromParent();
   ++NumPhiCommon;
@@ -444,8 +445,81 @@ static void setDeducedOverflowingFlags(Value *V, Instruction::BinaryOps Opcode,
 
 static bool processBinOp(BinaryOperator *BinOp, LazyValueInfo *LVI);
 
+// See if @llvm.abs argument is alays positive/negative, and simplify.
+// Notably, INT_MIN can belong to either range, regardless of the NSW,
+// because it is negation-invariant.
+static bool processAbsIntrinsic(IntrinsicInst *II, LazyValueInfo *LVI) {
+  Value *X = II->getArgOperand(0);
+  bool IsIntMinPoison = cast<ConstantInt>(II->getArgOperand(1))->isOne();
+
+  Type *Ty = X->getType();
+  Constant *IntMin =
+      ConstantInt::get(Ty, APInt::getSignedMinValue(Ty->getScalarSizeInBits()));
+  LazyValueInfo::Tristate Result;
+
+  // Is X in [0, IntMin]?  NOTE: INT_MIN is fine!
+  Result = LVI->getPredicateAt(CmpInst::Predicate::ICMP_ULE, X, IntMin, II,
+                               /*UseBlockValue=*/true);
+  if (Result == LazyValueInfo::True) {
+    ++NumAbs;
+    II->replaceAllUsesWith(X);
+    II->eraseFromParent();
+    return true;
+  }
+
+  // Is X in [IntMin, 0]?  NOTE: INT_MIN is fine!
+  Constant *Zero = ConstantInt::getNullValue(Ty);
+  Result = LVI->getPredicateAt(CmpInst::Predicate::ICMP_SLE, X, Zero, II,
+                               /*UseBlockValue=*/true);
+  assert(Result != LazyValueInfo::False && "Should have been handled already.");
+
+  if (Result == LazyValueInfo::Unknown) {
+    // Argument's range crosses zero.
+    bool Changed = false;
+    if (!IsIntMinPoison) {
+      // Can we at least tell that the argument is never INT_MIN?
+      Result = LVI->getPredicateAt(CmpInst::Predicate::ICMP_NE, X, IntMin, II,
+                                   /*UseBlockValue=*/true);
+      if (Result == LazyValueInfo::True) {
+        ++NumNSW;
+        ++NumSubNSW;
+        II->setArgOperand(1, ConstantInt::getTrue(II->getContext()));
+        Changed = true;
+      }
+    }
+    return Changed;
+  }
+
+  IRBuilder<> B(II);
+  Value *NegX = B.CreateNeg(X, II->getName(), /*HasNUW=*/false,
+                            /*HasNSW=*/IsIntMinPoison);
+  ++NumAbs;
+  II->replaceAllUsesWith(NegX);
+  II->eraseFromParent();
+
+  // See if we can infer some no-wrap flags.
+  if (auto *BO = dyn_cast<BinaryOperator>(NegX))
+    processBinOp(BO, LVI);
+
+  return true;
+}
+
+// See if this min/max intrinsic always picks it's one specific operand.
+static bool processMinMaxIntrinsic(MinMaxIntrinsic *MM, LazyValueInfo *LVI) {
+  CmpInst::Predicate Pred = CmpInst::getNonStrictPredicate(MM->getPredicate());
+  LazyValueInfo::Tristate Result = LVI->getPredicateAt(
+      Pred, MM->getLHS(), MM->getRHS(), MM, /*UseBlockValue=*/true);
+  if (Result == LazyValueInfo::Unknown)
+    return false;
+
+  ++NumMinMax;
+  MM->replaceAllUsesWith(MM->getOperand(!Result));
+  MM->eraseFromParent();
+  return true;
+}
+
 // Rewrite this with.overflow intrinsic as non-overflowing.
-static void processOverflowIntrinsic(WithOverflowInst *WO, LazyValueInfo *LVI) {
+static bool processOverflowIntrinsic(WithOverflowInst *WO, LazyValueInfo *LVI) {
   IRBuilder<> B(WO);
   Instruction::BinaryOps Opcode = WO->getBinaryOp();
   bool NSW = WO->isSigned();
@@ -467,9 +541,11 @@ static void processOverflowIntrinsic(WithOverflowInst *WO, LazyValueInfo *LVI) {
   // See if we can infer the other no-wrap too.
   if (auto *BO = dyn_cast<BinaryOperator>(NewOp))
     processBinOp(BO, LVI);
+
+  return true;
 }
 
-static void processSaturatingInst(SaturatingInst *SI, LazyValueInfo *LVI) {
+static bool processSaturatingInst(SaturatingInst *SI, LazyValueInfo *LVI) {
   Instruction::BinaryOps Opcode = SI->getBinaryOp();
   bool NSW = SI->isSigned();
   bool NUW = !SI->isSigned();
@@ -485,22 +561,30 @@ static void processSaturatingInst(SaturatingInst *SI, LazyValueInfo *LVI) {
   // See if we can infer the other no-wrap too.
   if (auto *BO = dyn_cast<BinaryOperator>(BinOp))
     processBinOp(BO, LVI);
+
+  return true;
 }
 
 /// Infer nonnull attributes for the arguments at the specified callsite.
 static bool processCallSite(CallBase &CB, LazyValueInfo *LVI) {
 
+  if (CB.getIntrinsicID() == Intrinsic::abs) {
+    return processAbsIntrinsic(&cast<IntrinsicInst>(CB), LVI);
+  }
+
+  if (auto *MM = dyn_cast<MinMaxIntrinsic>(&CB)) {
+    return processMinMaxIntrinsic(MM, LVI);
+  }
+
   if (auto *WO = dyn_cast<WithOverflowInst>(&CB)) {
     if (WO->getLHS()->getType()->isIntegerTy() && willNotOverflow(WO, LVI)) {
-      processOverflowIntrinsic(WO, LVI);
-      return true;
+      return processOverflowIntrinsic(WO, LVI);
     }
   }
 
   if (auto *SI = dyn_cast<SaturatingInst>(&CB)) {
     if (SI->getType()->isIntegerTy() && willNotOverflow(SI, LVI)) {
-      processSaturatingInst(SI, LVI);
-      return true;
+      return processSaturatingInst(SI, LVI);
     }
   }
 
@@ -538,8 +622,8 @@ static bool processCallSite(CallBase &CB, LazyValueInfo *LVI) {
     if (Type && !CB.paramHasAttr(ArgNo, Attribute::NonNull) &&
         !isa<Constant>(V) &&
         LVI->getPredicateAt(ICmpInst::ICMP_EQ, V,
-                            ConstantPointerNull::get(Type),
-                            &CB) == LazyValueInfo::False)
+                            ConstantPointerNull::get(Type), &CB,
+                            /*UseBlockValue=*/false) == LazyValueInfo::False)
       ArgNos.push_back(ArgNo);
     ArgNo++;
   }
@@ -549,6 +633,7 @@ static bool processCallSite(CallBase &CB, LazyValueInfo *LVI) {
   if (ArgNos.empty())
     return Changed;
 
+  NumNonNull += ArgNos.size();
   AttributeList AS = CB.getAttributes();
   LLVMContext &Ctx = CB.getContext();
   AS = AS.addParamAttribute(Ctx, ArgNos,
@@ -560,13 +645,15 @@ static bool processCallSite(CallBase &CB, LazyValueInfo *LVI) {
 
 static bool isNonNegative(Value *V, LazyValueInfo *LVI, Instruction *CxtI) {
   Constant *Zero = ConstantInt::get(V->getType(), 0);
-  auto Result = LVI->getPredicateAt(ICmpInst::ICMP_SGE, V, Zero, CxtI);
+  auto Result = LVI->getPredicateAt(ICmpInst::ICMP_SGE, V, Zero, CxtI,
+                                    /*UseBlockValue=*/true);
   return Result == LazyValueInfo::True;
 }
 
 static bool isNonPositive(Value *V, LazyValueInfo *LVI, Instruction *CxtI) {
   Constant *Zero = ConstantInt::get(V->getType(), 0);
-  auto Result = LVI->getPredicateAt(ICmpInst::ICMP_SLE, V, Zero, CxtI);
+  auto Result = LVI->getPredicateAt(ICmpInst::ICMP_SLE, V, Zero, CxtI,
+                                    /*UseBlockValue=*/true);
   return Result == LazyValueInfo::True;
 }
 
@@ -843,9 +930,6 @@ static bool processSExt(SExtInst *SDI, LazyValueInfo *LVI) {
 static bool processBinOp(BinaryOperator *BinOp, LazyValueInfo *LVI) {
   using OBO = OverflowingBinaryOperator;
 
-  if (DontAddNoWrapFlags)
-    return false;
-
   if (BinOp->getType()->isVectorTy())
     return false;
 
@@ -919,8 +1003,8 @@ static Constant *getConstantAt(Value *V, Instruction *At, LazyValueInfo *LVI) {
   Constant *Op1 = dyn_cast<Constant>(C->getOperand(1));
   if (!Op1) return nullptr;
 
-  LazyValueInfo::Tristate Result =
-    LVI->getPredicateAt(C->getPredicate(), Op0, Op1, At);
+  LazyValueInfo::Tristate Result = LVI->getPredicateAt(
+      C->getPredicate(), Op0, Op1, At, /*UseBlockValue=*/false);
   if (Result == LazyValueInfo::Unknown)
     return nullptr;
 
@@ -1034,7 +1118,6 @@ CorrelatedValuePropagationPass::run(Function &F, FunctionAnalysisManager &AM) {
   if (!Changed) {
     PA = PreservedAnalyses::all();
   } else {
-    PA.preserve<GlobalsAA>();
     PA.preserve<DominatorTreeAnalysis>();
     PA.preserve<LazyValueAnalysis>();
   }
diff --git a/llvm/lib/Transforms/Scalar/DCE.cpp b/llvm/lib/Transforms/Scalar/DCE.cpp
index d55adf7c2d12..d309799d95f0 100644
--- a/llvm/lib/Transforms/Scalar/DCE.cpp
+++ b/llvm/lib/Transforms/Scalar/DCE.cpp
@@ -125,14 +125,11 @@ static bool eliminateDeadCode(Function &F, TargetLibraryInfo *TLI) {
   // Iterate over the original function, only adding insts to the worklist
   // if they actually need to be revisited. This avoids having to pre-init
   // the worklist with the entire function's worth of instructions.
-  for (inst_iterator FI = inst_begin(F), FE = inst_end(F); FI != FE;) {
-    Instruction *I = &*FI;
-    ++FI;
-
+  for (Instruction &I : llvm::make_early_inc_range(instructions(F))) {
     // We're visiting this instruction now, so make sure it's not in the
     // worklist from an earlier visit.
-    if (!WorkList.count(I))
-      MadeChange |= DCEInstruction(I, WorkList, TLI);
+    if (!WorkList.count(&I))
+      MadeChange |= DCEInstruction(&I, WorkList, TLI);
   }
 
   while (!WorkList.empty()) {
diff --git a/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
new file mode 100644
index 000000000000..90679bcac4b7
--- /dev/null
+++ b/llvm/lib/Transforms/Scalar/DFAJumpThreading.cpp
@@ -0,0 +1,1286 @@
+//===- DFAJumpThreading.cpp - Threads a switch statement inside a loop ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Transform each threading path to effectively jump thread the DFA. For
+// example, the CFG below could be transformed as follows, where the cloned
+// blocks unconditionally branch to the next correct case based on what is
+// identified in the analysis.
+//
+//          sw.bb                        sw.bb
+//        /   |   \                    /   |   \
+//   case1  case2  case3          case1  case2  case3
+//        \   |   /                 |      |      |
+//       determinator            det.2   det.3  det.1
+//        br sw.bb                /        |        \
+//                          sw.bb.2     sw.bb.3     sw.bb.1
+//                           br case2    br case3    br case1§
+//
+// Definitions and Terminology:
+//
+// * Threading path:
+//   a list of basic blocks, the exit state, and the block that determines
+//   the next state, for which the following notation will be used:
+//   < path of BBs that form a cycle > [ state, determinator ]
+//
+// * Predictable switch:
+//   The switch variable is always a known constant so that all conditional
+//   jumps based on switch variable can be converted to unconditional jump.
+//
+// * Determinator:
+//   The basic block that determines the next state of the DFA.
+//
+// Representing the optimization in C-like pseudocode: the code pattern on the
+// left could functionally be transformed to the right pattern if the switch
+// condition is predictable.
+//
+//  X = A                       goto A
+//  for (...)                   A:
+//    switch (X)                  ...
+//      case A                    goto B
+//        X = B                 B:
+//      case B                    ...
+//        X = C                   goto C
+//
+// The pass first checks that switch variable X is decided by the control flow
+// path taken in the loop; for example, in case B, the next value of X is
+// decided to be C. It then enumerates through all paths in the loop and labels
+// the basic blocks where the next state is decided.
+//
+// Using this information it creates new paths that unconditionally branch to
+// the next case. This involves cloning code, so it only gets triggered if the
+// amount of code duplicated is below a threshold.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Scalar/DFAJumpThreading.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/CodeMetrics.h"
+#include "llvm/Analysis/LoopIterator.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Verifier.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/SSAUpdaterBulk.h"
+#include "llvm/Transforms/Utils/ValueMapper.h"
+#include <algorithm>
+#include <deque>
+#include <unordered_map>
+#include <unordered_set>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "dfa-jump-threading"
+
+STATISTIC(NumTransforms, "Number of transformations done");
+STATISTIC(NumCloned, "Number of blocks cloned");
+STATISTIC(NumPaths, "Number of individual paths threaded");
+
+static cl::opt<bool>
+    ClViewCfgBefore("dfa-jump-view-cfg-before",
+                    cl::desc("View the CFG before DFA Jump Threading"),
+                    cl::Hidden, cl::init(false));
+
+static cl::opt<unsigned> MaxPathLength(
+    "dfa-max-path-length",
+    cl::desc("Max number of blocks searched to find a threading path"),
+    cl::Hidden, cl::init(20));
+
+static cl::opt<unsigned>
+    CostThreshold("dfa-cost-threshold",
+                  cl::desc("Maximum cost accepted for the transformation"),
+                  cl::Hidden, cl::init(50));
+
+namespace {
+
+class SelectInstToUnfold {
+  SelectInst *SI;
+  PHINode *SIUse;
+
+public:
+  SelectInstToUnfold(SelectInst *SI, PHINode *SIUse) : SI(SI), SIUse(SIUse) {}
+
+  SelectInst *getInst() { return SI; }
+  PHINode *getUse() { return SIUse; }
+
+  explicit operator bool() const { return SI && SIUse; }
+};
+
+void unfold(DomTreeUpdater *DTU, SelectInstToUnfold SIToUnfold,
+            std::vector<SelectInstToUnfold> *NewSIsToUnfold,
+            std::vector<BasicBlock *> *NewBBs);
+
+class DFAJumpThreading {
+public:
+  DFAJumpThreading(AssumptionCache *AC, DominatorTree *DT,
+                   TargetTransformInfo *TTI, OptimizationRemarkEmitter *ORE)
+      : AC(AC), DT(DT), TTI(TTI), ORE(ORE) {}
+
+  bool run(Function &F);
+
+private:
+  void
+  unfoldSelectInstrs(DominatorTree *DT,
+                     const SmallVector<SelectInstToUnfold, 4> &SelectInsts) {
+    DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
+    SmallVector<SelectInstToUnfold, 4> Stack;
+    for (SelectInstToUnfold SIToUnfold : SelectInsts)
+      Stack.push_back(SIToUnfold);
+
+    while (!Stack.empty()) {
+      SelectInstToUnfold SIToUnfold = Stack.back();
+      Stack.pop_back();
+
+      std::vector<SelectInstToUnfold> NewSIsToUnfold;
+      std::vector<BasicBlock *> NewBBs;
+      unfold(&DTU, SIToUnfold, &NewSIsToUnfold, &NewBBs);
+
+      // Put newly discovered select instructions into the work list.
+      for (const SelectInstToUnfold &NewSIToUnfold : NewSIsToUnfold)
+        Stack.push_back(NewSIToUnfold);
+    }
+  }
+
+  AssumptionCache *AC;
+  DominatorTree *DT;
+  TargetTransformInfo *TTI;
+  OptimizationRemarkEmitter *ORE;
+};
+
+class DFAJumpThreadingLegacyPass : public FunctionPass {
+public:
+  static char ID; // Pass identification
+  DFAJumpThreadingLegacyPass() : FunctionPass(ID) {}
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addRequired<AssumptionCacheTracker>();
+    AU.addRequired<DominatorTreeWrapperPass>();
+    AU.addRequired<TargetTransformInfoWrapperPass>();
+    AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
+  }
+
+  bool runOnFunction(Function &F) override {
+    if (skipFunction(F))
+      return false;
+
+    AssumptionCache *AC =
+        &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+    DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+    TargetTransformInfo *TTI =
+        &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+    OptimizationRemarkEmitter *ORE =
+        &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
+
+    return DFAJumpThreading(AC, DT, TTI, ORE).run(F);
+  }
+};
+} // end anonymous namespace
+
+char DFAJumpThreadingLegacyPass::ID = 0;
+INITIALIZE_PASS_BEGIN(DFAJumpThreadingLegacyPass, "dfa-jump-threading",
+                      "DFA Jump Threading", false, false)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
+INITIALIZE_PASS_END(DFAJumpThreadingLegacyPass, "dfa-jump-threading",
+                    "DFA Jump Threading", false, false)
+
+// Public interface to the DFA Jump Threading pass
+FunctionPass *llvm::createDFAJumpThreadingPass() {
+  return new DFAJumpThreadingLegacyPass();
+}
+
+namespace {
+
+/// Create a new basic block and sink \p SIToSink into it.
+void createBasicBlockAndSinkSelectInst(
+    DomTreeUpdater *DTU, SelectInst *SI, PHINode *SIUse, SelectInst *SIToSink,
+    BasicBlock *EndBlock, StringRef NewBBName, BasicBlock **NewBlock,
+    BranchInst **NewBranch, std::vector<SelectInstToUnfold> *NewSIsToUnfold,
+    std::vector<BasicBlock *> *NewBBs) {
+  assert(SIToSink->hasOneUse());
+  assert(NewBlock);
+  assert(NewBranch);
+  *NewBlock = BasicBlock::Create(SI->getContext(), NewBBName,
+                                 EndBlock->getParent(), EndBlock);
+  NewBBs->push_back(*NewBlock);
+  *NewBranch = BranchInst::Create(EndBlock, *NewBlock);
+  SIToSink->moveBefore(*NewBranch);
+  NewSIsToUnfold->push_back(SelectInstToUnfold(SIToSink, SIUse));
+  DTU->applyUpdates({{DominatorTree::Insert, *NewBlock, EndBlock}});
+}
+
+/// Unfold the select instruction held in \p SIToUnfold by replacing it with
+/// control flow.
+///
+/// Put newly discovered select instructions into \p NewSIsToUnfold. Put newly
+/// created basic blocks into \p NewBBs.
+///
+/// TODO: merge it with CodeGenPrepare::optimizeSelectInst() if possible.
+void unfold(DomTreeUpdater *DTU, SelectInstToUnfold SIToUnfold,
+            std::vector<SelectInstToUnfold> *NewSIsToUnfold,
+            std::vector<BasicBlock *> *NewBBs) {
+  SelectInst *SI = SIToUnfold.getInst();
+  PHINode *SIUse = SIToUnfold.getUse();
+  BasicBlock *StartBlock = SI->getParent();
+  BasicBlock *EndBlock = SIUse->getParent();
+  BranchInst *StartBlockTerm =
+      dyn_cast<BranchInst>(StartBlock->getTerminator());
+
+  assert(StartBlockTerm && StartBlockTerm->isUnconditional());
+  assert(SI->hasOneUse());
+
+  // These are the new basic blocks for the conditional branch.
+  // At least one will become an actual new basic block.
+  BasicBlock *TrueBlock = nullptr;
+  BasicBlock *FalseBlock = nullptr;
+  BranchInst *TrueBranch = nullptr;
+  BranchInst *FalseBranch = nullptr;
+
+  // Sink select instructions to be able to unfold them later.
+  if (SelectInst *SIOp = dyn_cast<SelectInst>(SI->getTrueValue())) {
+    createBasicBlockAndSinkSelectInst(DTU, SI, SIUse, SIOp, EndBlock,
+                                      "si.unfold.true", &TrueBlock, &TrueBranch,
+                                      NewSIsToUnfold, NewBBs);
+  }
+  if (SelectInst *SIOp = dyn_cast<SelectInst>(SI->getFalseValue())) {
+    createBasicBlockAndSinkSelectInst(DTU, SI, SIUse, SIOp, EndBlock,
+                                      "si.unfold.false", &FalseBlock,
+                                      &FalseBranch, NewSIsToUnfold, NewBBs);
+  }
+
+  // If there was nothing to sink, then arbitrarily choose the 'false' side
+  // for a new input value to the PHI.
+  if (!TrueBlock && !FalseBlock) {
+    FalseBlock = BasicBlock::Create(SI->getContext(), "si.unfold.false",
+                                    EndBlock->getParent(), EndBlock);
+    NewBBs->push_back(FalseBlock);
+    BranchInst::Create(EndBlock, FalseBlock);
+    DTU->applyUpdates({{DominatorTree::Insert, FalseBlock, EndBlock}});
+  }
+
+  // Insert the real conditional branch based on the original condition.
+  // If we did not create a new block for one of the 'true' or 'false' paths
+  // of the condition, it means that side of the branch goes to the end block
+  // directly and the path originates from the start block from the point of
+  // view of the new PHI.
+  BasicBlock *TT = EndBlock;
+  BasicBlock *FT = EndBlock;
+  if (TrueBlock && FalseBlock) {
+    // A diamond.
+    TT = TrueBlock;
+    FT = FalseBlock;
+
+    // Update the phi node of SI.
+    SIUse->removeIncomingValue(StartBlock, /* DeletePHIIfEmpty = */ false);
+    SIUse->addIncoming(SI->getTrueValue(), TrueBlock);
+    SIUse->addIncoming(SI->getFalseValue(), FalseBlock);
+
+    // Update any other PHI nodes in EndBlock.
+    for (PHINode &Phi : EndBlock->phis()) {
+      if (&Phi != SIUse) {
+        Phi.addIncoming(Phi.getIncomingValueForBlock(StartBlock), TrueBlock);
+        Phi.addIncoming(Phi.getIncomingValueForBlock(StartBlock), FalseBlock);
+      }
+    }
+  } else {
+    BasicBlock *NewBlock = nullptr;
+    Value *SIOp1 = SI->getTrueValue();
+    Value *SIOp2 = SI->getFalseValue();
+
+    // A triangle pointing right.
+    if (!TrueBlock) {
+      NewBlock = FalseBlock;
+      FT = FalseBlock;
+    }
+    // A triangle pointing left.
+    else {
+      NewBlock = TrueBlock;
+      TT = TrueBlock;
+      std::swap(SIOp1, SIOp2);
+    }
+
+    // Update the phi node of SI.
+    for (unsigned Idx = 0; Idx < SIUse->getNumIncomingValues(); ++Idx) {
+      if (SIUse->getIncomingBlock(Idx) == StartBlock)
+        SIUse->setIncomingValue(Idx, SIOp1);
+    }
+    SIUse->addIncoming(SIOp2, NewBlock);
+
+    // Update any other PHI nodes in EndBlock.
+    for (auto II = EndBlock->begin(); PHINode *Phi = dyn_cast<PHINode>(II);
+         ++II) {
+      if (Phi != SIUse)
+        Phi->addIncoming(Phi->getIncomingValueForBlock(StartBlock), NewBlock);
+    }
+  }
+  StartBlockTerm->eraseFromParent();
+  BranchInst::Create(TT, FT, SI->getCondition(), StartBlock);
+  DTU->applyUpdates({{DominatorTree::Insert, StartBlock, TT},
+                     {DominatorTree::Insert, StartBlock, FT}});
+
+  // The select is now dead.
+  SI->eraseFromParent();
+}
+
+struct ClonedBlock {
+  BasicBlock *BB;
+  uint64_t State; ///< \p State corresponds to the next value of a switch stmnt.
+};
+
+typedef std::deque<BasicBlock *> PathType;
+typedef std::vector<PathType> PathsType;
+typedef std::set<const BasicBlock *> VisitedBlocks;
+typedef std::vector<ClonedBlock> CloneList;
+
+// This data structure keeps track of all blocks that have been cloned.  If two
+// different ThreadingPaths clone the same block for a certain state it should
+// be reused, and it can be looked up in this map.
+typedef DenseMap<BasicBlock *, CloneList> DuplicateBlockMap;
+
+// This map keeps track of all the new definitions for an instruction. This
+// information is needed when restoring SSA form after cloning blocks.
+typedef DenseMap<Instruction *, std::vector<Instruction *>> DefMap;
+
+inline raw_ostream &operator<<(raw_ostream &OS, const PathType &Path) {
+  OS << "< ";
+  for (const BasicBlock *BB : Path) {
+    std::string BBName;
+    if (BB->hasName())
+      raw_string_ostream(BBName) << BB->getName();
+    else
+      raw_string_ostream(BBName) << BB;
+    OS << BBName << " ";
+  }
+  OS << ">";
+  return OS;
+}
+
+/// ThreadingPath is a path in the control flow of a loop that can be threaded
+/// by cloning necessary basic blocks and replacing conditional branches with
+/// unconditional ones. A threading path includes a list of basic blocks, the
+/// exit state, and the block that determines the next state.
+struct ThreadingPath {
+  /// Exit value is DFA's exit state for the given path.
+  uint64_t getExitValue() const { return ExitVal; }
+  void setExitValue(const ConstantInt *V) {
+    ExitVal = V->getZExtValue();
+    IsExitValSet = true;
+  }
+  bool isExitValueSet() const { return IsExitValSet; }
+
+  /// Determinator is the basic block that determines the next state of the DFA.
+  const BasicBlock *getDeterminatorBB() const { return DBB; }
+  void setDeterminator(const BasicBlock *BB) { DBB = BB; }
+
+  /// Path is a list of basic blocks.
+  const PathType &getPath() const { return Path; }
+  void setPath(const PathType &NewPath) { Path = NewPath; }
+
+  void print(raw_ostream &OS) const {
+    OS << Path << " [ " << ExitVal << ", " << DBB->getName() << " ]";
+  }
+
+private:
+  PathType Path;
+  uint64_t ExitVal;
+  const BasicBlock *DBB = nullptr;
+  bool IsExitValSet = false;
+};
+
+#ifndef NDEBUG
+inline raw_ostream &operator<<(raw_ostream &OS, const ThreadingPath &TPath) {
+  TPath.print(OS);
+  return OS;
+}
+#endif
+
+struct MainSwitch {
+  MainSwitch(SwitchInst *SI, OptimizationRemarkEmitter *ORE) {
+    if (isPredictable(SI)) {
+      Instr = SI;
+    } else {
+      ORE->emit([&]() {
+        return OptimizationRemarkMissed(DEBUG_TYPE, "SwitchNotPredictable", SI)
+               << "Switch instruction is not predictable.";
+      });
+    }
+  }
+
+  virtual ~MainSwitch() = default;
+
+  SwitchInst *getInstr() const { return Instr; }
+  const SmallVector<SelectInstToUnfold, 4> getSelectInsts() {
+    return SelectInsts;
+  }
+
+private:
+  /// Do a use-def chain traversal. Make sure the value of the switch variable
+  /// is always a known constant. This means that all conditional jumps based on
+  /// switch variable can be converted to unconditional jumps.
+  bool isPredictable(const SwitchInst *SI) {
+    std::deque<Instruction *> Q;
+    SmallSet<Value *, 16> SeenValues;
+    SelectInsts.clear();
+
+    Value *FirstDef = SI->getOperand(0);
+    auto *Inst = dyn_cast<Instruction>(FirstDef);
+
+    // If this is a function argument or another non-instruction, then give up.
+    // We are interested in loop local variables.
+    if (!Inst)
+      return false;
+
+    // Require the first definition to be a PHINode
+    if (!isa<PHINode>(Inst))
+      return false;
+
+    LLVM_DEBUG(dbgs() << "\tisPredictable() FirstDef: " << *Inst << "\n");
+
+    Q.push_back(Inst);
+    SeenValues.insert(FirstDef);
+
+    while (!Q.empty()) {
+      Instruction *Current = Q.front();
+      Q.pop_front();
+
+      if (auto *Phi = dyn_cast<PHINode>(Current)) {
+        for (Value *Incoming : Phi->incoming_values()) {
+          if (!isPredictableValue(Incoming, SeenValues))
+            return false;
+          addInstToQueue(Incoming, Q, SeenValues);
+        }
+        LLVM_DEBUG(dbgs() << "\tisPredictable() phi: " << *Phi << "\n");
+      } else if (SelectInst *SelI = dyn_cast<SelectInst>(Current)) {
+        if (!isValidSelectInst(SelI))
+          return false;
+        if (!isPredictableValue(SelI->getTrueValue(), SeenValues) ||
+            !isPredictableValue(SelI->getFalseValue(), SeenValues)) {
+          return false;
+        }
+        addInstToQueue(SelI->getTrueValue(), Q, SeenValues);
+        addInstToQueue(SelI->getFalseValue(), Q, SeenValues);
+        LLVM_DEBUG(dbgs() << "\tisPredictable() select: " << *SelI << "\n");
+        if (auto *SelIUse = dyn_cast<PHINode>(SelI->user_back()))
+          SelectInsts.push_back(SelectInstToUnfold(SelI, SelIUse));
+      } else {
+        // If it is neither a phi nor a select, then we give up.
+        return false;
+      }
+    }
+
+    return true;
+  }
+
+  bool isPredictableValue(Value *InpVal, SmallSet<Value *, 16> &SeenValues) {
+    if (SeenValues.find(InpVal) != SeenValues.end())
+      return true;
+
+    if (isa<ConstantInt>(InpVal))
+      return true;
+
+    // If this is a function argument or another non-instruction, then give up.
+    if (!isa<Instruction>(InpVal))
+      return false;
+
+    return true;
+  }
+
+  void addInstToQueue(Value *Val, std::deque<Instruction *> &Q,
+                      SmallSet<Value *, 16> &SeenValues) {
+    if (SeenValues.find(Val) != SeenValues.end())
+      return;
+    if (Instruction *I = dyn_cast<Instruction>(Val))
+      Q.push_back(I);
+    SeenValues.insert(Val);
+  }
+
+  bool isValidSelectInst(SelectInst *SI) {
+    if (!SI->hasOneUse())
+      return false;
+
+    Instruction *SIUse = dyn_cast<Instruction>(SI->user_back());
+    // The use of the select inst should be either a phi or another select.
+    if (!SIUse && !(isa<PHINode>(SIUse) || isa<SelectInst>(SIUse)))
+      return false;
+
+    BasicBlock *SIBB = SI->getParent();
+
+    // Currently, we can only expand select instructions in basic blocks with
+    // one successor.
+    BranchInst *SITerm = dyn_cast<BranchInst>(SIBB->getTerminator());
+    if (!SITerm || !SITerm->isUnconditional())
+      return false;
+
+    if (isa<PHINode>(SIUse) &&
+        SIBB->getSingleSuccessor() != dyn_cast<Instruction>(SIUse)->getParent())
+      return false;
+
+    // If select will not be sunk during unfolding, and it is in the same basic
+    // block as another state defining select, then cannot unfold both.
+    for (SelectInstToUnfold SIToUnfold : SelectInsts) {
+      SelectInst *PrevSI = SIToUnfold.getInst();
+      if (PrevSI->getTrueValue() != SI && PrevSI->getFalseValue() != SI &&
+          PrevSI->getParent() == SI->getParent())
+        return false;
+    }
+
+    return true;
+  }
+
+  SwitchInst *Instr = nullptr;
+  SmallVector<SelectInstToUnfold, 4> SelectInsts;
+};
+
+struct AllSwitchPaths {
+  AllSwitchPaths(const MainSwitch *MSwitch, OptimizationRemarkEmitter *ORE)
+      : Switch(MSwitch->getInstr()), SwitchBlock(Switch->getParent()),
+        ORE(ORE) {}
+
+  std::vector<ThreadingPath> &getThreadingPaths() { return TPaths; }
+  unsigned getNumThreadingPaths() { return TPaths.size(); }
+  SwitchInst *getSwitchInst() { return Switch; }
+  BasicBlock *getSwitchBlock() { return SwitchBlock; }
+
+  void run() {
+    VisitedBlocks Visited;
+    PathsType LoopPaths = paths(SwitchBlock, Visited, /* PathDepth = */ 1);
+    StateDefMap StateDef = getStateDefMap();
+
+    for (PathType Path : LoopPaths) {
+      ThreadingPath TPath;
+
+      const BasicBlock *PrevBB = Path.back();
+      for (const BasicBlock *BB : Path) {
+        if (StateDef.count(BB) != 0) {
+          const PHINode *Phi = dyn_cast<PHINode>(StateDef[BB]);
+          assert(Phi && "Expected a state-defining instr to be a phi node.");
+
+          const Value *V = Phi->getIncomingValueForBlock(PrevBB);
+          if (const ConstantInt *C = dyn_cast<const ConstantInt>(V)) {
+            TPath.setExitValue(C);
+            TPath.setDeterminator(BB);
+            TPath.setPath(Path);
+          }
+        }
+
+        // Switch block is the determinator, this is the final exit value.
+        if (TPath.isExitValueSet() && BB == Path.front())
+          break;
+
+        PrevBB = BB;
+      }
+
+      if (TPath.isExitValueSet())
+        TPaths.push_back(TPath);
+    }
+  }
+
+private:
+  // Value: an instruction that defines a switch state;
+  // Key: the parent basic block of that instruction.
+  typedef DenseMap<const BasicBlock *, const PHINode *> StateDefMap;
+
+  PathsType paths(BasicBlock *BB, VisitedBlocks &Visited,
+                  unsigned PathDepth) const {
+    PathsType Res;
+
+    // Stop exploring paths after visiting MaxPathLength blocks
+    if (PathDepth > MaxPathLength) {
+      ORE->emit([&]() {
+        return OptimizationRemarkAnalysis(DEBUG_TYPE, "MaxPathLengthReached",
+                                          Switch)
+               << "Exploration stopped after visiting MaxPathLength="
+               << ore::NV("MaxPathLength", MaxPathLength) << " blocks.";
+      });
+      return Res;
+    }
+
+    Visited.insert(BB);
+
+    // Some blocks have multiple edges to the same successor, and this set
+    // is used to prevent a duplicate path from being generated
+    SmallSet<BasicBlock *, 4> Successors;
+
+    for (succ_iterator SI = succ_begin(BB), E = succ_end(BB); SI != E; ++SI) {
+      BasicBlock *Succ = *SI;
+
+      if (Successors.find(Succ) != Successors.end())
+        continue;
+      Successors.insert(Succ);
+
+      // Found a cycle through the SwitchBlock
+      if (Succ == SwitchBlock) {
+        Res.push_back({BB});
+        continue;
+      }
+
+      // We have encountered a cycle, do not get caught in it
+      if (Visited.find(Succ) != Visited.end())
+        continue;
+
+      PathsType SuccPaths = paths(Succ, Visited, PathDepth + 1);
+      for (PathType Path : SuccPaths) {
+        PathType NewPath(Path);
+        NewPath.push_front(BB);
+        Res.push_back(NewPath);
+      }
+    }
+    // This block could now be visited again from a different predecessor. Note
+    // that this will result in exponential runtime. Subpaths could possibly be
+    // cached but it takes a lot of memory to store them.
+    Visited.erase(BB);
+    return Res;
+  }
+
+  /// Walk the use-def chain and collect all the state-defining instructions.
+  StateDefMap getStateDefMap() const {
+    StateDefMap Res;
+
+    Value *FirstDef = Switch->getOperand(0);
+
+    assert(isa<PHINode>(FirstDef) && "After select unfolding, all state "
+                                     "definitions are expected to be phi "
+                                     "nodes.");
+
+    SmallVector<PHINode *, 8> Stack;
+    Stack.push_back(dyn_cast<PHINode>(FirstDef));
+    SmallSet<Value *, 16> SeenValues;
+
+    while (!Stack.empty()) {
+      PHINode *CurPhi = Stack.back();
+      Stack.pop_back();
+
+      Res[CurPhi->getParent()] = CurPhi;
+      SeenValues.insert(CurPhi);
+
+      for (Value *Incoming : CurPhi->incoming_values()) {
+        if (Incoming == FirstDef || isa<ConstantInt>(Incoming) ||
+            SeenValues.find(Incoming) != SeenValues.end()) {
+          continue;
+        }
+
+        assert(isa<PHINode>(Incoming) && "After select unfolding, all state "
+                                         "definitions are expected to be phi "
+                                         "nodes.");
+
+        Stack.push_back(cast<PHINode>(Incoming));
+      }
+    }
+
+    return Res;
+  }
+
+  SwitchInst *Switch;
+  BasicBlock *SwitchBlock;
+  OptimizationRemarkEmitter *ORE;
+  std::vector<ThreadingPath> TPaths;
+};
+
+struct TransformDFA {
+  TransformDFA(AllSwitchPaths *SwitchPaths, DominatorTree *DT,
+               AssumptionCache *AC, TargetTransformInfo *TTI,
+               OptimizationRemarkEmitter *ORE,
+               SmallPtrSet<const Value *, 32> EphValues)
+      : SwitchPaths(SwitchPaths), DT(DT), AC(AC), TTI(TTI), ORE(ORE),
+        EphValues(EphValues) {}
+
+  void run() {
+    if (isLegalAndProfitableToTransform()) {
+      createAllExitPaths();
+      NumTransforms++;
+    }
+  }
+
+private:
+  /// This function performs both a legality check and profitability check at
+  /// the same time since it is convenient to do so. It iterates through all
+  /// blocks that will be cloned, and keeps track of the duplication cost. It
+  /// also returns false if it is illegal to clone some required block.
+  bool isLegalAndProfitableToTransform() {
+    CodeMetrics Metrics;
+    SwitchInst *Switch = SwitchPaths->getSwitchInst();
+
+    // Note that DuplicateBlockMap is not being used as intended here. It is
+    // just being used to ensure (BB, State) pairs are only counted once.
+    DuplicateBlockMap DuplicateMap;
+
+    for (ThreadingPath &TPath : SwitchPaths->getThreadingPaths()) {
+      PathType PathBBs = TPath.getPath();
+      uint64_t NextState = TPath.getExitValue();
+      const BasicBlock *Determinator = TPath.getDeterminatorBB();
+
+      // Update Metrics for the Switch block, this is always cloned
+      BasicBlock *BB = SwitchPaths->getSwitchBlock();
+      BasicBlock *VisitedBB = getClonedBB(BB, NextState, DuplicateMap);
+      if (!VisitedBB) {
+        Metrics.analyzeBasicBlock(BB, *TTI, EphValues);
+        DuplicateMap[BB].push_back({BB, NextState});
+      }
+
+      // If the Switch block is the Determinator, then we can continue since
+      // this is the only block that is cloned and we already counted for it.
+      if (PathBBs.front() == Determinator)
+        continue;
+
+      // Otherwise update Metrics for all blocks that will be cloned. If any
+      // block is already cloned and would be reused, don't double count it.
+      auto DetIt = std::find(PathBBs.begin(), PathBBs.end(), Determinator);
+      for (auto BBIt = DetIt; BBIt != PathBBs.end(); BBIt++) {
+        BB = *BBIt;
+        VisitedBB = getClonedBB(BB, NextState, DuplicateMap);
+        if (VisitedBB)
+          continue;
+        Metrics.analyzeBasicBlock(BB, *TTI, EphValues);
+        DuplicateMap[BB].push_back({BB, NextState});
+      }
+
+      if (Metrics.notDuplicatable) {
+        LLVM_DEBUG(dbgs() << "DFA Jump Threading: Not jump threading, contains "
+                          << "non-duplicatable instructions.\n");
+        ORE->emit([&]() {
+          return OptimizationRemarkMissed(DEBUG_TYPE, "NonDuplicatableInst",
+                                          Switch)
+                 << "Contains non-duplicatable instructions.";
+        });
+        return false;
+      }
+
+      if (Metrics.convergent) {
+        LLVM_DEBUG(dbgs() << "DFA Jump Threading: Not jump threading, contains "
+                          << "convergent instructions.\n");
+        ORE->emit([&]() {
+          return OptimizationRemarkMissed(DEBUG_TYPE, "ConvergentInst", Switch)
+                 << "Contains convergent instructions.";
+        });
+        return false;
+      }
+    }
+
+    unsigned DuplicationCost = 0;
+
+    unsigned JumpTableSize = 0;
+    TTI->getEstimatedNumberOfCaseClusters(*Switch, JumpTableSize, nullptr,
+                                          nullptr);
+    if (JumpTableSize == 0) {
+      // Factor in the number of conditional branches reduced from jump
+      // threading. Assume that lowering the switch block is implemented by
+      // using binary search, hence the LogBase2().
+      unsigned CondBranches =
+          APInt(32, Switch->getNumSuccessors()).ceilLogBase2();
+      DuplicationCost = Metrics.NumInsts / CondBranches;
+    } else {
+      // Compared with jump tables, the DFA optimizer removes an indirect branch
+      // on each loop iteration, thus making branch prediction more precise. The
+      // more branch targets there are, the more likely it is for the branch
+      // predictor to make a mistake, and the more benefit there is in the DFA
+      // optimizer. Thus, the more branch targets there are, the lower is the
+      // cost of the DFA opt.
+      DuplicationCost = Metrics.NumInsts / JumpTableSize;
+    }
+
+    LLVM_DEBUG(dbgs() << "\nDFA Jump Threading: Cost to jump thread block "
+                      << SwitchPaths->getSwitchBlock()->getName()
+                      << " is: " << DuplicationCost << "\n\n");
+
+    if (DuplicationCost > CostThreshold) {
+      LLVM_DEBUG(dbgs() << "Not jump threading, duplication cost exceeds the "
+                        << "cost threshold.\n");
+      ORE->emit([&]() {
+        return OptimizationRemarkMissed(DEBUG_TYPE, "NotProfitable", Switch)
+               << "Duplication cost exceeds the cost threshold (cost="
+               << ore::NV("Cost", DuplicationCost)
+               << ", threshold=" << ore::NV("Threshold", CostThreshold) << ").";
+      });
+      return false;
+    }
+
+    ORE->emit([&]() {
+      return OptimizationRemark(DEBUG_TYPE, "JumpThreaded", Switch)
+             << "Switch statement jump-threaded.";
+    });
+
+    return true;
+  }
+
+  /// Transform each threading path to effectively jump thread the DFA.
+  void createAllExitPaths() {
+    DomTreeUpdater DTU(*DT, DomTreeUpdater::UpdateStrategy::Eager);
+
+    // Move the switch block to the end of the path, since it will be duplicated
+    BasicBlock *SwitchBlock = SwitchPaths->getSwitchBlock();
+    for (ThreadingPath &TPath : SwitchPaths->getThreadingPaths()) {
+      LLVM_DEBUG(dbgs() << TPath << "\n");
+      PathType NewPath(TPath.getPath());
+      NewPath.push_back(SwitchBlock);
+      TPath.setPath(NewPath);
+    }
+
+    // Transform the ThreadingPaths and keep track of the cloned values
+    DuplicateBlockMap DuplicateMap;
+    DefMap NewDefs;
+
+    SmallSet<BasicBlock *, 16> BlocksToClean;
+    for (BasicBlock *BB : successors(SwitchBlock))
+      BlocksToClean.insert(BB);
+
+    for (ThreadingPath &TPath : SwitchPaths->getThreadingPaths()) {
+      createExitPath(NewDefs, TPath, DuplicateMap, BlocksToClean, &DTU);
+      NumPaths++;
+    }
+
+    // After all paths are cloned, now update the last successor of the cloned
+    // path so it skips over the switch statement
+    for (ThreadingPath &TPath : SwitchPaths->getThreadingPaths())
+      updateLastSuccessor(TPath, DuplicateMap, &DTU);
+
+    // For each instruction that was cloned and used outside, update its uses
+    updateSSA(NewDefs);
+
+    // Clean PHI Nodes for the newly created blocks
+    for (BasicBlock *BB : BlocksToClean)
+      cleanPhiNodes(BB);
+  }
+
+  /// For a specific ThreadingPath \p Path, create an exit path starting from
+  /// the determinator block.
+  ///
+  /// To remember the correct destination, we have to duplicate blocks
+  /// corresponding to each state. Also update the terminating instruction of
+  /// the predecessors, and phis in the successor blocks.
+  void createExitPath(DefMap &NewDefs, ThreadingPath &Path,
+                      DuplicateBlockMap &DuplicateMap,
+                      SmallSet<BasicBlock *, 16> &BlocksToClean,
+                      DomTreeUpdater *DTU) {
+    uint64_t NextState = Path.getExitValue();
+    const BasicBlock *Determinator = Path.getDeterminatorBB();
+    PathType PathBBs = Path.getPath();
+
+    // Don't select the placeholder block in front
+    if (PathBBs.front() == Determinator)
+      PathBBs.pop_front();
+
+    auto DetIt = std::find(PathBBs.begin(), PathBBs.end(), Determinator);
+    auto Prev = std::prev(DetIt);
+    BasicBlock *PrevBB = *Prev;
+    for (auto BBIt = DetIt; BBIt != PathBBs.end(); BBIt++) {
+      BasicBlock *BB = *BBIt;
+      BlocksToClean.insert(BB);
+
+      // We already cloned BB for this NextState, now just update the branch
+      // and continue.
+      BasicBlock *NextBB = getClonedBB(BB, NextState, DuplicateMap);
+      if (NextBB) {
+        updatePredecessor(PrevBB, BB, NextBB, DTU);
+        PrevBB = NextBB;
+        continue;
+      }
+
+      // Clone the BB and update the successor of Prev to jump to the new block
+      BasicBlock *NewBB = cloneBlockAndUpdatePredecessor(
+          BB, PrevBB, NextState, DuplicateMap, NewDefs, DTU);
+      DuplicateMap[BB].push_back({NewBB, NextState});
+      BlocksToClean.insert(NewBB);
+      PrevBB = NewBB;
+    }
+  }
+
+  /// Restore SSA form after cloning blocks.
+  ///
+  /// Each cloned block creates new defs for a variable, and the uses need to be
+  /// updated to reflect this. The uses may be replaced with a cloned value, or
+  /// some derived phi instruction. Note that all uses of a value defined in the
+  /// same block were already remapped when cloning the block.
+  void updateSSA(DefMap &NewDefs) {
+    SSAUpdaterBulk SSAUpdate;
+    SmallVector<Use *, 16> UsesToRename;
+
+    for (auto KV : NewDefs) {
+      Instruction *I = KV.first;
+      BasicBlock *BB = I->getParent();
+      std::vector<Instruction *> Cloned = KV.second;
+
+      // Scan all uses of this instruction to see if it is used outside of its
+      // block, and if so, record them in UsesToRename.
+      for (Use &U : I->uses()) {
+        Instruction *User = cast<Instruction>(U.getUser());
+        if (PHINode *UserPN = dyn_cast<PHINode>(User)) {
+          if (UserPN->getIncomingBlock(U) == BB)
+            continue;
+        } else if (User->getParent() == BB) {
+          continue;
+        }
+
+        UsesToRename.push_back(&U);
+      }
+
+      // If there are no uses outside the block, we're done with this
+      // instruction.
+      if (UsesToRename.empty())
+        continue;
+      LLVM_DEBUG(dbgs() << "DFA-JT: Renaming non-local uses of: " << *I
+                        << "\n");
+
+      // We found a use of I outside of BB.  Rename all uses of I that are
+      // outside its block to be uses of the appropriate PHI node etc.  See
+      // ValuesInBlocks with the values we know.
+      unsigned VarNum = SSAUpdate.AddVariable(I->getName(), I->getType());
+      SSAUpdate.AddAvailableValue(VarNum, BB, I);
+      for (Instruction *New : Cloned)
+        SSAUpdate.AddAvailableValue(VarNum, New->getParent(), New);
+
+      while (!UsesToRename.empty())
+        SSAUpdate.AddUse(VarNum, UsesToRename.pop_back_val());
+
+      LLVM_DEBUG(dbgs() << "\n");
+    }
+    // SSAUpdater handles phi placement and renaming uses with the appropriate
+    // value.
+    SSAUpdate.RewriteAllUses(DT);
+  }
+
+  /// Clones a basic block, and adds it to the CFG.
+  ///
+  /// This function also includes updating phi nodes in the successors of the
+  /// BB, and remapping uses that were defined locally in the cloned BB.
+  BasicBlock *cloneBlockAndUpdatePredecessor(BasicBlock *BB, BasicBlock *PrevBB,
+                                             uint64_t NextState,
+                                             DuplicateBlockMap &DuplicateMap,
+                                             DefMap &NewDefs,
+                                             DomTreeUpdater *DTU) {
+    ValueToValueMapTy VMap;
+    BasicBlock *NewBB = CloneBasicBlock(
+        BB, VMap, ".jt" + std::to_string(NextState), BB->getParent());
+    NewBB->moveAfter(BB);
+    NumCloned++;
+
+    for (Instruction &I : *NewBB) {
+      // Do not remap operands of PHINode in case a definition in BB is an
+      // incoming value to a phi in the same block. This incoming value will
+      // be renamed later while restoring SSA.
+      if (isa<PHINode>(&I))
+        continue;
+      RemapInstruction(&I, VMap,
+                       RF_IgnoreMissingLocals | RF_NoModuleLevelChanges);
+      if (AssumeInst *II = dyn_cast<AssumeInst>(&I))
+        AC->registerAssumption(II);
+    }
+
+    updateSuccessorPhis(BB, NewBB, NextState, VMap, DuplicateMap);
+    updatePredecessor(PrevBB, BB, NewBB, DTU);
+    updateDefMap(NewDefs, VMap);
+
+    // Add all successors to the DominatorTree
+    SmallPtrSet<BasicBlock *, 4> SuccSet;
+    for (auto *SuccBB : successors(NewBB)) {
+      if (SuccSet.insert(SuccBB).second)
+        DTU->applyUpdates({{DominatorTree::Insert, NewBB, SuccBB}});
+    }
+    SuccSet.clear();
+    return NewBB;
+  }
+
+  /// Update the phi nodes in BB's successors.
+  ///
+  /// This means creating a new incoming value from NewBB with the new
+  /// instruction wherever there is an incoming value from BB.
+  void updateSuccessorPhis(BasicBlock *BB, BasicBlock *ClonedBB,
+                           uint64_t NextState, ValueToValueMapTy &VMap,
+                           DuplicateBlockMap &DuplicateMap) {
+    std::vector<BasicBlock *> BlocksToUpdate;
+
+    // If BB is the last block in the path, we can simply update the one case
+    // successor that will be reached.
+    if (BB == SwitchPaths->getSwitchBlock()) {
+      SwitchInst *Switch = SwitchPaths->getSwitchInst();
+      BasicBlock *NextCase = getNextCaseSuccessor(Switch, NextState);
+      BlocksToUpdate.push_back(NextCase);
+      BasicBlock *ClonedSucc = getClonedBB(NextCase, NextState, DuplicateMap);
+      if (ClonedSucc)
+        BlocksToUpdate.push_back(ClonedSucc);
+    }
+    // Otherwise update phis in all successors.
+    else {
+      for (BasicBlock *Succ : successors(BB)) {
+        BlocksToUpdate.push_back(Succ);
+
+        // Check if a successor has already been cloned for the particular exit
+        // value. In this case if a successor was already cloned, the phi nodes
+        // in the cloned block should be updated directly.
+        BasicBlock *ClonedSucc = getClonedBB(Succ, NextState, DuplicateMap);
+        if (ClonedSucc)
+          BlocksToUpdate.push_back(ClonedSucc);
+      }
+    }
+
+    // If there is a phi with an incoming value from BB, create a new incoming
+    // value for the new predecessor ClonedBB. The value will either be the same
+    // value from BB or a cloned value.
+    for (BasicBlock *Succ : BlocksToUpdate) {
+      for (auto II = Succ->begin(); PHINode *Phi = dyn_cast<PHINode>(II);
+           ++II) {
+        Value *Incoming = Phi->getIncomingValueForBlock(BB);
+        if (Incoming) {
+          if (isa<Constant>(Incoming)) {
+            Phi->addIncoming(Incoming, ClonedBB);
+            continue;
+          }
+          Value *ClonedVal = VMap[Incoming];
+          if (ClonedVal)
+            Phi->addIncoming(ClonedVal, ClonedBB);
+          else
+            Phi->addIncoming(Incoming, ClonedBB);
+        }
+      }
+    }
+  }
+
+  /// Sets the successor of PrevBB to be NewBB instead of OldBB. Note that all
+  /// other successors are kept as well.
+  void updatePredecessor(BasicBlock *PrevBB, BasicBlock *OldBB,
+                         BasicBlock *NewBB, DomTreeUpdater *DTU) {
+    // When a path is reused, there is a chance that predecessors were already
+    // updated before. Check if the predecessor needs to be updated first.
+    if (!isPredecessor(OldBB, PrevBB))
+      return;
+
+    Instruction *PrevTerm = PrevBB->getTerminator();
+    for (unsigned Idx = 0; Idx < PrevTerm->getNumSuccessors(); Idx++) {
+      if (PrevTerm->getSuccessor(Idx) == OldBB) {
+        OldBB->removePredecessor(PrevBB, /* KeepOneInputPHIs = */ true);
+        PrevTerm->setSuccessor(Idx, NewBB);
+      }
+    }
+    DTU->applyUpdates({{DominatorTree::Delete, PrevBB, OldBB},
+                       {DominatorTree::Insert, PrevBB, NewBB}});
+  }
+
+  /// Add new value mappings to the DefMap to keep track of all new definitions
+  /// for a particular instruction. These will be used while updating SSA form.
+  void updateDefMap(DefMap &NewDefs, ValueToValueMapTy &VMap) {
+    for (auto Entry : VMap) {
+      Instruction *Inst =
+          dyn_cast<Instruction>(const_cast<Value *>(Entry.first));
+      if (!Inst || !Entry.second || isa<BranchInst>(Inst) ||
+          isa<SwitchInst>(Inst)) {
+        continue;
+      }
+
+      Instruction *Cloned = dyn_cast<Instruction>(Entry.second);
+      if (!Cloned)
+        continue;
+
+      if (NewDefs.find(Inst) == NewDefs.end())
+        NewDefs[Inst] = {Cloned};
+      else
+        NewDefs[Inst].push_back(Cloned);
+    }
+  }
+
+  /// Update the last branch of a particular cloned path to point to the correct
+  /// case successor.
+  ///
+  /// Note that this is an optional step and would have been done in later
+  /// optimizations, but it makes the CFG significantly easier to work with.
+  void updateLastSuccessor(ThreadingPath &TPath,
+                           DuplicateBlockMap &DuplicateMap,
+                           DomTreeUpdater *DTU) {
+    uint64_t NextState = TPath.getExitValue();
+    BasicBlock *BB = TPath.getPath().back();
+    BasicBlock *LastBlock = getClonedBB(BB, NextState, DuplicateMap);
+
+    // Note multiple paths can end at the same block so check that it is not
+    // updated yet
+    if (!isa<SwitchInst>(LastBlock->getTerminator()))
+      return;
+    SwitchInst *Switch = cast<SwitchInst>(LastBlock->getTerminator());
+    BasicBlock *NextCase = getNextCaseSuccessor(Switch, NextState);
+
+    std::vector<DominatorTree::UpdateType> DTUpdates;
+    SmallPtrSet<BasicBlock *, 4> SuccSet;
+    for (BasicBlock *Succ : successors(LastBlock)) {
+      if (Succ != NextCase && SuccSet.insert(Succ).second)
+        DTUpdates.push_back({DominatorTree::Delete, LastBlock, Succ});
+    }
+
+    Switch->eraseFromParent();
+    BranchInst::Create(NextCase, LastBlock);
+
+    DTU->applyUpdates(DTUpdates);
+  }
+
+  /// After cloning blocks, some of the phi nodes have extra incoming values
+  /// that are no longer used. This function removes them.
+  void cleanPhiNodes(BasicBlock *BB) {
+    // If BB is no longer reachable, remove any remaining phi nodes
+    if (pred_empty(BB)) {
+      std::vector<PHINode *> PhiToRemove;
+      for (auto II = BB->begin(); PHINode *Phi = dyn_cast<PHINode>(II); ++II) {
+        PhiToRemove.push_back(Phi);
+      }
+      for (PHINode *PN : PhiToRemove) {
+        PN->replaceAllUsesWith(UndefValue::get(PN->getType()));
+        PN->eraseFromParent();
+      }
+      return;
+    }
+
+    // Remove any incoming values that come from an invalid predecessor
+    for (auto II = BB->begin(); PHINode *Phi = dyn_cast<PHINode>(II); ++II) {
+      std::vector<BasicBlock *> BlocksToRemove;
+      for (BasicBlock *IncomingBB : Phi->blocks()) {
+        if (!isPredecessor(BB, IncomingBB))
+          BlocksToRemove.push_back(IncomingBB);
+      }
+      for (BasicBlock *BB : BlocksToRemove)
+        Phi->removeIncomingValue(BB);
+    }
+  }
+
+  /// Checks if BB was already cloned for a particular next state value. If it
+  /// was then it returns this cloned block, and otherwise null.
+  BasicBlock *getClonedBB(BasicBlock *BB, uint64_t NextState,
+                          DuplicateBlockMap &DuplicateMap) {
+    CloneList ClonedBBs = DuplicateMap[BB];
+
+    // Find an entry in the CloneList with this NextState. If it exists then
+    // return the corresponding BB
+    auto It = llvm::find_if(ClonedBBs, [NextState](const ClonedBlock &C) {
+      return C.State == NextState;
+    });
+    return It != ClonedBBs.end() ? (*It).BB : nullptr;
+  }
+
+  /// Helper to get the successor corresponding to a particular case value for
+  /// a switch statement.
+  BasicBlock *getNextCaseSuccessor(SwitchInst *Switch, uint64_t NextState) {
+    BasicBlock *NextCase = nullptr;
+    for (auto Case : Switch->cases()) {
+      if (Case.getCaseValue()->getZExtValue() == NextState) {
+        NextCase = Case.getCaseSuccessor();
+        break;
+      }
+    }
+    if (!NextCase)
+      NextCase = Switch->getDefaultDest();
+    return NextCase;
+  }
+
+  /// Returns true if IncomingBB is a predecessor of BB.
+  bool isPredecessor(BasicBlock *BB, BasicBlock *IncomingBB) {
+    return llvm::find(predecessors(BB), IncomingBB) != pred_end(BB);
+  }
+
+  AllSwitchPaths *SwitchPaths;
+  DominatorTree *DT;
+  AssumptionCache *AC;
+  TargetTransformInfo *TTI;
+  OptimizationRemarkEmitter *ORE;
+  SmallPtrSet<const Value *, 32> EphValues;
+  std::vector<ThreadingPath> TPaths;
+};
+
+bool DFAJumpThreading::run(Function &F) {
+  LLVM_DEBUG(dbgs() << "\nDFA Jump threading: " << F.getName() << "\n");
+
+  if (F.hasOptSize()) {
+    LLVM_DEBUG(dbgs() << "Skipping due to the 'minsize' attribute\n");
+    return false;
+  }
+
+  if (ClViewCfgBefore)
+    F.viewCFG();
+
+  SmallVector<AllSwitchPaths, 2> ThreadableLoops;
+  bool MadeChanges = false;
+
+  for (BasicBlock &BB : F) {
+    auto *SI = dyn_cast<SwitchInst>(BB.getTerminator());
+    if (!SI)
+      continue;
+
+    LLVM_DEBUG(dbgs() << "\nCheck if SwitchInst in BB " << BB.getName()
+                      << " is predictable\n");
+    MainSwitch Switch(SI, ORE);
+
+    if (!Switch.getInstr())
+      continue;
+
+    LLVM_DEBUG(dbgs() << "\nSwitchInst in BB " << BB.getName() << " is a "
+                      << "candidate for jump threading\n");
+    LLVM_DEBUG(SI->dump());
+
+    unfoldSelectInstrs(DT, Switch.getSelectInsts());
+    if (!Switch.getSelectInsts().empty())
+      MadeChanges = true;
+
+    AllSwitchPaths SwitchPaths(&Switch, ORE);
+    SwitchPaths.run();
+
+    if (SwitchPaths.getNumThreadingPaths() > 0) {
+      ThreadableLoops.push_back(SwitchPaths);
+
+      // For the time being limit this optimization to occurring once in a
+      // function since it can change the CFG significantly. This is not a
+      // strict requirement but it can cause buggy behavior if there is an
+      // overlap of blocks in different opportunities. There is a lot of room to
+      // experiment with catching more opportunities here.
+      break;
+    }
+  }
+
+  SmallPtrSet<const Value *, 32> EphValues;
+  if (ThreadableLoops.size() > 0)
+    CodeMetrics::collectEphemeralValues(&F, AC, EphValues);
+
+  for (AllSwitchPaths SwitchPaths : ThreadableLoops) {
+    TransformDFA Transform(&SwitchPaths, DT, AC, TTI, ORE, EphValues);
+    Transform.run();
+    MadeChanges = true;
+  }
+
+#ifdef EXPENSIVE_CHECKS
+  assert(DT->verify(DominatorTree::VerificationLevel::Full));
+  verifyFunction(F, &dbgs());
+#endif
+
+  return MadeChanges;
+}
+
+} // end anonymous namespace
+
+/// Integrate with the new Pass Manager
+PreservedAnalyses DFAJumpThreadingPass::run(Function &F,
+                                            FunctionAnalysisManager &AM) {
+  AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(F);
+  DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  TargetTransformInfo &TTI = AM.getResult<TargetIRAnalysis>(F);
+  OptimizationRemarkEmitter ORE(&F);
+
+  if (!DFAJumpThreading(&AC, &DT, &TTI, &ORE).run(F))
+    return PreservedAnalyses::all();
+
+  PreservedAnalyses PA;
+  PA.preserve<DominatorTreeAnalysis>();
+  return PA;
+}
diff --git a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 2979225c6016..d22b3f409585 100644
--- a/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -1,4 +1,4 @@
-//===- DeadStoreElimination.cpp - Fast Dead Store Elimination -------------===//
+//===- DeadStoreElimination.cpp - MemorySSA Backed Dead Store Elimination -===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,11 +6,24 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements a trivial dead store elimination that only considers
-// basic-block local redundant stores.
+// The code below implements dead store elimination using MemorySSA. It uses
+// the following general approach: given a MemoryDef, walk upwards to find
+// clobbering MemoryDefs that may be killed by the starting def. Then check
+// that there are no uses that may read the location of the original MemoryDef
+// in between both MemoryDefs. A bit more concretely:
 //
-// FIXME: This should eventually be extended to be a post-dominator tree
-// traversal.  Doing so would be pretty trivial.
+// For all MemoryDefs StartDef:
+// 1. Get the next dominating clobbering MemoryDef (EarlierAccess) by walking
+//    upwards.
+// 2. Check that there are no reads between EarlierAccess and the StartDef by
+//    checking all uses starting at EarlierAccess and walking until we see
+//    StartDef.
+// 3. For each found CurrentDef, check that:
+//   1. There are no barrier instructions between CurrentDef and StartDef (like
+//       throws or stores with ordering constraints).
+//   2. StartDef is executed whenever CurrentDef is executed.
+//   3. StartDef completely overwrites CurrentDef.
+// 4. Erase CurrentDef from the function and MemorySSA.
 //
 //===----------------------------------------------------------------------===//
 
@@ -27,11 +40,12 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
-#include "llvm/Analysis/MemoryDependenceAnalysis.h"
 #include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/MustExecute.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -105,10 +119,6 @@ EnablePartialStoreMerging("enable-dse-partial-store-merging",
   cl::init(true), cl::Hidden,
   cl::desc("Enable partial store merging in DSE"));
 
-static cl::opt<bool>
-    EnableMemorySSA("enable-dse-memoryssa", cl::init(true), cl::Hidden,
-                    cl::desc("Use the new MemorySSA-backed DSE."));
-
 static cl::opt<unsigned>
     MemorySSAScanLimit("dse-memoryssa-scanlimit", cl::init(150), cl::Hidden,
                        cl::desc("The number of memory instructions to scan for "
@@ -153,69 +163,6 @@ static cl::opt<unsigned> MemorySSAPathCheckLimit(
 using OverlapIntervalsTy = std::map<int64_t, int64_t>;
 using InstOverlapIntervalsTy = DenseMap<Instruction *, OverlapIntervalsTy>;
 
-/// Delete this instruction.  Before we do, go through and zero out all the
-/// operands of this instruction.  If any of them become dead, delete them and
-/// the computation tree that feeds them.
-/// If ValueSet is non-null, remove any deleted instructions from it as well.
-static void
-deleteDeadInstruction(Instruction *I, BasicBlock::iterator *BBI,
-                      MemoryDependenceResults &MD, const TargetLibraryInfo &TLI,
-                      InstOverlapIntervalsTy &IOL,
-                      MapVector<Instruction *, bool> &ThrowableInst,
-                      SmallSetVector<const Value *, 16> *ValueSet = nullptr) {
-  SmallVector<Instruction*, 32> NowDeadInsts;
-
-  NowDeadInsts.push_back(I);
-  --NumFastOther;
-
-  // Keeping the iterator straight is a pain, so we let this routine tell the
-  // caller what the next instruction is after we're done mucking about.
-  BasicBlock::iterator NewIter = *BBI;
-
-  // Before we touch this instruction, remove it from memdep!
-  do {
-    Instruction *DeadInst = NowDeadInsts.pop_back_val();
-    // Mark the DeadInst as dead in the list of throwable instructions.
-    auto It = ThrowableInst.find(DeadInst);
-    if (It != ThrowableInst.end())
-      ThrowableInst[It->first] = false;
-    ++NumFastOther;
-
-    // Try to preserve debug information attached to the dead instruction.
-    salvageDebugInfo(*DeadInst);
-    salvageKnowledge(DeadInst);
-
-    // This instruction is dead, zap it, in stages.  Start by removing it from
-    // MemDep, which needs to know the operands and needs it to be in the
-    // function.
-    MD.removeInstruction(DeadInst);
-
-    for (unsigned op = 0, e = DeadInst->getNumOperands(); op != e; ++op) {
-      Value *Op = DeadInst->getOperand(op);
-      DeadInst->setOperand(op, nullptr);
-
-      // If this operand just became dead, add it to the NowDeadInsts list.
-      if (!Op->use_empty()) continue;
-
-      if (Instruction *OpI = dyn_cast<Instruction>(Op))
-        if (isInstructionTriviallyDead(OpI, &TLI))
-          NowDeadInsts.push_back(OpI);
-    }
-
-    if (ValueSet) ValueSet->remove(DeadInst);
-    IOL.erase(DeadInst);
-
-    if (NewIter == DeadInst->getIterator())
-      NewIter = DeadInst->eraseFromParent();
-    else
-      DeadInst->eraseFromParent();
-  } while (!NowDeadInsts.empty());
-  *BBI = NewIter;
-  // Pop dead entries from back of ThrowableInst till we find an alive entry.
-  while (!ThrowableInst.empty() && !ThrowableInst.back().second)
-    ThrowableInst.pop_back();
-}
-
 /// Does this instruction write some memory?  This only returns true for things
 /// that we can analyze with other helpers below.
 static bool hasAnalyzableMemoryWrite(Instruction *I,
@@ -289,19 +236,6 @@ static MemoryLocation getLocForWrite(Instruction *Inst,
   return MemoryLocation();
 }
 
-/// Return the location read by the specified "hasAnalyzableMemoryWrite"
-/// instruction if any.
-static MemoryLocation getLocForRead(Instruction *Inst,
-                                    const TargetLibraryInfo &TLI) {
-  assert(hasAnalyzableMemoryWrite(Inst, TLI) && "Unknown instruction case");
-
-  // The only instructions that both read and write are the mem transfer
-  // instructions (memcpy/memmove).
-  if (auto *MTI = dyn_cast<AnyMemTransferInst>(Inst))
-    return MemoryLocation::getForSource(MTI);
-  return MemoryLocation();
-}
-
 /// If the value of this instruction and the memory it writes to is unused, may
 /// we delete this instruction?
 static bool isRemovable(Instruction *I) {
@@ -373,17 +307,6 @@ static bool isShortenableAtTheBeginning(Instruction *I) {
   return isa<AnyMemSetInst>(I);
 }
 
-/// Return the pointer that is being written to.
-static Value *getStoredPointerOperand(Instruction *I,
-                                      const TargetLibraryInfo &TLI) {
-  //TODO: factor this to reuse getLocForWrite
-  MemoryLocation Loc = getLocForWrite(I, TLI);
-  assert(Loc.Ptr &&
-         "unable to find pointer written for analyzable instruction?");
-  // TODO: most APIs don't expect const Value *
-  return const_cast<Value*>(Loc.Ptr);
-}
-
 static uint64_t getPointerSize(const Value *V, const DataLayout &DL,
                                const TargetLibraryInfo &TLI,
                                const Function *F) {
@@ -412,10 +335,9 @@ enum OverwriteResult {
 /// Check if two instruction are masked stores that completely
 /// overwrite one another. More specifically, \p Later has to
 /// overwrite \p Earlier.
-template <typename AATy>
 static OverwriteResult isMaskedStoreOverwrite(const Instruction *Later,
                                               const Instruction *Earlier,
-                                              AATy &AA) {
+                                              BatchAAResults &AA) {
   const auto *IIL = dyn_cast<IntrinsicInst>(Later);
   const auto *IIE = dyn_cast<IntrinsicInst>(Earlier);
   if (IIL == nullptr || IIE == nullptr)
@@ -435,106 +357,6 @@ static OverwriteResult isMaskedStoreOverwrite(const Instruction *Later,
   return OW_Complete;
 }
 
-/// Return 'OW_Complete' if a store to the 'Later' location (by \p LaterI
-/// instruction) completely overwrites a store to the 'Earlier' location.
-/// (by \p EarlierI instruction).
-/// Return OW_MaybePartial if \p Later does not completely overwrite
-/// \p Earlier, but they both write to the same underlying object. In that
-/// case, use isPartialOverwrite to check if \p Later partially overwrites
-/// \p Earlier. Returns 'OW_Unknown' if nothing can be determined.
-template <typename AATy>
-static OverwriteResult
-isOverwrite(const Instruction *LaterI, const Instruction *EarlierI,
-            const MemoryLocation &Later, const MemoryLocation &Earlier,
-            const DataLayout &DL, const TargetLibraryInfo &TLI,
-            int64_t &EarlierOff, int64_t &LaterOff, AATy &AA,
-            const Function *F) {
-  // FIXME: Vet that this works for size upper-bounds. Seems unlikely that we'll
-  // get imprecise values here, though (except for unknown sizes).
-  if (!Later.Size.isPrecise() || !Earlier.Size.isPrecise()) {
-    // Masked stores have imprecise locations, but we can reason about them
-    // to some extent.
-    return isMaskedStoreOverwrite(LaterI, EarlierI, AA);
-  }
-
-  const uint64_t LaterSize = Later.Size.getValue();
-  const uint64_t EarlierSize = Earlier.Size.getValue();
-
-  const Value *P1 = Earlier.Ptr->stripPointerCasts();
-  const Value *P2 = Later.Ptr->stripPointerCasts();
-
-  // If the start pointers are the same, we just have to compare sizes to see if
-  // the later store was larger than the earlier store.
-  if (P1 == P2 || AA.isMustAlias(P1, P2)) {
-    // Make sure that the Later size is >= the Earlier size.
-    if (LaterSize >= EarlierSize)
-      return OW_Complete;
-  }
-
-  // Check to see if the later store is to the entire object (either a global,
-  // an alloca, or a byval/inalloca argument).  If so, then it clearly
-  // overwrites any other store to the same object.
-  const Value *UO1 = getUnderlyingObject(P1), *UO2 = getUnderlyingObject(P2);
-
-  // If we can't resolve the same pointers to the same object, then we can't
-  // analyze them at all.
-  if (UO1 != UO2)
-    return OW_Unknown;
-
-  // If the "Later" store is to a recognizable object, get its size.
-  uint64_t ObjectSize = getPointerSize(UO2, DL, TLI, F);
-  if (ObjectSize != MemoryLocation::UnknownSize)
-    if (ObjectSize == LaterSize && ObjectSize >= EarlierSize)
-      return OW_Complete;
-
-  // Okay, we have stores to two completely different pointers.  Try to
-  // decompose the pointer into a "base + constant_offset" form.  If the base
-  // pointers are equal, then we can reason about the two stores.
-  EarlierOff = 0;
-  LaterOff = 0;
-  const Value *BP1 = GetPointerBaseWithConstantOffset(P1, EarlierOff, DL);
-  const Value *BP2 = GetPointerBaseWithConstantOffset(P2, LaterOff, DL);
-
-  // If the base pointers still differ, we have two completely different stores.
-  if (BP1 != BP2)
-    return OW_Unknown;
-
-  // The later access completely overlaps the earlier store if and only if
-  // both start and end of the earlier one is "inside" the later one:
-  //    |<->|--earlier--|<->|
-  //    |-------later-------|
-  // Accesses may overlap if and only if start of one of them is "inside"
-  // another one:
-  //    |<->|--earlier--|<----->|
-  //    |-------later-------|
-  //           OR
-  //    |----- earlier -----|
-  //    |<->|---later---|<----->|
-  //
-  // We have to be careful here as *Off is signed while *.Size is unsigned.
-
-  // Check if the earlier access starts "not before" the later one.
-  if (EarlierOff >= LaterOff) {
-    // If the earlier access ends "not after" the later access then the earlier
-    // one is completely overwritten by the later one.
-    if (uint64_t(EarlierOff - LaterOff) + EarlierSize <= LaterSize)
-      return OW_Complete;
-    // If start of the earlier access is "before" end of the later access then
-    // accesses overlap.
-    else if ((uint64_t)(EarlierOff - LaterOff) < LaterSize)
-      return OW_MaybePartial;
-  }
-  // If start of the later access is "before" end of the earlier access then
-  // accesses overlap.
-  else if ((uint64_t)(LaterOff - EarlierOff) < EarlierSize) {
-    return OW_MaybePartial;
-  }
-
-  // Can reach here only if accesses are known not to overlap. There is no
-  // dedicated code to indicate no overlap so signal "unknown".
-  return OW_Unknown;
-}
-
 /// Return 'OW_Complete' if a store to the 'Later' location completely
 /// overwrites a store to the 'Earlier' location, 'OW_End' if the end of the
 /// 'Earlier' location is completely overwritten by 'Later', 'OW_Begin' if the
@@ -659,69 +481,14 @@ static OverwriteResult isPartialOverwrite(const MemoryLocation &Later,
   return OW_Unknown;
 }
 
-/// If 'Inst' might be a self read (i.e. a noop copy of a
-/// memory region into an identical pointer) then it doesn't actually make its
-/// input dead in the traditional sense.  Consider this case:
-///
-///   memmove(A <- B)
-///   memmove(A <- A)
-///
-/// In this case, the second store to A does not make the first store to A dead.
-/// The usual situation isn't an explicit A<-A store like this (which can be
-/// trivially removed) but a case where two pointers may alias.
-///
-/// This function detects when it is unsafe to remove a dependent instruction
-/// because the DSE inducing instruction may be a self-read.
-static bool isPossibleSelfRead(Instruction *Inst,
-                               const MemoryLocation &InstStoreLoc,
-                               Instruction *DepWrite,
-                               const TargetLibraryInfo &TLI,
-                               AliasAnalysis &AA) {
-  // Self reads can only happen for instructions that read memory.  Get the
-  // location read.
-  MemoryLocation InstReadLoc = getLocForRead(Inst, TLI);
-  if (!InstReadLoc.Ptr)
-    return false; // Not a reading instruction.
-
-  // If the read and written loc obviously don't alias, it isn't a read.
-  if (AA.isNoAlias(InstReadLoc, InstStoreLoc))
-    return false;
-
-  if (isa<AnyMemCpyInst>(Inst)) {
-    // LLVM's memcpy overlap semantics are not fully fleshed out (see PR11763)
-    // but in practice memcpy(A <- B) either means that A and B are disjoint or
-    // are equal (i.e. there are not partial overlaps).  Given that, if we have:
-    //
-    //   memcpy/memmove(A <- B)  // DepWrite
-    //   memcpy(A <- B)  // Inst
-    //
-    // with Inst reading/writing a >= size than DepWrite, we can reason as
-    // follows:
-    //
-    //   - If A == B then both the copies are no-ops, so the DepWrite can be
-    //     removed.
-    //   - If A != B then A and B are disjoint locations in Inst.  Since
-    //     Inst.size >= DepWrite.size A and B are disjoint in DepWrite too.
-    //     Therefore DepWrite can be removed.
-    MemoryLocation DepReadLoc = getLocForRead(DepWrite, TLI);
-
-    if (DepReadLoc.Ptr && AA.isMustAlias(InstReadLoc.Ptr, DepReadLoc.Ptr))
-      return false;
-  }
-
-  // If DepWrite doesn't read memory or if we can't prove it is a must alias,
-  // then it can't be considered dead.
-  return true;
-}
-
 /// Returns true if the memory which is accessed by the second instruction is not
 /// modified between the first and the second instruction.
 /// Precondition: Second instruction must be dominated by the first
 /// instruction.
-template <typename AATy>
 static bool
-memoryIsNotModifiedBetween(Instruction *FirstI, Instruction *SecondI, AATy &AA,
-                           const DataLayout &DL, DominatorTree *DT) {
+memoryIsNotModifiedBetween(Instruction *FirstI, Instruction *SecondI,
+                           BatchAAResults &AA, const DataLayout &DL,
+                           DominatorTree *DT) {
   // Do a backwards scan through the CFG from SecondI to FirstI. Look for
   // instructions which can modify the memory location accessed by SecondI.
   //
@@ -776,16 +543,16 @@ memoryIsNotModifiedBetween(Instruction *FirstI, Instruction *SecondI, AATy &AA,
     if (B != FirstBB) {
       assert(B != &FirstBB->getParent()->getEntryBlock() &&
           "Should not hit the entry block because SI must be dominated by LI");
-      for (auto PredI = pred_begin(B), PE = pred_end(B); PredI != PE; ++PredI) {
+      for (BasicBlock *Pred : predecessors(B)) {
         PHITransAddr PredAddr = Addr;
         if (PredAddr.NeedsPHITranslationFromBlock(B)) {
           if (!PredAddr.IsPotentiallyPHITranslatable())
             return false;
-          if (PredAddr.PHITranslateValue(B, *PredI, DT, false))
+          if (PredAddr.PHITranslateValue(B, Pred, DT, false))
             return false;
         }
         Value *TranslatedPtr = PredAddr.getAddr();
-        auto Inserted = Visited.insert(std::make_pair(*PredI, TranslatedPtr));
+        auto Inserted = Visited.insert(std::make_pair(Pred, TranslatedPtr));
         if (!Inserted.second) {
           // We already visited this block before. If it was with a different
           // address - bail out!
@@ -794,332 +561,111 @@ memoryIsNotModifiedBetween(Instruction *FirstI, Instruction *SecondI, AATy &AA,
           // ... otherwise just skip it.
           continue;
         }
-        WorkList.push_back(std::make_pair(*PredI, PredAddr));
+        WorkList.push_back(std::make_pair(Pred, PredAddr));
       }
     }
   }
   return true;
 }
 
-/// Find all blocks that will unconditionally lead to the block BB and append
-/// them to F.
-static void findUnconditionalPreds(SmallVectorImpl<BasicBlock *> &Blocks,
-                                   BasicBlock *BB, DominatorTree *DT) {
-  for (pred_iterator I = pred_begin(BB), E = pred_end(BB); I != E; ++I) {
-    BasicBlock *Pred = *I;
-    if (Pred == BB) continue;
-    Instruction *PredTI = Pred->getTerminator();
-    if (PredTI->getNumSuccessors() != 1)
-      continue;
-
-    if (DT->isReachableFromEntry(Pred))
-      Blocks.push_back(Pred);
-  }
-}
-
-/// Handle frees of entire structures whose dependency is a store
-/// to a field of that structure.
-static bool handleFree(CallInst *F, AliasAnalysis *AA,
-                       MemoryDependenceResults *MD, DominatorTree *DT,
-                       const TargetLibraryInfo *TLI,
-                       InstOverlapIntervalsTy &IOL,
-                       MapVector<Instruction *, bool> &ThrowableInst) {
-  bool MadeChange = false;
-
-  MemoryLocation Loc = MemoryLocation::getAfter(F->getOperand(0));
-  SmallVector<BasicBlock *, 16> Blocks;
-  Blocks.push_back(F->getParent());
-
-  while (!Blocks.empty()) {
-    BasicBlock *BB = Blocks.pop_back_val();
-    Instruction *InstPt = BB->getTerminator();
-    if (BB == F->getParent()) InstPt = F;
-
-    MemDepResult Dep =
-        MD->getPointerDependencyFrom(Loc, false, InstPt->getIterator(), BB);
-    while (Dep.isDef() || Dep.isClobber()) {
-      Instruction *Dependency = Dep.getInst();
-      if (!hasAnalyzableMemoryWrite(Dependency, *TLI) ||
-          !isRemovable(Dependency))
-        break;
-
-      Value *DepPointer =
-          getUnderlyingObject(getStoredPointerOperand(Dependency, *TLI));
-
-      // Check for aliasing.
-      if (!AA->isMustAlias(F->getArgOperand(0), DepPointer))
-        break;
-
-      LLVM_DEBUG(
-          dbgs() << "DSE: Dead Store to soon to be freed memory:\n  DEAD: "
-                 << *Dependency << '\n');
-
-      // DCE instructions only used to calculate that store.
-      BasicBlock::iterator BBI(Dependency);
-      deleteDeadInstruction(Dependency, &BBI, *MD, *TLI, IOL,
-                            ThrowableInst);
-      ++NumFastStores;
-      MadeChange = true;
-
-      // Inst's old Dependency is now deleted. Compute the next dependency,
-      // which may also be dead, as in
-      //    s[0] = 0;
-      //    s[1] = 0; // This has just been deleted.
-      //    free(s);
-      Dep = MD->getPointerDependencyFrom(Loc, false, BBI, BB);
-    }
-
-    if (Dep.isNonLocal())
-      findUnconditionalPreds(Blocks, BB, DT);
-  }
-
-  return MadeChange;
-}
-
-/// Check to see if the specified location may alias any of the stack objects in
-/// the DeadStackObjects set. If so, they become live because the location is
-/// being loaded.
-static void removeAccessedObjects(const MemoryLocation &LoadedLoc,
-                                  SmallSetVector<const Value *, 16> &DeadStackObjects,
-                                  const DataLayout &DL, AliasAnalysis *AA,
-                                  const TargetLibraryInfo *TLI,
-                                  const Function *F) {
-  const Value *UnderlyingPointer = getUnderlyingObject(LoadedLoc.Ptr);
-
-  // A constant can't be in the dead pointer set.
-  if (isa<Constant>(UnderlyingPointer))
-    return;
-
-  // If the kill pointer can be easily reduced to an alloca, don't bother doing
-  // extraneous AA queries.
-  if (isa<AllocaInst>(UnderlyingPointer) || isa<Argument>(UnderlyingPointer)) {
-    DeadStackObjects.remove(UnderlyingPointer);
-    return;
-  }
-
-  // Remove objects that could alias LoadedLoc.
-  DeadStackObjects.remove_if([&](const Value *I) {
-    // See if the loaded location could alias the stack location.
-    MemoryLocation StackLoc(I, getPointerSize(I, DL, *TLI, F));
-    return !AA->isNoAlias(StackLoc, LoadedLoc);
-  });
-}
-
-/// Remove dead stores to stack-allocated locations in the function end block.
-/// Ex:
-/// %A = alloca i32
-/// ...
-/// store i32 1, i32* %A
-/// ret void
-static bool handleEndBlock(BasicBlock &BB, AliasAnalysis *AA,
-                           MemoryDependenceResults *MD,
-                           const TargetLibraryInfo *TLI,
-                           InstOverlapIntervalsTy &IOL,
-                           MapVector<Instruction *, bool> &ThrowableInst) {
-  bool MadeChange = false;
-
-  // Keep track of all of the stack objects that are dead at the end of the
-  // function.
-  SmallSetVector<const Value*, 16> DeadStackObjects;
-
-  // Find all of the alloca'd pointers in the entry block.
-  BasicBlock &Entry = BB.getParent()->front();
-  for (Instruction &I : Entry) {
-    if (isa<AllocaInst>(&I))
-      DeadStackObjects.insert(&I);
-
-    // Okay, so these are dead heap objects, but if the pointer never escapes
-    // then it's leaked by this function anyways.
-    else if (isAllocLikeFn(&I, TLI) && !PointerMayBeCaptured(&I, true, true))
-      DeadStackObjects.insert(&I);
-  }
-
-  // Treat byval or inalloca arguments the same, stores to them are dead at the
-  // end of the function.
-  for (Argument &AI : BB.getParent()->args())
-    if (AI.hasPassPointeeByValueCopyAttr())
-      DeadStackObjects.insert(&AI);
-
-  const DataLayout &DL = BB.getModule()->getDataLayout();
-
-  // Scan the basic block backwards
-  for (BasicBlock::iterator BBI = BB.end(); BBI != BB.begin(); ){
-    --BBI;
-
-    // If we find a store, check to see if it points into a dead stack value.
-    if (hasAnalyzableMemoryWrite(&*BBI, *TLI) && isRemovable(&*BBI)) {
-      // See through pointer-to-pointer bitcasts
-      SmallVector<const Value *, 4> Pointers;
-      getUnderlyingObjects(getStoredPointerOperand(&*BBI, *TLI), Pointers);
-
-      // Stores to stack values are valid candidates for removal.
-      bool AllDead = true;
-      for (const Value *Pointer : Pointers)
-        if (!DeadStackObjects.count(Pointer)) {
-          AllDead = false;
-          break;
-        }
-
-      if (AllDead) {
-        Instruction *Dead = &*BBI;
-
-        LLVM_DEBUG(dbgs() << "DSE: Dead Store at End of Block:\n  DEAD: "
-                          << *Dead << "\n  Objects: ";
-                   for (SmallVectorImpl<const Value *>::iterator I =
-                            Pointers.begin(),
-                        E = Pointers.end();
-                        I != E; ++I) {
-                     dbgs() << **I;
-                     if (std::next(I) != E)
-                       dbgs() << ", ";
-                   } dbgs()
-                   << '\n');
-
-        // DCE instructions only used to calculate that store.
-        deleteDeadInstruction(Dead, &BBI, *MD, *TLI, IOL, ThrowableInst,
-                              &DeadStackObjects);
-        ++NumFastStores;
-        MadeChange = true;
-        continue;
-      }
-    }
-
-    // Remove any dead non-memory-mutating instructions.
-    if (isInstructionTriviallyDead(&*BBI, TLI)) {
-      LLVM_DEBUG(dbgs() << "DSE: Removing trivially dead instruction:\n  DEAD: "
-                        << *&*BBI << '\n');
-      deleteDeadInstruction(&*BBI, &BBI, *MD, *TLI, IOL, ThrowableInst,
-                            &DeadStackObjects);
-      ++NumFastOther;
-      MadeChange = true;
-      continue;
-    }
-
-    if (isa<AllocaInst>(BBI)) {
-      // Remove allocas from the list of dead stack objects; there can't be
-      // any references before the definition.
-      DeadStackObjects.remove(&*BBI);
-      continue;
-    }
-
-    if (auto *Call = dyn_cast<CallBase>(&*BBI)) {
-      // Remove allocation function calls from the list of dead stack objects;
-      // there can't be any references before the definition.
-      if (isAllocLikeFn(&*BBI, TLI))
-        DeadStackObjects.remove(&*BBI);
-
-      // If this call does not access memory, it can't be loading any of our
-      // pointers.
-      if (AA->doesNotAccessMemory(Call))
-        continue;
-
-      // If the call might load from any of our allocas, then any store above
-      // the call is live.
-      DeadStackObjects.remove_if([&](const Value *I) {
-        // See if the call site touches the value.
-        return isRefSet(AA->getModRefInfo(
-            Call, I, getPointerSize(I, DL, *TLI, BB.getParent())));
-      });
-
-      // If all of the allocas were clobbered by the call then we're not going
-      // to find anything else to process.
-      if (DeadStackObjects.empty())
-        break;
+static bool tryToShorten(Instruction *EarlierWrite, int64_t &EarlierStart,
+                         uint64_t &EarlierSize, int64_t LaterStart,
+                         uint64_t LaterSize, bool IsOverwriteEnd) {
+  auto *EarlierIntrinsic = cast<AnyMemIntrinsic>(EarlierWrite);
+  Align PrefAlign = EarlierIntrinsic->getDestAlign().valueOrOne();
 
-      continue;
-    }
+  // We assume that memet/memcpy operates in chunks of the "largest" native
+  // type size and aligned on the same value. That means optimal start and size
+  // of memset/memcpy should be modulo of preferred alignment of that type. That
+  // is it there is no any sense in trying to reduce store size any further
+  // since any "extra" stores comes for free anyway.
+  // On the other hand, maximum alignment we can achieve is limited by alignment
+  // of initial store.
 
-    // We can remove the dead stores, irrespective of the fence and its ordering
-    // (release/acquire/seq_cst). Fences only constraints the ordering of
-    // already visible stores, it does not make a store visible to other
-    // threads. So, skipping over a fence does not change a store from being
-    // dead.
-    if (isa<FenceInst>(*BBI))
-      continue;
+  // TODO: Limit maximum alignment by preferred (or abi?) alignment of the
+  // "largest" native type.
+  // Note: What is the proper way to get that value?
+  // Should TargetTransformInfo::getRegisterBitWidth be used or anything else?
+  // PrefAlign = std::min(DL.getPrefTypeAlign(LargestType), PrefAlign);
 
-    MemoryLocation LoadedLoc;
-
-    // If we encounter a use of the pointer, it is no longer considered dead
-    if (LoadInst *L = dyn_cast<LoadInst>(BBI)) {
-      if (!L->isUnordered()) // Be conservative with atomic/volatile load
-        break;
-      LoadedLoc = MemoryLocation::get(L);
-    } else if (VAArgInst *V = dyn_cast<VAArgInst>(BBI)) {
-      LoadedLoc = MemoryLocation::get(V);
-    } else if (!BBI->mayReadFromMemory()) {
-      // Instruction doesn't read memory.  Note that stores that weren't removed
-      // above will hit this case.
-      continue;
-    } else {
-      // Unknown inst; assume it clobbers everything.
-      break;
+  int64_t ToRemoveStart = 0;
+  uint64_t ToRemoveSize = 0;
+  // Compute start and size of the region to remove. Make sure 'PrefAlign' is
+  // maintained on the remaining store.
+  if (IsOverwriteEnd) {
+    // Calculate required adjustment for 'LaterStart'in order to keep remaining
+    // store size aligned on 'PerfAlign'.
+    uint64_t Off =
+        offsetToAlignment(uint64_t(LaterStart - EarlierStart), PrefAlign);
+    ToRemoveStart = LaterStart + Off;
+    if (EarlierSize <= uint64_t(ToRemoveStart - EarlierStart))
+      return false;
+    ToRemoveSize = EarlierSize - uint64_t(ToRemoveStart - EarlierStart);
+  } else {
+    ToRemoveStart = EarlierStart;
+    assert(LaterSize >= uint64_t(EarlierStart - LaterStart) &&
+           "Not overlapping accesses?");
+    ToRemoveSize = LaterSize - uint64_t(EarlierStart - LaterStart);
+    // Calculate required adjustment for 'ToRemoveSize'in order to keep
+    // start of the remaining store aligned on 'PerfAlign'.
+    uint64_t Off = offsetToAlignment(ToRemoveSize, PrefAlign);
+    if (Off != 0) {
+      if (ToRemoveSize <= (PrefAlign.value() - Off))
+        return false;
+      ToRemoveSize -= PrefAlign.value() - Off;
     }
-
-    // Remove any allocas from the DeadPointer set that are loaded, as this
-    // makes any stores above the access live.
-    removeAccessedObjects(LoadedLoc, DeadStackObjects, DL, AA, TLI, BB.getParent());
-
-    // If all of the allocas were clobbered by the access then we're not going
-    // to find anything else to process.
-    if (DeadStackObjects.empty())
-      break;
+    assert(isAligned(PrefAlign, ToRemoveSize) &&
+           "Should preserve selected alignment");
   }
 
-  return MadeChange;
-}
-
-static bool tryToShorten(Instruction *EarlierWrite, int64_t &EarlierOffset,
-                         uint64_t &EarlierSize, int64_t LaterOffset,
-                         uint64_t LaterSize, bool IsOverwriteEnd) {
-  // TODO: base this on the target vector size so that if the earlier
-  // store was too small to get vector writes anyway then its likely
-  // a good idea to shorten it
-  // Power of 2 vector writes are probably always a bad idea to optimize
-  // as any store/memset/memcpy is likely using vector instructions so
-  // shortening it to not vector size is likely to be slower
-  auto *EarlierIntrinsic = cast<AnyMemIntrinsic>(EarlierWrite);
-  unsigned EarlierWriteAlign = EarlierIntrinsic->getDestAlignment();
-  if (!IsOverwriteEnd)
-    LaterOffset = int64_t(LaterOffset + LaterSize);
-
-  if (!(isPowerOf2_64(LaterOffset) && EarlierWriteAlign <= LaterOffset) &&
-      !((EarlierWriteAlign != 0) && LaterOffset % EarlierWriteAlign == 0))
-    return false;
-
-  int64_t NewLength = IsOverwriteEnd
-                          ? LaterOffset - EarlierOffset
-                          : EarlierSize - (LaterOffset - EarlierOffset);
+  assert(ToRemoveSize > 0 && "Shouldn't reach here if nothing to remove");
+  assert(EarlierSize > ToRemoveSize && "Can't remove more than original size");
 
+  uint64_t NewSize = EarlierSize - ToRemoveSize;
   if (auto *AMI = dyn_cast<AtomicMemIntrinsic>(EarlierWrite)) {
     // When shortening an atomic memory intrinsic, the newly shortened
     // length must remain an integer multiple of the element size.
     const uint32_t ElementSize = AMI->getElementSizeInBytes();
-    if (0 != NewLength % ElementSize)
+    if (0 != NewSize % ElementSize)
       return false;
   }
 
   LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  OW "
                     << (IsOverwriteEnd ? "END" : "BEGIN") << ": "
-                    << *EarlierWrite << "\n  KILLER (offset " << LaterOffset
-                    << ", " << EarlierSize << ")\n");
+                    << *EarlierWrite << "\n  KILLER [" << ToRemoveStart << ", "
+                    << int64_t(ToRemoveStart + ToRemoveSize) << ")\n");
 
   Value *EarlierWriteLength = EarlierIntrinsic->getLength();
   Value *TrimmedLength =
-      ConstantInt::get(EarlierWriteLength->getType(), NewLength);
+      ConstantInt::get(EarlierWriteLength->getType(), NewSize);
   EarlierIntrinsic->setLength(TrimmedLength);
+  EarlierIntrinsic->setDestAlignment(PrefAlign);
 
-  EarlierSize = NewLength;
   if (!IsOverwriteEnd) {
-    int64_t OffsetMoved = (LaterOffset - EarlierOffset);
+    Value *OrigDest = EarlierIntrinsic->getRawDest();
+    Type *Int8PtrTy =
+        Type::getInt8PtrTy(EarlierIntrinsic->getContext(),
+                           OrigDest->getType()->getPointerAddressSpace());
+    Value *Dest = OrigDest;
+    if (OrigDest->getType() != Int8PtrTy)
+      Dest = CastInst::CreatePointerCast(OrigDest, Int8PtrTy, "", EarlierWrite);
     Value *Indices[1] = {
-        ConstantInt::get(EarlierWriteLength->getType(), OffsetMoved)};
-    GetElementPtrInst *NewDestGEP = GetElementPtrInst::CreateInBounds(
-        EarlierIntrinsic->getRawDest()->getType()->getPointerElementType(),
-        EarlierIntrinsic->getRawDest(), Indices, "", EarlierWrite);
+        ConstantInt::get(EarlierWriteLength->getType(), ToRemoveSize)};
+    Instruction *NewDestGEP = GetElementPtrInst::CreateInBounds(
+        Type::getInt8Ty(EarlierIntrinsic->getContext()),
+        Dest, Indices, "", EarlierWrite);
     NewDestGEP->setDebugLoc(EarlierIntrinsic->getDebugLoc());
+    if (NewDestGEP->getType() != OrigDest->getType())
+      NewDestGEP = CastInst::CreatePointerCast(NewDestGEP, OrigDest->getType(),
+                                               "", EarlierWrite);
     EarlierIntrinsic->setDest(NewDestGEP);
-    EarlierOffset = EarlierOffset + OffsetMoved;
   }
+
+  // Finally update start and size of earlier access.
+  if (!IsOverwriteEnd)
+    EarlierStart += ToRemoveSize;
+  EarlierSize = NewSize;
+
   return true;
 }
 
@@ -1204,59 +750,10 @@ static bool removePartiallyOverlappedStores(const DataLayout &DL,
   return Changed;
 }
 
-static bool eliminateNoopStore(Instruction *Inst, BasicBlock::iterator &BBI,
-                               AliasAnalysis *AA, MemoryDependenceResults *MD,
-                               const DataLayout &DL,
-                               const TargetLibraryInfo *TLI,
-                               InstOverlapIntervalsTy &IOL,
-                               MapVector<Instruction *, bool> &ThrowableInst,
-                               DominatorTree *DT) {
-  // Must be a store instruction.
-  StoreInst *SI = dyn_cast<StoreInst>(Inst);
-  if (!SI)
-    return false;
-
-  // If we're storing the same value back to a pointer that we just loaded from,
-  // then the store can be removed.
-  if (LoadInst *DepLoad = dyn_cast<LoadInst>(SI->getValueOperand())) {
-    if (SI->getPointerOperand() == DepLoad->getPointerOperand() &&
-        isRemovable(SI) &&
-        memoryIsNotModifiedBetween(DepLoad, SI, *AA, DL, DT)) {
-
-      LLVM_DEBUG(
-          dbgs() << "DSE: Remove Store Of Load from same pointer:\n  LOAD: "
-                 << *DepLoad << "\n  STORE: " << *SI << '\n');
-
-      deleteDeadInstruction(SI, &BBI, *MD, *TLI, IOL, ThrowableInst);
-      ++NumRedundantStores;
-      return true;
-    }
-  }
-
-  // Remove null stores into the calloc'ed objects
-  Constant *StoredConstant = dyn_cast<Constant>(SI->getValueOperand());
-  if (StoredConstant && StoredConstant->isNullValue() && isRemovable(SI)) {
-    Instruction *UnderlyingPointer =
-        dyn_cast<Instruction>(getUnderlyingObject(SI->getPointerOperand()));
-
-    if (UnderlyingPointer && isCallocLikeFn(UnderlyingPointer, TLI) &&
-        memoryIsNotModifiedBetween(UnderlyingPointer, SI, *AA, DL, DT)) {
-      LLVM_DEBUG(
-          dbgs() << "DSE: Remove null store to the calloc'ed object:\n  DEAD: "
-                 << *Inst << "\n  OBJECT: " << *UnderlyingPointer << '\n');
-
-      deleteDeadInstruction(SI, &BBI, *MD, *TLI, IOL, ThrowableInst);
-      ++NumRedundantStores;
-      return true;
-    }
-  }
-  return false;
-}
-
-template <typename AATy>
 static Constant *tryToMergePartialOverlappingStores(
     StoreInst *Earlier, StoreInst *Later, int64_t InstWriteOffset,
-    int64_t DepWriteOffset, const DataLayout &DL, AATy &AA, DominatorTree *DT) {
+    int64_t DepWriteOffset, const DataLayout &DL, BatchAAResults &AA,
+    DominatorTree *DT) {
 
   if (Earlier && isa<ConstantInt>(Earlier->getValueOperand()) &&
       DL.typeSizeEqualsStoreSize(Earlier->getValueOperand()->getType()) &&
@@ -1298,251 +795,7 @@ static Constant *tryToMergePartialOverlappingStores(
   return nullptr;
 }
 
-static bool eliminateDeadStores(BasicBlock &BB, AliasAnalysis *AA,
-                                MemoryDependenceResults *MD, DominatorTree *DT,
-                                const TargetLibraryInfo *TLI) {
-  const DataLayout &DL = BB.getModule()->getDataLayout();
-  bool MadeChange = false;
-
-  MapVector<Instruction *, bool> ThrowableInst;
-
-  // A map of interval maps representing partially-overwritten value parts.
-  InstOverlapIntervalsTy IOL;
-
-  // Do a top-down walk on the BB.
-  for (BasicBlock::iterator BBI = BB.begin(), BBE = BB.end(); BBI != BBE; ) {
-    // Handle 'free' calls specially.
-    if (CallInst *F = isFreeCall(&*BBI, TLI)) {
-      MadeChange |= handleFree(F, AA, MD, DT, TLI, IOL, ThrowableInst);
-      // Increment BBI after handleFree has potentially deleted instructions.
-      // This ensures we maintain a valid iterator.
-      ++BBI;
-      continue;
-    }
-
-    Instruction *Inst = &*BBI++;
-
-    if (Inst->mayThrow()) {
-      ThrowableInst[Inst] = true;
-      continue;
-    }
-
-    // Check to see if Inst writes to memory.  If not, continue.
-    if (!hasAnalyzableMemoryWrite(Inst, *TLI))
-      continue;
-
-    // eliminateNoopStore will update in iterator, if necessary.
-    if (eliminateNoopStore(Inst, BBI, AA, MD, DL, TLI, IOL,
-                           ThrowableInst, DT)) {
-      MadeChange = true;
-      continue;
-    }
-
-    // If we find something that writes memory, get its memory dependence.
-    MemDepResult InstDep = MD->getDependency(Inst);
-
-    // Ignore any store where we can't find a local dependence.
-    // FIXME: cross-block DSE would be fun. :)
-    if (!InstDep.isDef() && !InstDep.isClobber())
-      continue;
-
-    // Figure out what location is being stored to.
-    MemoryLocation Loc = getLocForWrite(Inst, *TLI);
-
-    // If we didn't get a useful location, fail.
-    if (!Loc.Ptr)
-      continue;
-
-    // Loop until we find a store we can eliminate or a load that
-    // invalidates the analysis. Without an upper bound on the number of
-    // instructions examined, this analysis can become very time-consuming.
-    // However, the potential gain diminishes as we process more instructions
-    // without eliminating any of them. Therefore, we limit the number of
-    // instructions we look at.
-    auto Limit = MD->getDefaultBlockScanLimit();
-    while (InstDep.isDef() || InstDep.isClobber()) {
-      // Get the memory clobbered by the instruction we depend on.  MemDep will
-      // skip any instructions that 'Loc' clearly doesn't interact with.  If we
-      // end up depending on a may- or must-aliased load, then we can't optimize
-      // away the store and we bail out.  However, if we depend on something
-      // that overwrites the memory location we *can* potentially optimize it.
-      //
-      // Find out what memory location the dependent instruction stores.
-      Instruction *DepWrite = InstDep.getInst();
-      if (!hasAnalyzableMemoryWrite(DepWrite, *TLI))
-        break;
-      MemoryLocation DepLoc = getLocForWrite(DepWrite, *TLI);
-      // If we didn't get a useful location, or if it isn't a size, bail out.
-      if (!DepLoc.Ptr)
-        break;
-
-      // Find the last throwable instruction not removed by call to
-      // deleteDeadInstruction.
-      Instruction *LastThrowing = nullptr;
-      if (!ThrowableInst.empty())
-        LastThrowing = ThrowableInst.back().first;
-
-      // Make sure we don't look past a call which might throw. This is an
-      // issue because MemoryDependenceAnalysis works in the wrong direction:
-      // it finds instructions which dominate the current instruction, rather than
-      // instructions which are post-dominated by the current instruction.
-      //
-      // If the underlying object is a non-escaping memory allocation, any store
-      // to it is dead along the unwind edge. Otherwise, we need to preserve
-      // the store.
-      if (LastThrowing && DepWrite->comesBefore(LastThrowing)) {
-        const Value *Underlying = getUnderlyingObject(DepLoc.Ptr);
-        bool IsStoreDeadOnUnwind = isa<AllocaInst>(Underlying);
-        if (!IsStoreDeadOnUnwind) {
-            // We're looking for a call to an allocation function
-            // where the allocation doesn't escape before the last
-            // throwing instruction; PointerMayBeCaptured
-            // reasonably fast approximation.
-            IsStoreDeadOnUnwind = isAllocLikeFn(Underlying, TLI) &&
-                !PointerMayBeCaptured(Underlying, false, true);
-        }
-        if (!IsStoreDeadOnUnwind)
-          break;
-      }
-
-      // If we find a write that is a) removable (i.e., non-volatile), b) is
-      // completely obliterated by the store to 'Loc', and c) which we know that
-      // 'Inst' doesn't load from, then we can remove it.
-      // Also try to merge two stores if a later one only touches memory written
-      // to by the earlier one.
-      if (isRemovable(DepWrite) &&
-          !isPossibleSelfRead(Inst, Loc, DepWrite, *TLI, *AA)) {
-        int64_t InstWriteOffset, DepWriteOffset;
-        OverwriteResult OR = isOverwrite(Inst, DepWrite, Loc, DepLoc, DL, *TLI,
-                                         DepWriteOffset, InstWriteOffset, *AA,
-                                         BB.getParent());
-        if (OR == OW_MaybePartial)
-          OR = isPartialOverwrite(Loc, DepLoc, DepWriteOffset, InstWriteOffset,
-                                  DepWrite, IOL);
-
-        if (OR == OW_Complete) {
-          LLVM_DEBUG(dbgs() << "DSE: Remove Dead Store:\n  DEAD: " << *DepWrite
-                            << "\n  KILLER: " << *Inst << '\n');
-
-          // Delete the store and now-dead instructions that feed it.
-          deleteDeadInstruction(DepWrite, &BBI, *MD, *TLI, IOL,
-                                ThrowableInst);
-          ++NumFastStores;
-          MadeChange = true;
-
-          // We erased DepWrite; start over.
-          InstDep = MD->getDependency(Inst);
-          continue;
-        } else if ((OR == OW_End && isShortenableAtTheEnd(DepWrite)) ||
-                   ((OR == OW_Begin &&
-                     isShortenableAtTheBeginning(DepWrite)))) {
-          assert(!EnablePartialOverwriteTracking && "Do not expect to perform "
-                                                    "when partial-overwrite "
-                                                    "tracking is enabled");
-          // The overwrite result is known, so these must be known, too.
-          uint64_t EarlierSize = DepLoc.Size.getValue();
-          uint64_t LaterSize = Loc.Size.getValue();
-          bool IsOverwriteEnd = (OR == OW_End);
-          MadeChange |= tryToShorten(DepWrite, DepWriteOffset, EarlierSize,
-                                    InstWriteOffset, LaterSize, IsOverwriteEnd);
-        } else if (EnablePartialStoreMerging &&
-                   OR == OW_PartialEarlierWithFullLater) {
-          auto *Earlier = dyn_cast<StoreInst>(DepWrite);
-          auto *Later = dyn_cast<StoreInst>(Inst);
-          if (Constant *C = tryToMergePartialOverlappingStores(
-                  Earlier, Later, InstWriteOffset, DepWriteOffset, DL, *AA,
-                  DT)) {
-            auto *SI = new StoreInst(
-                C, Earlier->getPointerOperand(), false, Earlier->getAlign(),
-                Earlier->getOrdering(), Earlier->getSyncScopeID(), DepWrite);
-
-            unsigned MDToKeep[] = {LLVMContext::MD_dbg, LLVMContext::MD_tbaa,
-                                   LLVMContext::MD_alias_scope,
-                                   LLVMContext::MD_noalias,
-                                   LLVMContext::MD_nontemporal};
-            SI->copyMetadata(*DepWrite, MDToKeep);
-            ++NumModifiedStores;
-
-            // Delete the old stores and now-dead instructions that feed them.
-            deleteDeadInstruction(Inst, &BBI, *MD, *TLI, IOL,
-                                  ThrowableInst);
-            deleteDeadInstruction(DepWrite, &BBI, *MD, *TLI, IOL,
-                                  ThrowableInst);
-            MadeChange = true;
-
-            // We erased DepWrite and Inst (Loc); start over.
-            break;
-          }
-        }
-      }
-
-      // If this is a may-aliased store that is clobbering the store value, we
-      // can keep searching past it for another must-aliased pointer that stores
-      // to the same location.  For example, in:
-      //   store -> P
-      //   store -> Q
-      //   store -> P
-      // we can remove the first store to P even though we don't know if P and Q
-      // alias.
-      if (DepWrite == &BB.front()) break;
-
-      // Can't look past this instruction if it might read 'Loc'.
-      if (isRefSet(AA->getModRefInfo(DepWrite, Loc)))
-        break;
-
-      InstDep = MD->getPointerDependencyFrom(Loc, /*isLoad=*/ false,
-                                             DepWrite->getIterator(), &BB,
-                                             /*QueryInst=*/ nullptr, &Limit);
-    }
-  }
-
-  if (EnablePartialOverwriteTracking)
-    MadeChange |= removePartiallyOverlappedStores(DL, IOL, *TLI);
-
-  // If this block ends in a return, unwind, or unreachable, all allocas are
-  // dead at its end, which means stores to them are also dead.
-  if (BB.getTerminator()->getNumSuccessors() == 0)
-    MadeChange |= handleEndBlock(BB, AA, MD, TLI, IOL, ThrowableInst);
-
-  return MadeChange;
-}
-
-static bool eliminateDeadStores(Function &F, AliasAnalysis *AA,
-                                MemoryDependenceResults *MD, DominatorTree *DT,
-                                const TargetLibraryInfo *TLI) {
-  bool MadeChange = false;
-  for (BasicBlock &BB : F)
-    // Only check non-dead blocks.  Dead blocks may have strange pointer
-    // cycles that will confuse alias analysis.
-    if (DT->isReachableFromEntry(&BB))
-      MadeChange |= eliminateDeadStores(BB, AA, MD, DT, TLI);
-
-  return MadeChange;
-}
-
 namespace {
-//=============================================================================
-// MemorySSA backed dead store elimination.
-//
-// The code below implements dead store elimination using MemorySSA. It uses
-// the following general approach: given a MemoryDef, walk upwards to find
-// clobbering MemoryDefs that may be killed by the starting def. Then check
-// that there are no uses that may read the location of the original MemoryDef
-// in between both MemoryDefs. A bit more concretely:
-//
-// For all MemoryDefs StartDef:
-// 1. Get the next dominating clobbering MemoryDef (EarlierAccess) by walking
-//    upwards.
-// 2. Check that there are no reads between EarlierAccess and the StartDef by
-//    checking all uses starting at EarlierAccess and walking until we see
-//    StartDef.
-// 3. For each found CurrentDef, check that:
-//   1. There are no barrier instructions between CurrentDef and StartDef (like
-//       throws or stores with ordering constraints).
-//   2. StartDef is executed whenever CurrentDef is executed.
-//   3. StartDef completely overwrites CurrentDef.
-// 4. Erase CurrentDef from the function and MemorySSA.
-
 // Returns true if \p I is an intrisnic that does not read or write memory.
 bool isNoopIntrinsic(Instruction *I) {
   if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
@@ -1612,6 +865,11 @@ struct DSEState {
   PostDominatorTree &PDT;
   const TargetLibraryInfo &TLI;
   const DataLayout &DL;
+  const LoopInfo &LI;
+
+  // Whether the function contains any irreducible control flow, useful for
+  // being accurately able to detect loops.
+  bool ContainsIrreducibleLoops;
 
   // All MemoryDefs that potentially could kill other MemDefs.
   SmallVector<MemoryDef *, 64> MemDefs;
@@ -1635,14 +893,15 @@ struct DSEState {
   DenseMap<BasicBlock *, InstOverlapIntervalsTy> IOLs;
 
   DSEState(Function &F, AliasAnalysis &AA, MemorySSA &MSSA, DominatorTree &DT,
-           PostDominatorTree &PDT, const TargetLibraryInfo &TLI)
+           PostDominatorTree &PDT, const TargetLibraryInfo &TLI,
+           const LoopInfo &LI)
       : F(F), AA(AA), BatchAA(AA), MSSA(MSSA), DT(DT), PDT(PDT), TLI(TLI),
-        DL(F.getParent()->getDataLayout()) {}
+        DL(F.getParent()->getDataLayout()), LI(LI) {}
 
   static DSEState get(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
                       DominatorTree &DT, PostDominatorTree &PDT,
-                      const TargetLibraryInfo &TLI) {
-    DSEState State(F, AA, MSSA, DT, PDT, TLI);
+                      const TargetLibraryInfo &TLI, const LoopInfo &LI) {
+    DSEState State(F, AA, MSSA, DT, PDT, TLI, LI);
     // Collect blocks with throwing instructions not modeled in MemorySSA and
     // alloc-like objects.
     unsigned PO = 0;
@@ -1670,9 +929,135 @@ struct DSEState {
         State.InvisibleToCallerAfterRet.insert({&AI, true});
       }
 
+    // Collect whether there is any irreducible control flow in the function.
+    State.ContainsIrreducibleLoops = mayContainIrreducibleControl(F, &LI);
+
     return State;
   }
 
+  /// Return 'OW_Complete' if a store to the 'Later' location (by \p LaterI
+  /// instruction) completely overwrites a store to the 'Earlier' location.
+  /// (by \p EarlierI instruction).
+  /// Return OW_MaybePartial if \p Later does not completely overwrite
+  /// \p Earlier, but they both write to the same underlying object. In that
+  /// case, use isPartialOverwrite to check if \p Later partially overwrites
+  /// \p Earlier. Returns 'OW_Unknown' if nothing can be determined.
+  OverwriteResult
+  isOverwrite(const Instruction *LaterI, const Instruction *EarlierI,
+              const MemoryLocation &Later, const MemoryLocation &Earlier,
+              int64_t &EarlierOff, int64_t &LaterOff) {
+    // AliasAnalysis does not always account for loops. Limit overwrite checks
+    // to dependencies for which we can guarantee they are independant of any
+    // loops they are in.
+    if (!isGuaranteedLoopIndependent(EarlierI, LaterI, Earlier))
+      return OW_Unknown;
+
+    // FIXME: Vet that this works for size upper-bounds. Seems unlikely that we'll
+    // get imprecise values here, though (except for unknown sizes).
+    if (!Later.Size.isPrecise() || !Earlier.Size.isPrecise()) {
+      // In case no constant size is known, try to an IR values for the number
+      // of bytes written and check if they match.
+      const auto *LaterMemI = dyn_cast<MemIntrinsic>(LaterI);
+      const auto *EarlierMemI = dyn_cast<MemIntrinsic>(EarlierI);
+      if (LaterMemI && EarlierMemI) {
+        const Value *LaterV = LaterMemI->getLength();
+        const Value *EarlierV = EarlierMemI->getLength();
+        if (LaterV == EarlierV && BatchAA.isMustAlias(Earlier, Later))
+          return OW_Complete;
+      }
+
+      // Masked stores have imprecise locations, but we can reason about them
+      // to some extent.
+      return isMaskedStoreOverwrite(LaterI, EarlierI, BatchAA);
+    }
+
+    const uint64_t LaterSize = Later.Size.getValue();
+    const uint64_t EarlierSize = Earlier.Size.getValue();
+
+    // Query the alias information
+    AliasResult AAR = BatchAA.alias(Later, Earlier);
+
+    // If the start pointers are the same, we just have to compare sizes to see if
+    // the later store was larger than the earlier store.
+    if (AAR == AliasResult::MustAlias) {
+      // Make sure that the Later size is >= the Earlier size.
+      if (LaterSize >= EarlierSize)
+        return OW_Complete;
+    }
+
+    // If we hit a partial alias we may have a full overwrite
+    if (AAR == AliasResult::PartialAlias && AAR.hasOffset()) {
+      int32_t Off = AAR.getOffset();
+      if (Off >= 0 && (uint64_t)Off + EarlierSize <= LaterSize)
+        return OW_Complete;
+    }
+
+    // Check to see if the later store is to the entire object (either a global,
+    // an alloca, or a byval/inalloca argument).  If so, then it clearly
+    // overwrites any other store to the same object.
+    const Value *P1 = Earlier.Ptr->stripPointerCasts();
+    const Value *P2 = Later.Ptr->stripPointerCasts();
+    const Value *UO1 = getUnderlyingObject(P1), *UO2 = getUnderlyingObject(P2);
+
+    // If we can't resolve the same pointers to the same object, then we can't
+    // analyze them at all.
+    if (UO1 != UO2)
+      return OW_Unknown;
+
+    // If the "Later" store is to a recognizable object, get its size.
+    uint64_t ObjectSize = getPointerSize(UO2, DL, TLI, &F);
+    if (ObjectSize != MemoryLocation::UnknownSize)
+      if (ObjectSize == LaterSize && ObjectSize >= EarlierSize)
+        return OW_Complete;
+
+    // Okay, we have stores to two completely different pointers.  Try to
+    // decompose the pointer into a "base + constant_offset" form.  If the base
+    // pointers are equal, then we can reason about the two stores.
+    EarlierOff = 0;
+    LaterOff = 0;
+    const Value *BP1 = GetPointerBaseWithConstantOffset(P1, EarlierOff, DL);
+    const Value *BP2 = GetPointerBaseWithConstantOffset(P2, LaterOff, DL);
+
+    // If the base pointers still differ, we have two completely different stores.
+    if (BP1 != BP2)
+      return OW_Unknown;
+
+    // The later access completely overlaps the earlier store if and only if
+    // both start and end of the earlier one is "inside" the later one:
+    //    |<->|--earlier--|<->|
+    //    |-------later-------|
+    // Accesses may overlap if and only if start of one of them is "inside"
+    // another one:
+    //    |<->|--earlier--|<----->|
+    //    |-------later-------|
+    //           OR
+    //    |----- earlier -----|
+    //    |<->|---later---|<----->|
+    //
+    // We have to be careful here as *Off is signed while *.Size is unsigned.
+
+    // Check if the earlier access starts "not before" the later one.
+    if (EarlierOff >= LaterOff) {
+      // If the earlier access ends "not after" the later access then the earlier
+      // one is completely overwritten by the later one.
+      if (uint64_t(EarlierOff - LaterOff) + EarlierSize <= LaterSize)
+        return OW_Complete;
+      // If start of the earlier access is "before" end of the later access then
+      // accesses overlap.
+      else if ((uint64_t)(EarlierOff - LaterOff) < LaterSize)
+        return OW_MaybePartial;
+    }
+    // If start of the later access is "before" end of the earlier access then
+    // accesses overlap.
+    else if ((uint64_t)(LaterOff - EarlierOff) < EarlierSize) {
+      return OW_MaybePartial;
+    }
+
+    // Can reach here only if accesses are known not to overlap. There is no
+    // dedicated code to indicate no overlap so signal "unknown".
+    return OW_Unknown;
+  }
+
   bool isInvisibleToCallerAfterRet(const Value *V) {
     if (isa<AllocaInst>(V))
       return true;
@@ -1760,8 +1145,8 @@ struct DSEState {
 
     int64_t InstWriteOffset, DepWriteOffset;
     if (auto CC = getLocForWriteEx(UseInst))
-      return isOverwrite(UseInst, DefInst, *CC, DefLoc, DL, TLI, DepWriteOffset,
-                         InstWriteOffset, BatchAA, &F) == OW_Complete;
+      return isOverwrite(UseInst, DefInst, *CC, DefLoc, DepWriteOffset,
+                         InstWriteOffset) == OW_Complete;
     return false;
   }
 
@@ -1864,9 +1249,8 @@ struct DSEState {
       return BatchAA.isMustAlias(TermLoc.Ptr, LocUO);
     }
     int64_t InstWriteOffset, DepWriteOffset;
-    return isOverwrite(MaybeTerm, AccessI, TermLoc, Loc, DL, TLI,
-                       DepWriteOffset, InstWriteOffset, BatchAA,
-                       &F) == OW_Complete;
+    return isOverwrite(MaybeTerm, AccessI, TermLoc, Loc, DepWriteOffset,
+                       InstWriteOffset) == OW_Complete;
   }
 
   // Returns true if \p Use may read from \p DefLoc.
@@ -1893,11 +1277,33 @@ struct DSEState {
     return isRefSet(BatchAA.getModRefInfo(UseInst, DefLoc));
   }
 
+  /// Returns true if a dependency between \p Current and \p KillingDef is
+  /// guaranteed to be loop invariant for the loops that they are in. Either
+  /// because they are known to be in the same block, in the same loop level or
+  /// by guaranteeing that \p CurrentLoc only references a single MemoryLocation
+  /// during execution of the containing function.
+  bool isGuaranteedLoopIndependent(const Instruction *Current,
+                                   const Instruction *KillingDef,
+                                   const MemoryLocation &CurrentLoc) {
+    // If the dependency is within the same block or loop level (being careful
+    // of irreducible loops), we know that AA will return a valid result for the
+    // memory dependency. (Both at the function level, outside of any loop,
+    // would also be valid but we currently disable that to limit compile time).
+    if (Current->getParent() == KillingDef->getParent())
+      return true;
+    const Loop *CurrentLI = LI.getLoopFor(Current->getParent());
+    if (!ContainsIrreducibleLoops && CurrentLI &&
+        CurrentLI == LI.getLoopFor(KillingDef->getParent()))
+      return true;
+    // Otherwise check the memory location is invariant to any loops.
+    return isGuaranteedLoopInvariant(CurrentLoc.Ptr);
+  }
+
   /// Returns true if \p Ptr is guaranteed to be loop invariant for any possible
   /// loop. In particular, this guarantees that it only references a single
   /// MemoryLocation during execution of the containing function.
-  bool IsGuaranteedLoopInvariant(Value *Ptr) {
-    auto IsGuaranteedLoopInvariantBase = [this](Value *Ptr) {
+  bool isGuaranteedLoopInvariant(const Value *Ptr) {
+    auto IsGuaranteedLoopInvariantBase = [this](const Value *Ptr) {
       Ptr = Ptr->stripPointerCasts();
       if (auto *I = dyn_cast<Instruction>(Ptr)) {
         if (isa<AllocaInst>(Ptr))
@@ -1912,6 +1318,10 @@ struct DSEState {
     };
 
     Ptr = Ptr->stripPointerCasts();
+    if (auto *I = dyn_cast<Instruction>(Ptr)) {
+      if (I->getParent()->isEntryBlock())
+        return true;
+    }
     if (auto *GEP = dyn_cast<GEPOperator>(Ptr)) {
       return IsGuaranteedLoopInvariantBase(GEP->getPointerOperand()) &&
              GEP->hasAllConstantIndices();
@@ -1937,13 +1347,11 @@ struct DSEState {
 
     MemoryAccess *Current = StartAccess;
     Instruction *KillingI = KillingDef->getMemoryInst();
-    bool StepAgain;
     LLVM_DEBUG(dbgs() << "  trying to get dominating access\n");
 
     // Find the next clobbering Mod access for DefLoc, starting at StartAccess.
     Optional<MemoryLocation> CurrentLoc;
-    do {
-      StepAgain = false;
+    for (;; Current = cast<MemoryDef>(Current)->getDefiningAccess()) {
       LLVM_DEBUG({
         dbgs() << "   visiting " << *Current;
         if (!MSSA.isLiveOnEntryDef(Current) && isa<MemoryUseOrDef>(Current))
@@ -1981,11 +1389,8 @@ struct DSEState {
       MemoryDef *CurrentDef = cast<MemoryDef>(Current);
       Instruction *CurrentI = CurrentDef->getMemoryInst();
 
-      if (canSkipDef(CurrentDef, !isInvisibleToCallerBeforeRet(DefUO))) {
-        StepAgain = true;
-        Current = CurrentDef->getDefiningAccess();
+      if (canSkipDef(CurrentDef, !isInvisibleToCallerBeforeRet(DefUO)))
         continue;
-      }
 
       // Before we try to remove anything, check for any extra throwing
       // instructions that block us from DSEing
@@ -2021,27 +1426,19 @@ struct DSEState {
 
       // If Current cannot be analyzed or is not removable, check the next
       // candidate.
-      if (!hasAnalyzableMemoryWrite(CurrentI, TLI) || !isRemovable(CurrentI)) {
-        StepAgain = true;
-        Current = CurrentDef->getDefiningAccess();
+      if (!hasAnalyzableMemoryWrite(CurrentI, TLI) || !isRemovable(CurrentI))
         continue;
-      }
 
       // If Current does not have an analyzable write location, skip it
       CurrentLoc = getLocForWriteEx(CurrentI);
-      if (!CurrentLoc) {
-        StepAgain = true;
-        Current = CurrentDef->getDefiningAccess();
+      if (!CurrentLoc)
         continue;
-      }
 
       // AliasAnalysis does not account for loops. Limit elimination to
       // candidates for which we can guarantee they always store to the same
-      // memory location and not multiple locations in a loop.
-      if (Current->getBlock() != KillingDef->getBlock() &&
-          !IsGuaranteedLoopInvariant(const_cast<Value *>(CurrentLoc->Ptr))) {
-        StepAgain = true;
-        Current = CurrentDef->getDefiningAccess();
+      // memory location and not located in different loops.
+      if (!isGuaranteedLoopIndependent(CurrentI, KillingI, *CurrentLoc)) {
+        LLVM_DEBUG(dbgs() << "  ... not guaranteed loop independent\n");
         WalkerStepLimit -= 1;
         continue;
       }
@@ -2050,35 +1447,30 @@ struct DSEState {
         // If the killing def is a memory terminator (e.g. lifetime.end), check
         // the next candidate if the current Current does not write the same
         // underlying object as the terminator.
-        if (!isMemTerminator(*CurrentLoc, CurrentI, KillingI)) {
-          StepAgain = true;
-          Current = CurrentDef->getDefiningAccess();
-        }
-        continue;
+        if (!isMemTerminator(*CurrentLoc, CurrentI, KillingI))
+          continue;
       } else {
         int64_t InstWriteOffset, DepWriteOffset;
-        auto OR = isOverwrite(KillingI, CurrentI, DefLoc, *CurrentLoc, DL, TLI,
-                              DepWriteOffset, InstWriteOffset, BatchAA, &F);
+        auto OR = isOverwrite(KillingI, CurrentI, DefLoc, *CurrentLoc,
+                              DepWriteOffset, InstWriteOffset);
         // If Current does not write to the same object as KillingDef, check
         // the next candidate.
-        if (OR == OW_Unknown) {
-          StepAgain = true;
-          Current = CurrentDef->getDefiningAccess();
-        } else if (OR == OW_MaybePartial) {
+        if (OR == OW_Unknown)
+          continue;
+        else if (OR == OW_MaybePartial) {
           // If KillingDef only partially overwrites Current, check the next
           // candidate if the partial step limit is exceeded. This aggressively
           // limits the number of candidates for partial store elimination,
           // which are less likely to be removable in the end.
           if (PartialLimit <= 1) {
-            StepAgain = true;
-            Current = CurrentDef->getDefiningAccess();
             WalkerStepLimit -= 1;
             continue;
           }
           PartialLimit -= 1;
         }
       }
-    } while (StepAgain);
+      break;
+    };
 
     // Accesses to objects accessible after the function returns can only be
     // eliminated if the access is killed along all paths to the exit. Collect
@@ -2168,8 +1560,16 @@ struct DSEState {
         return None;
       }
 
-      // For the KillingDef and EarlierAccess we only have to check if it reads
-      // the memory location.
+      // If this worklist walks back to the original memory access (and the
+      // pointer is not guarenteed loop invariant) then we cannot assume that a
+      // store kills itself.
+      if (EarlierAccess == UseAccess &&
+          !isGuaranteedLoopInvariant(CurrentLoc->Ptr)) {
+        LLVM_DEBUG(dbgs() << "    ... found not loop invariant self access\n");
+        return None;
+      }
+      // Otherwise, for the KillingDef and EarlierAccess we only have to check
+      // if it reads the memory location.
       // TODO: It would probably be better to check for self-reads before
       // calling the function.
       if (KillingDef == UseAccess || EarlierAccess == UseAccess) {
@@ -2189,16 +1589,18 @@ struct DSEState {
       //                  stores [0,1]
       if (MemoryDef *UseDef = dyn_cast<MemoryDef>(UseAccess)) {
         if (isCompleteOverwrite(*CurrentLoc, EarlierMemInst, UseInst)) {
-          if (!isInvisibleToCallerAfterRet(DefUO) &&
-              UseAccess != EarlierAccess) {
-            BasicBlock *MaybeKillingBlock = UseInst->getParent();
-            if (PostOrderNumbers.find(MaybeKillingBlock)->second <
-                PostOrderNumbers.find(EarlierAccess->getBlock())->second) {
-
+          BasicBlock *MaybeKillingBlock = UseInst->getParent();
+          if (PostOrderNumbers.find(MaybeKillingBlock)->second <
+              PostOrderNumbers.find(EarlierAccess->getBlock())->second) {
+            if (!isInvisibleToCallerAfterRet(DefUO)) {
               LLVM_DEBUG(dbgs()
                          << "    ... found killing def " << *UseInst << "\n");
               KillingDefs.insert(UseInst);
             }
+          } else {
+            LLVM_DEBUG(dbgs()
+                       << "    ... found preceeding def " << *UseInst << "\n");
+            return None;
           }
         } else
           PushMemUses(UseDef);
@@ -2405,6 +1807,25 @@ struct DSEState {
   bool storeIsNoop(MemoryDef *Def, const MemoryLocation &DefLoc,
                    const Value *DefUO) {
     StoreInst *Store = dyn_cast<StoreInst>(Def->getMemoryInst());
+    MemSetInst *MemSet = dyn_cast<MemSetInst>(Def->getMemoryInst());
+    Constant *StoredConstant = nullptr;
+    if (Store)
+      StoredConstant = dyn_cast<Constant>(Store->getOperand(0));
+    if (MemSet)
+      StoredConstant = dyn_cast<Constant>(MemSet->getValue());
+
+    if (StoredConstant && StoredConstant->isNullValue()) {
+      auto *DefUOInst = dyn_cast<Instruction>(DefUO);
+      if (DefUOInst && isCallocLikeFn(DefUOInst, &TLI)) {
+        auto *UnderlyingDef = cast<MemoryDef>(MSSA.getMemoryAccess(DefUOInst));
+        // If UnderlyingDef is the clobbering access of Def, no instructions
+        // between them can modify the memory location.
+        auto *ClobberDef =
+            MSSA.getSkipSelfWalker()->getClobberingMemoryAccess(Def);
+        return UnderlyingDef == ClobberDef;
+      }
+    }
+
     if (!Store)
       return false;
 
@@ -2452,29 +1873,17 @@ struct DSEState {
       }
     }
 
-    Constant *StoredConstant = dyn_cast<Constant>(Store->getOperand(0));
-    if (StoredConstant && StoredConstant->isNullValue()) {
-      auto *DefUOInst = dyn_cast<Instruction>(DefUO);
-      if (DefUOInst && isCallocLikeFn(DefUOInst, &TLI)) {
-        auto *UnderlyingDef = cast<MemoryDef>(MSSA.getMemoryAccess(DefUOInst));
-        // If UnderlyingDef is the clobbering access of Def, no instructions
-        // between them can modify the memory location.
-        auto *ClobberDef =
-            MSSA.getSkipSelfWalker()->getClobberingMemoryAccess(Def);
-        return UnderlyingDef == ClobberDef;
-      }
-    }
     return false;
   }
 };
 
-bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
-                                  MemorySSA &MSSA, DominatorTree &DT,
-                                  PostDominatorTree &PDT,
-                                  const TargetLibraryInfo &TLI) {
+static bool eliminateDeadStores(Function &F, AliasAnalysis &AA, MemorySSA &MSSA,
+                                DominatorTree &DT, PostDominatorTree &PDT,
+                                const TargetLibraryInfo &TLI,
+                                const LoopInfo &LI) {
   bool MadeChange = false;
 
-  DSEState State = DSEState::get(F, AA, MSSA, DT, PDT, TLI);
+  DSEState State = DSEState::get(F, AA, MSSA, DT, PDT, TLI, LI);
   // For each store:
   for (unsigned I = 0; I < State.MemDefs.size(); I++) {
     MemoryDef *KillingDef = State.MemDefs[I];
@@ -2500,7 +1909,7 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
 
     MemoryAccess *Current = KillingDef;
     LLVM_DEBUG(dbgs() << "Trying to eliminate MemoryDefs killed by "
-                      << *KillingDef << " (" << *SI << ")\n");
+                      << *Current << " (" << *SI << ")\n");
 
     unsigned ScanLimit = MemorySSAScanLimit;
     unsigned WalkerStepLimit = MemorySSAUpwardsStepLimit;
@@ -2569,9 +1978,8 @@ bool eliminateDeadStoresMemorySSA(Function &F, AliasAnalysis &AA,
       } else {
         // Check if NI overwrites SI.
         int64_t InstWriteOffset, DepWriteOffset;
-        OverwriteResult OR =
-            isOverwrite(SI, NI, SILoc, NILoc, State.DL, TLI, DepWriteOffset,
-                        InstWriteOffset, State.BatchAA, &F);
+        OverwriteResult OR = State.isOverwrite(SI, NI, SILoc, NILoc,
+                                               DepWriteOffset, InstWriteOffset);
         if (OR == OW_MaybePartial) {
           auto Iter = State.IOLs.insert(
               std::make_pair<BasicBlock *, InstOverlapIntervalsTy>(
@@ -2646,18 +2054,11 @@ PreservedAnalyses DSEPass::run(Function &F, FunctionAnalysisManager &AM) {
   AliasAnalysis &AA = AM.getResult<AAManager>(F);
   const TargetLibraryInfo &TLI = AM.getResult<TargetLibraryAnalysis>(F);
   DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(F);
+  MemorySSA &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
+  PostDominatorTree &PDT = AM.getResult<PostDominatorTreeAnalysis>(F);
+  LoopInfo &LI = AM.getResult<LoopAnalysis>(F);
 
-  bool Changed = false;
-  if (EnableMemorySSA) {
-    MemorySSA &MSSA = AM.getResult<MemorySSAAnalysis>(F).getMSSA();
-    PostDominatorTree &PDT = AM.getResult<PostDominatorTreeAnalysis>(F);
-
-    Changed = eliminateDeadStoresMemorySSA(F, AA, MSSA, DT, PDT, TLI);
-  } else {
-    MemoryDependenceResults &MD = AM.getResult<MemoryDependenceAnalysis>(F);
-
-    Changed = eliminateDeadStores(F, &AA, &MD, &DT, &TLI);
-  }
+  bool Changed = eliminateDeadStores(F, AA, MSSA, DT, PDT, TLI, LI);
 
 #ifdef LLVM_ENABLE_STATS
   if (AreStatisticsEnabled())
@@ -2670,11 +2071,8 @@ PreservedAnalyses DSEPass::run(Function &F, FunctionAnalysisManager &AM) {
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
-  PA.preserve<GlobalsAA>();
-  if (EnableMemorySSA)
-    PA.preserve<MemorySSAAnalysis>();
-  else
-    PA.preserve<MemoryDependenceAnalysis>();
+  PA.preserve<MemorySSAAnalysis>();
+  PA.preserve<LoopAnalysis>();
   return PA;
 }
 
@@ -2697,20 +2095,12 @@ public:
     DominatorTree &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
     const TargetLibraryInfo &TLI =
         getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
+    MemorySSA &MSSA = getAnalysis<MemorySSAWrapperPass>().getMSSA();
+    PostDominatorTree &PDT =
+        getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree();
+    LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
 
-    bool Changed = false;
-    if (EnableMemorySSA) {
-      MemorySSA &MSSA = getAnalysis<MemorySSAWrapperPass>().getMSSA();
-      PostDominatorTree &PDT =
-          getAnalysis<PostDominatorTreeWrapperPass>().getPostDomTree();
-
-      Changed = eliminateDeadStoresMemorySSA(F, AA, MSSA, DT, PDT, TLI);
-    } else {
-      MemoryDependenceResults &MD =
-          getAnalysis<MemoryDependenceWrapperPass>().getMemDep();
-
-      Changed = eliminateDeadStores(F, &AA, &MD, &DT, &TLI);
-    }
+    bool Changed = eliminateDeadStores(F, AA, MSSA, DT, PDT, TLI, LI);
 
 #ifdef LLVM_ENABLE_STATS
     if (AreStatisticsEnabled())
@@ -2728,16 +2118,12 @@ public:
     AU.addPreserved<GlobalsAAWrapperPass>();
     AU.addRequired<DominatorTreeWrapperPass>();
     AU.addPreserved<DominatorTreeWrapperPass>();
-
-    if (EnableMemorySSA) {
-      AU.addRequired<PostDominatorTreeWrapperPass>();
-      AU.addRequired<MemorySSAWrapperPass>();
-      AU.addPreserved<PostDominatorTreeWrapperPass>();
-      AU.addPreserved<MemorySSAWrapperPass>();
-    } else {
-      AU.addRequired<MemoryDependenceWrapperPass>();
-      AU.addPreserved<MemoryDependenceWrapperPass>();
-    }
+    AU.addRequired<PostDominatorTreeWrapperPass>();
+    AU.addRequired<MemorySSAWrapperPass>();
+    AU.addPreserved<PostDominatorTreeWrapperPass>();
+    AU.addPreserved<MemorySSAWrapperPass>();
+    AU.addRequired<LoopInfoWrapperPass>();
+    AU.addPreserved<LoopInfoWrapperPass>();
   }
 };
 
@@ -2754,6 +2140,7 @@ INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
 INITIALIZE_PASS_END(DSELegacyPass, "dse", "Dead Store Elimination", false,
                     false)
 
diff --git a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp
index 3c6c444d6649..c77769368ede 100644
--- a/llvm/lib/Transforms/Scalar/DivRemPairs.cpp
+++ b/llvm/lib/Transforms/Scalar/DivRemPairs.cpp
@@ -238,8 +238,52 @@ static bool optimizeDivRem(Function &F, const TargetTransformInfo &TTI,
     if (!DivDominates && !DT.dominates(RemInst, DivInst)) {
       // We have matching div-rem pair, but they are in two different blocks,
       // neither of which dominates one another.
-      // FIXME: We could hoist both ops to the common predecessor block?
-      continue;
+
+      BasicBlock *PredBB = nullptr;
+      BasicBlock *DivBB = DivInst->getParent();
+      BasicBlock *RemBB = RemInst->getParent();
+
+      // It's only safe to hoist if every instruction before the Div/Rem in the
+      // basic block is guaranteed to transfer execution.
+      auto IsSafeToHoist = [](Instruction *DivOrRem, BasicBlock *ParentBB) {
+        for (auto I = ParentBB->begin(), E = DivOrRem->getIterator(); I != E;
+             ++I)
+          if (!isGuaranteedToTransferExecutionToSuccessor(&*I))
+            return false;
+
+        return true;
+      };
+
+      // Look for something like this
+      // PredBB
+      //   |  \
+      //   |  Rem
+      //   |  /
+      //  Div
+      //
+      // If the Rem block has a single predecessor and successor, and all paths
+      // from PredBB go to either RemBB or DivBB, and execution of RemBB and
+      // DivBB will always reach the Div/Rem, we can hoist Div to PredBB. If
+      // we have a DivRem operation we can also hoist Rem. Otherwise we'll leave
+      // Rem where it is and rewrite it to mul/sub.
+      // FIXME: We could handle more hoisting cases.
+      if (RemBB->getSingleSuccessor() == DivBB)
+        PredBB = RemBB->getUniquePredecessor();
+
+      if (PredBB && IsSafeToHoist(RemInst, RemBB) &&
+          IsSafeToHoist(DivInst, DivBB) &&
+          llvm::all_of(successors(PredBB), [&](BasicBlock *BB) {
+            return BB == DivBB || BB == RemBB;
+          })) {
+        DivDominates = true;
+        DivInst->moveBefore(PredBB->getTerminator());
+        Changed = true;
+        if (HasDivRemOp) {
+          RemInst->moveBefore(PredBB->getTerminator());
+          continue;
+        }
+      } else
+        continue;
     }
 
     // The target does not have a single div/rem operation,
@@ -394,6 +438,5 @@ PreservedAnalyses DivRemPairsPass::run(Function &F,
   // TODO: This pass just hoists/replaces math ops - all analyses are preserved?
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
diff --git a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
index 180a82917fa9..978c6a77b8dc 100644
--- a/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/llvm/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -41,7 +41,6 @@
 #include "llvm/IR/LLVMContext.h"
 #include "llvm/IR/PassManager.h"
 #include "llvm/IR/PatternMatch.h"
-#include "llvm/IR/Statepoint.h"
 #include "llvm/IR/Type.h"
 #include "llvm/IR/Use.h"
 #include "llvm/IR/Value.h"
@@ -109,8 +108,29 @@ struct SimpleValue {
 
   static bool canHandle(Instruction *Inst) {
     // This can only handle non-void readnone functions.
-    if (CallInst *CI = dyn_cast<CallInst>(Inst))
+    // Also handled are constrained intrinsic that look like the types
+    // of instruction handled below (UnaryOperator, etc.).
+    if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
+      if (Function *F = CI->getCalledFunction()) {
+        switch ((Intrinsic::ID)F->getIntrinsicID()) {
+        case Intrinsic::experimental_constrained_fadd:
+        case Intrinsic::experimental_constrained_fsub:
+        case Intrinsic::experimental_constrained_fmul:
+        case Intrinsic::experimental_constrained_fdiv:
+        case Intrinsic::experimental_constrained_frem:
+        case Intrinsic::experimental_constrained_fptosi:
+        case Intrinsic::experimental_constrained_sitofp:
+        case Intrinsic::experimental_constrained_fptoui:
+        case Intrinsic::experimental_constrained_uitofp:
+        case Intrinsic::experimental_constrained_fcmp:
+        case Intrinsic::experimental_constrained_fcmps: {
+          auto *CFP = cast<ConstrainedFPIntrinsic>(CI);
+          return CFP->isDefaultFPEnvironment();
+        }
+        }
+      }
       return CI->doesNotAccessMemory() && !CI->getType()->isVoidTy();
+    }
     return isa<CastInst>(Inst) || isa<UnaryOperator>(Inst) ||
            isa<BinaryOperator>(Inst) || isa<GetElementPtrInst>(Inst) ||
            isa<CmpInst>(Inst) || isa<SelectInst>(Inst) ||
@@ -280,6 +300,13 @@ static unsigned getHashValueImpl(SimpleValue Val) {
     return hash_combine(II->getOpcode(), LHS, RHS);
   }
 
+  // gc.relocate is 'special' call: its second and third operands are
+  // not real values, but indices into statepoint's argument list.
+  // Get values they point to.
+  if (const GCRelocateInst *GCR = dyn_cast<GCRelocateInst>(Inst))
+    return hash_combine(GCR->getOpcode(), GCR->getOperand(0),
+                        GCR->getBasePtr(), GCR->getDerivedPtr());
+
   // Mix in the opcode.
   return hash_combine(
       Inst->getOpcode(),
@@ -341,6 +368,13 @@ static bool isEqualImpl(SimpleValue LHS, SimpleValue RHS) {
            LII->getArgOperand(1) == RII->getArgOperand(0);
   }
 
+  // See comment above in `getHashValue()`.
+  if (const GCRelocateInst *GCR1 = dyn_cast<GCRelocateInst>(LHSI))
+    if (const GCRelocateInst *GCR2 = dyn_cast<GCRelocateInst>(RHSI))
+      return GCR1->getOperand(0) == GCR2->getOperand(0) &&
+             GCR1->getBasePtr() == GCR2->getBasePtr() &&
+             GCR1->getDerivedPtr() == GCR2->getDerivedPtr();
+
   // Min/max can occur with commuted operands, non-canonical predicates,
   // and/or non-canonical operands.
   // Selects can be non-trivially equivalent via inverted conditions and swaps.
@@ -454,13 +488,6 @@ template <> struct DenseMapInfo<CallValue> {
 unsigned DenseMapInfo<CallValue>::getHashValue(CallValue Val) {
   Instruction *Inst = Val.Inst;
 
-  // gc.relocate is 'special' call: its second and third operands are
-  // not real values, but indices into statepoint's argument list.
-  // Get values they point to.
-  if (const GCRelocateInst *GCR = dyn_cast<GCRelocateInst>(Inst))
-    return hash_combine(GCR->getOpcode(), GCR->getOperand(0),
-                        GCR->getBasePtr(), GCR->getDerivedPtr());
-
   // Hash all of the operands as pointers and mix in the opcode.
   return hash_combine(
       Inst->getOpcode(),
@@ -472,13 +499,6 @@ bool DenseMapInfo<CallValue>::isEqual(CallValue LHS, CallValue RHS) {
   if (LHS.isSentinel() || RHS.isSentinel())
     return LHSI == RHSI;
 
-  // See comment above in `getHashValue()`.
-  if (const GCRelocateInst *GCR1 = dyn_cast<GCRelocateInst>(LHSI))
-    if (const GCRelocateInst *GCR2 = dyn_cast<GCRelocateInst>(RHSI))
-      return GCR1->getOperand(0) == GCR2->getOperand(0) &&
-             GCR1->getBasePtr() == GCR2->getBasePtr() &&
-             GCR1->getDerivedPtr() == GCR2->getDerivedPtr();
-
   return LHSI->isIdenticalTo(RHSI);
 }
 
@@ -1220,9 +1240,8 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
     // they're marked as such to ensure preservation of control dependencies),
     // and this pass will not bother with its removal. However, we should mark
     // its condition as true for all dominated blocks.
-    if (match(&Inst, m_Intrinsic<Intrinsic::assume>())) {
-      auto *CondI =
-          dyn_cast<Instruction>(cast<CallInst>(Inst).getArgOperand(0));
+    if (auto *Assume = dyn_cast<AssumeInst>(&Inst)) {
+      auto *CondI = dyn_cast<Instruction>(Assume->getArgOperand(0));
       if (CondI && SimpleValue::canHandle(CondI)) {
         LLVM_DEBUG(dbgs() << "EarlyCSE considering assumption: " << Inst
                           << '\n');
@@ -1618,7 +1637,6 @@ PreservedAnalyses EarlyCSEPass::run(Function &F,
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
-  PA.preserve<GlobalsAA>();
   if (UseMemorySSA)
     PA.preserve<MemorySSAAnalysis>();
   return PA;
diff --git a/llvm/lib/Transforms/Scalar/Float2Int.cpp b/llvm/lib/Transforms/Scalar/Float2Int.cpp
index b6d82685e884..8a5d4f568774 100644
--- a/llvm/lib/Transforms/Scalar/Float2Int.cpp
+++ b/llvm/lib/Transforms/Scalar/Float2Int.cpp
@@ -13,8 +13,6 @@
 
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/CommandLine.h"
-#define DEBUG_TYPE "float2int"
-
 #include "llvm/Transforms/Scalar/Float2Int.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/APSInt.h"
@@ -31,6 +29,9 @@
 #include "llvm/Transforms/Scalar.h"
 #include <deque>
 #include <functional> // For std::function
+
+#define DEBUG_TYPE "float2int"
+
 using namespace llvm;
 
 // The algorithm is simple. Start at instructions that convert from the
@@ -544,7 +545,6 @@ PreservedAnalyses Float2IntPass::run(Function &F, FunctionAnalysisManager &AM) {
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
 } // End namespace llvm
diff --git a/llvm/lib/Transforms/Scalar/GVN.cpp b/llvm/lib/Transforms/Scalar/GVN.cpp
index c6b6d75aefe8..16368aec7c3f 100644
--- a/llvm/lib/Transforms/Scalar/GVN.cpp
+++ b/llvm/lib/Transforms/Scalar/GVN.cpp
@@ -82,7 +82,6 @@
 #include <cassert>
 #include <cstdint>
 #include <utility>
-#include <vector>
 
 using namespace llvm;
 using namespace llvm::gvn;
@@ -91,13 +90,14 @@ using namespace PatternMatch;
 
 #define DEBUG_TYPE "gvn"
 
-STATISTIC(NumGVNInstr,  "Number of instructions deleted");
-STATISTIC(NumGVNLoad,   "Number of loads deleted");
-STATISTIC(NumGVNPRE,    "Number of instructions PRE'd");
+STATISTIC(NumGVNInstr, "Number of instructions deleted");
+STATISTIC(NumGVNLoad, "Number of loads deleted");
+STATISTIC(NumGVNPRE, "Number of instructions PRE'd");
 STATISTIC(NumGVNBlocks, "Number of blocks merged");
-STATISTIC(NumGVNSimpl,  "Number of instructions simplified");
+STATISTIC(NumGVNSimpl, "Number of instructions simplified");
 STATISTIC(NumGVNEqProp, "Number of equalities propagated");
-STATISTIC(NumPRELoad,   "Number of loads PRE'd");
+STATISTIC(NumPRELoad, "Number of loads PRE'd");
+STATISTIC(NumPRELoopLoad, "Number of loop loads PRE'd");
 
 STATISTIC(IsValueFullyAvailableInBlockNumSpeculationsMax,
           "Number of blocks speculated as available in "
@@ -207,9 +207,9 @@ struct llvm::gvn::AvailableValue {
     return Res;
   }
 
-  static AvailableValue getLoad(LoadInst *LI, unsigned Offset = 0) {
+  static AvailableValue getLoad(LoadInst *Load, unsigned Offset = 0) {
     AvailableValue Res;
-    Res.Val.setPointer(LI);
+    Res.Val.setPointer(Load);
     Res.Val.setInt(LoadVal);
     Res.Offset = Offset;
     return Res;
@@ -245,7 +245,7 @@ struct llvm::gvn::AvailableValue {
 
   /// Emit code at the specified insertion point to adjust the value defined
   /// here to the specified type. This handles various coercion cases.
-  Value *MaterializeAdjustedValue(LoadInst *LI, Instruction *InsertPt,
+  Value *MaterializeAdjustedValue(LoadInst *Load, Instruction *InsertPt,
                                   GVN &gvn) const;
 };
 
@@ -276,8 +276,8 @@ struct llvm::gvn::AvailableValueInBlock {
 
   /// Emit code at the end of this block to adjust the value defined here to
   /// the specified type. This handles various coercion cases.
-  Value *MaterializeAdjustedValue(LoadInst *LI, GVN &gvn) const {
-    return AV.MaterializeAdjustedValue(LI, BB->getTerminator(), gvn);
+  Value *MaterializeAdjustedValue(LoadInst *Load, GVN &gvn) const {
+    return AV.MaterializeAdjustedValue(Load, BB->getTerminator(), gvn);
   }
 };
 
@@ -289,9 +289,17 @@ GVN::Expression GVN::ValueTable::createExpr(Instruction *I) {
   Expression e;
   e.type = I->getType();
   e.opcode = I->getOpcode();
-  for (Instruction::op_iterator OI = I->op_begin(), OE = I->op_end();
-       OI != OE; ++OI)
-    e.varargs.push_back(lookupOrAdd(*OI));
+  if (const GCRelocateInst *GCR = dyn_cast<GCRelocateInst>(I)) {
+    // gc.relocate is 'special' call: its second and third operands are
+    // not real values, but indices into statepoint's argument list.
+    // Use the refered to values for purposes of identity.
+    e.varargs.push_back(lookupOrAdd(GCR->getOperand(0)));
+    e.varargs.push_back(lookupOrAdd(GCR->getBasePtr()));
+    e.varargs.push_back(lookupOrAdd(GCR->getDerivedPtr()));
+  } else {
+    for (Use &Op : I->operands())
+      e.varargs.push_back(lookupOrAdd(Op));
+  }
   if (I->isCommutative()) {
     // Ensure that commutative instructions that only differ by a permutation
     // of their operands get the same value number by sorting the operand value
@@ -362,9 +370,8 @@ GVN::Expression GVN::ValueTable::createExtractvalueExpr(ExtractValueInst *EI) {
   // Not a recognised intrinsic. Fall back to producing an extract value
   // expression.
   e.opcode = EI->getOpcode();
-  for (Instruction::op_iterator OI = EI->op_begin(), OE = EI->op_end();
-       OI != OE; ++OI)
-    e.varargs.push_back(lookupOrAdd(*OI));
+  for (Use &Op : EI->operands())
+    e.varargs.push_back(lookupOrAdd(Op));
 
   append_range(e.varargs, EI->indices());
 
@@ -669,7 +676,6 @@ PreservedAnalyses GVN::run(Function &F, FunctionAnalysisManager &AM) {
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
   PA.preserve<DominatorTreeAnalysis>();
-  PA.preserve<GlobalsAA>();
   PA.preserve<TargetLibraryAnalysis>();
   if (MSSA)
     PA.preserve<MemorySSAAnalysis>();
@@ -681,10 +687,9 @@ PreservedAnalyses GVN::run(Function &F, FunctionAnalysisManager &AM) {
 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 LLVM_DUMP_METHOD void GVN::dump(DenseMap<uint32_t, Value*>& d) const {
   errs() << "{\n";
-  for (DenseMap<uint32_t, Value*>::iterator I = d.begin(),
-       E = d.end(); I != E; ++I) {
-      errs() << I->first << "\n";
-      I->second->dump();
+  for (auto &I : d) {
+    errs() << I.first << "\n";
+    I.second->dump();
   }
   errs() << "}\n";
 }
@@ -727,7 +732,7 @@ static bool IsValueFullyAvailableInBlock(
 
   Worklist.emplace_back(BB);
   while (!Worklist.empty()) {
-    BasicBlock *CurrBB = Worklist.pop_back_val(); // LIFO - depth-first!
+    BasicBlock *CurrBB = Worklist.pop_back_val(); // LoadFO - depth-first!
     // Optimistically assume that the block is Speculatively Available and check
     // to see if we already know about this block in one lookup.
     std::pair<DenseMap<BasicBlock *, AvailabilityState>::iterator, bool> IV =
@@ -825,29 +830,33 @@ static bool IsValueFullyAvailableInBlock(
 }
 
 /// Given a set of loads specified by ValuesPerBlock,
-/// construct SSA form, allowing us to eliminate LI.  This returns the value
-/// that should be used at LI's definition site.
-static Value *ConstructSSAForLoadSet(LoadInst *LI,
-                         SmallVectorImpl<AvailableValueInBlock> &ValuesPerBlock,
-                                     GVN &gvn) {
+/// construct SSA form, allowing us to eliminate Load.  This returns the value
+/// that should be used at Load's definition site.
+static Value *
+ConstructSSAForLoadSet(LoadInst *Load,
+                       SmallVectorImpl<AvailableValueInBlock> &ValuesPerBlock,
+                       GVN &gvn) {
   // Check for the fully redundant, dominating load case.  In this case, we can
   // just use the dominating value directly.
   if (ValuesPerBlock.size() == 1 &&
       gvn.getDominatorTree().properlyDominates(ValuesPerBlock[0].BB,
-                                               LI->getParent())) {
+                                               Load->getParent())) {
     assert(!ValuesPerBlock[0].AV.isUndefValue() &&
            "Dead BB dominate this block");
-    return ValuesPerBlock[0].MaterializeAdjustedValue(LI, gvn);
+    return ValuesPerBlock[0].MaterializeAdjustedValue(Load, gvn);
   }
 
   // Otherwise, we have to construct SSA form.
   SmallVector<PHINode*, 8> NewPHIs;
   SSAUpdater SSAUpdate(&NewPHIs);
-  SSAUpdate.Initialize(LI->getType(), LI->getName());
+  SSAUpdate.Initialize(Load->getType(), Load->getName());
 
   for (const AvailableValueInBlock &AV : ValuesPerBlock) {
     BasicBlock *BB = AV.BB;
 
+    if (AV.AV.isUndefValue())
+      continue;
+
     if (SSAUpdate.HasValueForBlock(BB))
       continue;
 
@@ -855,24 +864,24 @@ static Value *ConstructSSAForLoadSet(LoadInst *LI,
     // available in is the block that the load is in, then don't add it as
     // SSAUpdater will resolve the value to the relevant phi which may let it
     // avoid phi construction entirely if there's actually only one value.
-    if (BB == LI->getParent() &&
-        ((AV.AV.isSimpleValue() && AV.AV.getSimpleValue() == LI) ||
-         (AV.AV.isCoercedLoadValue() && AV.AV.getCoercedLoadValue() == LI)))
+    if (BB == Load->getParent() &&
+        ((AV.AV.isSimpleValue() && AV.AV.getSimpleValue() == Load) ||
+         (AV.AV.isCoercedLoadValue() && AV.AV.getCoercedLoadValue() == Load)))
       continue;
 
-    SSAUpdate.AddAvailableValue(BB, AV.MaterializeAdjustedValue(LI, gvn));
+    SSAUpdate.AddAvailableValue(BB, AV.MaterializeAdjustedValue(Load, gvn));
   }
 
   // Perform PHI construction.
-  return SSAUpdate.GetValueInMiddleOfBlock(LI->getParent());
+  return SSAUpdate.GetValueInMiddleOfBlock(Load->getParent());
 }
 
-Value *AvailableValue::MaterializeAdjustedValue(LoadInst *LI,
+Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
                                                 Instruction *InsertPt,
                                                 GVN &gvn) const {
   Value *Res;
-  Type *LoadTy = LI->getType();
-  const DataLayout &DL = LI->getModule()->getDataLayout();
+  Type *LoadTy = Load->getType();
+  const DataLayout &DL = Load->getModule()->getDataLayout();
   if (isSimpleValue()) {
     Res = getSimpleValue();
     if (Res->getType() != LoadTy) {
@@ -884,17 +893,17 @@ Value *AvailableValue::MaterializeAdjustedValue(LoadInst *LI,
                         << "\n\n\n");
     }
   } else if (isCoercedLoadValue()) {
-    LoadInst *Load = getCoercedLoadValue();
-    if (Load->getType() == LoadTy && Offset == 0) {
-      Res = Load;
+    LoadInst *CoercedLoad = getCoercedLoadValue();
+    if (CoercedLoad->getType() == LoadTy && Offset == 0) {
+      Res = CoercedLoad;
     } else {
-      Res = getLoadValueForLoad(Load, Offset, LoadTy, InsertPt, DL);
+      Res = getLoadValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt, DL);
       // We would like to use gvn.markInstructionForDeletion here, but we can't
       // because the load is already memoized into the leader map table that GVN
       // tracks.  It is potentially possible to remove the load from the table,
       // but then there all of the operations based on it would need to be
       // rehashed.  Just leave the dead load around.
-      gvn.getMemDep().removeInstruction(Load);
+      gvn.getMemDep().removeInstruction(CoercedLoad);
       LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset
                         << "  " << *getCoercedLoadValue() << '\n'
                         << *Res << '\n'
@@ -908,9 +917,7 @@ Value *AvailableValue::MaterializeAdjustedValue(LoadInst *LI,
                       << *Res << '\n'
                       << "\n\n\n");
   } else {
-    assert(isUndefValue() && "Should be UndefVal");
-    LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL Undef:\n";);
-    return UndefValue::get(LoadTy);
+    llvm_unreachable("Should not materialize value from dead block");
   }
   assert(Res && "failed to materialize?");
   return Res;
@@ -922,30 +929,70 @@ static bool isLifetimeStart(const Instruction *Inst) {
   return false;
 }
 
+/// Assuming To can be reached from both From and Between, does Between lie on
+/// every path from From to To?
+static bool liesBetween(const Instruction *From, Instruction *Between,
+                        const Instruction *To, DominatorTree *DT) {
+  if (From->getParent() == Between->getParent())
+    return DT->dominates(From, Between);
+  SmallSet<BasicBlock *, 1> Exclusion;
+  Exclusion.insert(Between->getParent());
+  return !isPotentiallyReachable(From, To, &Exclusion, DT);
+}
+
 /// Try to locate the three instruction involved in a missed
 /// load-elimination case that is due to an intervening store.
-static void reportMayClobberedLoad(LoadInst *LI, MemDepResult DepInfo,
+static void reportMayClobberedLoad(LoadInst *Load, MemDepResult DepInfo,
                                    DominatorTree *DT,
                                    OptimizationRemarkEmitter *ORE) {
   using namespace ore;
 
   User *OtherAccess = nullptr;
 
-  OptimizationRemarkMissed R(DEBUG_TYPE, "LoadClobbered", LI);
-  R << "load of type " << NV("Type", LI->getType()) << " not eliminated"
+  OptimizationRemarkMissed R(DEBUG_TYPE, "LoadClobbered", Load);
+  R << "load of type " << NV("Type", Load->getType()) << " not eliminated"
     << setExtraArgs();
 
-  for (auto *U : LI->getPointerOperand()->users())
-    if (U != LI && (isa<LoadInst>(U) || isa<StoreInst>(U)) &&
-        DT->dominates(cast<Instruction>(U), LI)) {
-      // FIXME: for now give up if there are multiple memory accesses that
-      // dominate the load.  We need further analysis to decide which one is
-      // that we're forwarding from.
-      if (OtherAccess)
-        OtherAccess = nullptr;
-      else
+  for (auto *U : Load->getPointerOperand()->users()) {
+    if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U)) &&
+        cast<Instruction>(U)->getFunction() == Load->getFunction() &&
+        DT->dominates(cast<Instruction>(U), Load)) {
+      // Use the most immediately dominating value
+      if (OtherAccess) {
+        if (DT->dominates(cast<Instruction>(OtherAccess), cast<Instruction>(U)))
+          OtherAccess = U;
+        else
+          assert(DT->dominates(cast<Instruction>(U),
+                               cast<Instruction>(OtherAccess)));
+      } else
         OtherAccess = U;
     }
+  }
+
+  if (!OtherAccess) {
+    // There is no dominating use, check if we can find a closest non-dominating
+    // use that lies between any other potentially available use and Load.
+    for (auto *U : Load->getPointerOperand()->users()) {
+      if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U)) &&
+          cast<Instruction>(U)->getFunction() == Load->getFunction() &&
+          isPotentiallyReachable(cast<Instruction>(U), Load, nullptr, DT)) {
+        if (OtherAccess) {
+          if (liesBetween(cast<Instruction>(OtherAccess), cast<Instruction>(U),
+                          Load, DT)) {
+            OtherAccess = U;
+          } else if (!liesBetween(cast<Instruction>(U),
+                                  cast<Instruction>(OtherAccess), Load, DT)) {
+            // These uses are both partially available at Load were it not for
+            // the clobber, but neither lies strictly after the other.
+            OtherAccess = nullptr;
+            break;
+          } // else: keep current OtherAccess since it lies between U and Load
+        } else {
+          OtherAccess = U;
+        }
+      }
+    }
+  }
 
   if (OtherAccess)
     R << " in favor of " << NV("OtherAccess", OtherAccess);
@@ -955,13 +1002,13 @@ static void reportMayClobberedLoad(LoadInst *LI, MemDepResult DepInfo,
   ORE->emit(R);
 }
 
-bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
+bool GVN::AnalyzeLoadAvailability(LoadInst *Load, MemDepResult DepInfo,
                                   Value *Address, AvailableValue &Res) {
   assert((DepInfo.isDef() || DepInfo.isClobber()) &&
          "expected a local dependence");
-  assert(LI->isUnordered() && "rules below are incorrect for ordered access");
+  assert(Load->isUnordered() && "rules below are incorrect for ordered access");
 
-  const DataLayout &DL = LI->getModule()->getDataLayout();
+  const DataLayout &DL = Load->getModule()->getDataLayout();
 
   Instruction *DepInst = DepInfo.getInst();
   if (DepInfo.isClobber()) {
@@ -970,9 +1017,9 @@ bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
     // stored value.
     if (StoreInst *DepSI = dyn_cast<StoreInst>(DepInst)) {
       // Can't forward from non-atomic to atomic without violating memory model.
-      if (Address && LI->isAtomic() <= DepSI->isAtomic()) {
+      if (Address && Load->isAtomic() <= DepSI->isAtomic()) {
         int Offset =
-          analyzeLoadFromClobberingStore(LI->getType(), Address, DepSI, DL);
+            analyzeLoadFromClobberingStore(Load->getType(), Address, DepSI, DL);
         if (Offset != -1) {
           Res = AvailableValue::get(DepSI->getValueOperand(), Offset);
           return true;
@@ -984,16 +1031,29 @@ bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
     //    load i32* P
     //    load i8* (P+1)
     // if we have this, replace the later with an extraction from the former.
-    if (LoadInst *DepLI = dyn_cast<LoadInst>(DepInst)) {
+    if (LoadInst *DepLoad = dyn_cast<LoadInst>(DepInst)) {
       // If this is a clobber and L is the first instruction in its block, then
       // we have the first instruction in the entry block.
       // Can't forward from non-atomic to atomic without violating memory model.
-      if (DepLI != LI && Address && LI->isAtomic() <= DepLI->isAtomic()) {
-        int Offset =
-          analyzeLoadFromClobberingLoad(LI->getType(), Address, DepLI, DL);
+      if (DepLoad != Load && Address &&
+          Load->isAtomic() <= DepLoad->isAtomic()) {
+        Type *LoadType = Load->getType();
+        int Offset = -1;
 
+        // If MD reported clobber, check it was nested.
+        if (DepInfo.isClobber() &&
+            canCoerceMustAliasedValueToLoad(DepLoad, LoadType, DL)) {
+          const auto ClobberOff = MD->getClobberOffset(DepLoad);
+          // GVN has no deal with a negative offset.
+          Offset = (ClobberOff == None || ClobberOff.getValue() < 0)
+                       ? -1
+                       : ClobberOff.getValue();
+        }
+        if (Offset == -1)
+          Offset =
+              analyzeLoadFromClobberingLoad(LoadType, Address, DepLoad, DL);
         if (Offset != -1) {
-          Res = AvailableValue::getLoad(DepLI, Offset);
+          Res = AvailableValue::getLoad(DepLoad, Offset);
           return true;
         }
       }
@@ -1002,8 +1062,8 @@ bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
     // If the clobbering value is a memset/memcpy/memmove, see if we can
     // forward a value on from it.
     if (MemIntrinsic *DepMI = dyn_cast<MemIntrinsic>(DepInst)) {
-      if (Address && !LI->isAtomic()) {
-        int Offset = analyzeLoadFromClobberingMemInst(LI->getType(), Address,
+      if (Address && !Load->isAtomic()) {
+        int Offset = analyzeLoadFromClobberingMemInst(Load->getType(), Address,
                                                       DepMI, DL);
         if (Offset != -1) {
           Res = AvailableValue::getMI(DepMI, Offset);
@@ -1014,10 +1074,10 @@ bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
     // Nothing known about this clobber, have to be conservative
     LLVM_DEBUG(
         // fast print dep, using operator<< on instruction is too slow.
-        dbgs() << "GVN: load "; LI->printAsOperand(dbgs());
+        dbgs() << "GVN: load "; Load->printAsOperand(dbgs());
         dbgs() << " is clobbered by " << *DepInst << '\n';);
     if (ORE->allowExtraAnalysis(DEBUG_TYPE))
-      reportMayClobberedLoad(LI, DepInfo, DT, ORE);
+      reportMayClobberedLoad(Load, DepInfo, DT, ORE);
 
     return false;
   }
@@ -1028,13 +1088,13 @@ bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
       isAlignedAllocLikeFn(DepInst, TLI) ||
       // Loading immediately after lifetime begin -> undef.
       isLifetimeStart(DepInst)) {
-    Res = AvailableValue::get(UndefValue::get(LI->getType()));
+    Res = AvailableValue::get(UndefValue::get(Load->getType()));
     return true;
   }
 
   // Loading from calloc (which zero initializes memory) -> zero
   if (isCallocLikeFn(DepInst, TLI)) {
-    Res = AvailableValue::get(Constant::getNullValue(LI->getType()));
+    Res = AvailableValue::get(Constant::getNullValue(Load->getType()));
     return true;
   }
 
@@ -1042,12 +1102,12 @@ bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
     // Reject loads and stores that are to the same address but are of
     // different types if we have to. If the stored value is convertable to
     // the loaded value, we can reuse it.
-    if (!canCoerceMustAliasedValueToLoad(S->getValueOperand(), LI->getType(),
+    if (!canCoerceMustAliasedValueToLoad(S->getValueOperand(), Load->getType(),
                                          DL))
       return false;
 
     // Can't forward from non-atomic to atomic without violating memory model.
-    if (S->isAtomic() < LI->isAtomic())
+    if (S->isAtomic() < Load->isAtomic())
       return false;
 
     Res = AvailableValue::get(S->getValueOperand());
@@ -1058,11 +1118,11 @@ bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
     // If the types mismatch and we can't handle it, reject reuse of the load.
     // If the stored value is larger or equal to the loaded value, we can reuse
     // it.
-    if (!canCoerceMustAliasedValueToLoad(LD, LI->getType(), DL))
+    if (!canCoerceMustAliasedValueToLoad(LD, Load->getType(), DL))
       return false;
 
     // Can't forward from non-atomic to atomic without violating memory model.
-    if (LD->isAtomic() < LI->isAtomic())
+    if (LD->isAtomic() < Load->isAtomic())
       return false;
 
     Res = AvailableValue::getLoad(LD);
@@ -1072,12 +1132,12 @@ bool GVN::AnalyzeLoadAvailability(LoadInst *LI, MemDepResult DepInfo,
   // Unknown def - must be conservative
   LLVM_DEBUG(
       // fast print dep, using operator<< on instruction is too slow.
-      dbgs() << "GVN: load "; LI->printAsOperand(dbgs());
+      dbgs() << "GVN: load "; Load->printAsOperand(dbgs());
       dbgs() << " has unknown def " << *DepInst << '\n';);
   return false;
 }
 
-void GVN::AnalyzeLoadAvailability(LoadInst *LI, LoadDepVect &Deps,
+void GVN::AnalyzeLoadAvailability(LoadInst *Load, LoadDepVect &Deps,
                                   AvailValInBlkVect &ValuesPerBlock,
                                   UnavailBlkVect &UnavailableBlocks) {
   // Filter out useless results (non-locals, etc).  Keep track of the blocks
@@ -1107,7 +1167,7 @@ void GVN::AnalyzeLoadAvailability(LoadInst *LI, LoadDepVect &Deps,
     Value *Address = Deps[i].getAddress();
 
     AvailableValue AV;
-    if (AnalyzeLoadAvailability(LI, DepInfo, Address, AV)) {
+    if (AnalyzeLoadAvailability(Load, DepInfo, Address, AV)) {
       // subtlety: because we know this was a non-local dependency, we know
       // it's safe to materialize anywhere between the instruction within
       // DepInfo and the end of it's block.
@@ -1122,7 +1182,82 @@ void GVN::AnalyzeLoadAvailability(LoadInst *LI, LoadDepVect &Deps,
          "post condition violation");
 }
 
-bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
+void GVN::eliminatePartiallyRedundantLoad(
+    LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+    MapVector<BasicBlock *, Value *> &AvailableLoads) {
+  for (const auto &AvailableLoad : AvailableLoads) {
+    BasicBlock *UnavailableBlock = AvailableLoad.first;
+    Value *LoadPtr = AvailableLoad.second;
+
+    auto *NewLoad =
+        new LoadInst(Load->getType(), LoadPtr, Load->getName() + ".pre",
+                     Load->isVolatile(), Load->getAlign(), Load->getOrdering(),
+                     Load->getSyncScopeID(), UnavailableBlock->getTerminator());
+    NewLoad->setDebugLoc(Load->getDebugLoc());
+    if (MSSAU) {
+      auto *MSSA = MSSAU->getMemorySSA();
+      // Get the defining access of the original load or use the load if it is a
+      // MemoryDef (e.g. because it is volatile). The inserted loads are
+      // guaranteed to load from the same definition.
+      auto *LoadAcc = MSSA->getMemoryAccess(Load);
+      auto *DefiningAcc =
+          isa<MemoryDef>(LoadAcc) ? LoadAcc : LoadAcc->getDefiningAccess();
+      auto *NewAccess = MSSAU->createMemoryAccessInBB(
+          NewLoad, DefiningAcc, NewLoad->getParent(),
+          MemorySSA::BeforeTerminator);
+      if (auto *NewDef = dyn_cast<MemoryDef>(NewAccess))
+        MSSAU->insertDef(NewDef, /*RenameUses=*/true);
+      else
+        MSSAU->insertUse(cast<MemoryUse>(NewAccess), /*RenameUses=*/true);
+    }
+
+    // Transfer the old load's AA tags to the new load.
+    AAMDNodes Tags;
+    Load->getAAMetadata(Tags);
+    if (Tags)
+      NewLoad->setAAMetadata(Tags);
+
+    if (auto *MD = Load->getMetadata(LLVMContext::MD_invariant_load))
+      NewLoad->setMetadata(LLVMContext::MD_invariant_load, MD);
+    if (auto *InvGroupMD = Load->getMetadata(LLVMContext::MD_invariant_group))
+      NewLoad->setMetadata(LLVMContext::MD_invariant_group, InvGroupMD);
+    if (auto *RangeMD = Load->getMetadata(LLVMContext::MD_range))
+      NewLoad->setMetadata(LLVMContext::MD_range, RangeMD);
+    if (auto *AccessMD = Load->getMetadata(LLVMContext::MD_access_group))
+      if (LI &&
+          LI->getLoopFor(Load->getParent()) == LI->getLoopFor(UnavailableBlock))
+        NewLoad->setMetadata(LLVMContext::MD_access_group, AccessMD);
+
+    // We do not propagate the old load's debug location, because the new
+    // load now lives in a different BB, and we want to avoid a jumpy line
+    // table.
+    // FIXME: How do we retain source locations without causing poor debugging
+    // behavior?
+
+    // Add the newly created load.
+    ValuesPerBlock.push_back(
+        AvailableValueInBlock::get(UnavailableBlock, NewLoad));
+    MD->invalidateCachedPointerInfo(LoadPtr);
+    LLVM_DEBUG(dbgs() << "GVN INSERTED " << *NewLoad << '\n');
+  }
+
+  // Perform PHI construction.
+  Value *V = ConstructSSAForLoadSet(Load, ValuesPerBlock, *this);
+  Load->replaceAllUsesWith(V);
+  if (isa<PHINode>(V))
+    V->takeName(Load);
+  if (Instruction *I = dyn_cast<Instruction>(V))
+    I->setDebugLoc(Load->getDebugLoc());
+  if (V->getType()->isPtrOrPtrVectorTy())
+    MD->invalidateCachedPointerInfo(V);
+  markInstructionForDeletion(Load);
+  ORE->emit([&]() {
+    return OptimizationRemark(DEBUG_TYPE, "LoadPRE", Load)
+           << "load eliminated by PRE";
+  });
+}
+
+bool GVN::PerformLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
                          UnavailBlkVect &UnavailableBlocks) {
   // Okay, we have *some* definitions of the value.  This means that the value
   // is available in some of our (transitive) predecessors.  Lets think about
@@ -1137,7 +1272,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
 
   // Let's find the first basic block with more than one predecessor.  Walk
   // backwards through predecessors if needed.
-  BasicBlock *LoadBB = LI->getParent();
+  BasicBlock *LoadBB = Load->getParent();
   BasicBlock *TmpBB = LoadBB;
 
   // Check that there is no implicit control flow instructions above our load in
@@ -1156,7 +1291,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
   // access the array.
   // Check that there is no guard in this block above our instruction.
   bool MustEnsureSafetyOfSpeculativeExecution =
-      ICF->isDominatedByICFIFromSameBlock(LI);
+      ICF->isDominatedByICFIFromSameBlock(Load);
 
   while (TmpBB->getSinglePredecessor()) {
     TmpBB = TmpBB->getSinglePredecessor();
@@ -1197,7 +1332,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
     if (Pred->getTerminator()->isEHPad()) {
       LLVM_DEBUG(
           dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD PREDECESSOR '"
-                 << Pred->getName() << "': " << *LI << '\n');
+                 << Pred->getName() << "': " << *Load << '\n');
       return false;
     }
 
@@ -1209,7 +1344,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
       if (isa<IndirectBrInst>(Pred->getTerminator())) {
         LLVM_DEBUG(
             dbgs() << "COULD NOT PRE LOAD BECAUSE OF INDBR CRITICAL EDGE '"
-                   << Pred->getName() << "': " << *LI << '\n');
+                   << Pred->getName() << "': " << *Load << '\n');
         return false;
       }
 
@@ -1217,14 +1352,14 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
       if (isa<CallBrInst>(Pred->getTerminator())) {
         LLVM_DEBUG(
             dbgs() << "COULD NOT PRE LOAD BECAUSE OF CALLBR CRITICAL EDGE '"
-                   << Pred->getName() << "': " << *LI << '\n');
+                   << Pred->getName() << "': " << *Load << '\n');
         return false;
       }
 
       if (LoadBB->isEHPad()) {
         LLVM_DEBUG(
             dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '"
-                   << Pred->getName() << "': " << *LI << '\n');
+                   << Pred->getName() << "': " << *Load << '\n');
         return false;
       }
 
@@ -1234,7 +1369,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
           LLVM_DEBUG(
               dbgs()
               << "COULD NOT PRE LOAD BECAUSE OF A BACKEDGE CRITICAL EDGE '"
-              << Pred->getName() << "': " << *LI << '\n');
+              << Pred->getName() << "': " << *Load << '\n');
           return false;
         }
 
@@ -1252,7 +1387,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
 
   // If this load is unavailable in multiple predecessors, reject it.
   // FIXME: If we could restructure the CFG, we could make a common pred with
-  // all the preds that don't have an available LI and insert a new load into
+  // all the preds that don't have an available Load and insert a new load into
   // that one block.
   if (NumUnavailablePreds != 1)
       return false;
@@ -1261,10 +1396,10 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
   // to speculatively execute the load at that points.
   if (MustEnsureSafetyOfSpeculativeExecution) {
     if (CriticalEdgePred.size())
-      if (!isSafeToSpeculativelyExecute(LI, LoadBB->getFirstNonPHI(), DT))
+      if (!isSafeToSpeculativelyExecute(Load, LoadBB->getFirstNonPHI(), DT))
         return false;
     for (auto &PL : PredLoads)
-      if (!isSafeToSpeculativelyExecute(LI, PL.first->getTerminator(), DT))
+      if (!isSafeToSpeculativelyExecute(Load, PL.first->getTerminator(), DT))
         return false;
   }
 
@@ -1279,21 +1414,21 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
 
   // Check if the load can safely be moved to all the unavailable predecessors.
   bool CanDoPRE = true;
-  const DataLayout &DL = LI->getModule()->getDataLayout();
+  const DataLayout &DL = Load->getModule()->getDataLayout();
   SmallVector<Instruction*, 8> NewInsts;
   for (auto &PredLoad : PredLoads) {
     BasicBlock *UnavailablePred = PredLoad.first;
 
     // Do PHI translation to get its value in the predecessor if necessary.  The
     // returned pointer (if non-null) is guaranteed to dominate UnavailablePred.
-    // We do the translation for each edge we skipped by going from LI's block
+    // We do the translation for each edge we skipped by going from Load's block
     // to LoadBB, otherwise we might miss pieces needing translation.
 
     // If all preds have a single successor, then we know it is safe to insert
     // the load on the pred (?!?), so we can insert code to materialize the
     // pointer if it is not available.
-    Value *LoadPtr = LI->getPointerOperand();
-    BasicBlock *Cur = LI->getParent();
+    Value *LoadPtr = Load->getPointerOperand();
+    BasicBlock *Cur = Load->getParent();
     while (Cur != LoadBB) {
       PHITransAddr Address(LoadPtr, DL, AC);
       LoadPtr = Address.PHITranslateWithInsertion(
@@ -1314,7 +1449,7 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
     // we fail PRE.
     if (!LoadPtr) {
       LLVM_DEBUG(dbgs() << "COULDN'T INSERT PHI TRANSLATED VALUE OF: "
-                        << *LI->getPointerOperand() << "\n");
+                        << *Load->getPointerOperand() << "\n");
       CanDoPRE = false;
       break;
     }
@@ -1339,10 +1474,10 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
   // Okay, we can eliminate this load by inserting a reload in the predecessor
   // and using PHI construction to get the value in the other predecessors, do
   // it.
-  LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOAD: " << *LI << '\n');
-  LLVM_DEBUG(if (!NewInsts.empty()) dbgs()
-             << "INSERTED " << NewInsts.size() << " INSTS: " << *NewInsts.back()
-             << '\n');
+  LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOAD: " << *Load << '\n');
+  LLVM_DEBUG(if (!NewInsts.empty()) dbgs() << "INSERTED " << NewInsts.size()
+                                           << " INSTS: " << *NewInsts.back()
+                                           << '\n');
 
   // Assign value numbers to the new instructions.
   for (Instruction *I : NewInsts) {
@@ -1358,83 +1493,96 @@ bool GVN::PerformLoadPRE(LoadInst *LI, AvailValInBlkVect &ValuesPerBlock,
     VN.lookupOrAdd(I);
   }
 
-  for (const auto &PredLoad : PredLoads) {
-    BasicBlock *UnavailablePred = PredLoad.first;
-    Value *LoadPtr = PredLoad.second;
+  eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, PredLoads);
+  ++NumPRELoad;
+  return true;
+}
 
-    auto *NewLoad = new LoadInst(
-        LI->getType(), LoadPtr, LI->getName() + ".pre", LI->isVolatile(),
-        LI->getAlign(), LI->getOrdering(), LI->getSyncScopeID(),
-        UnavailablePred->getTerminator());
-    NewLoad->setDebugLoc(LI->getDebugLoc());
-    if (MSSAU) {
-      auto *MSSA = MSSAU->getMemorySSA();
-      // Get the defining access of the original load or use the load if it is a
-      // MemoryDef (e.g. because it is volatile). The inserted loads are
-      // guaranteed to load from the same definition.
-      auto *LIAcc = MSSA->getMemoryAccess(LI);
-      auto *DefiningAcc =
-          isa<MemoryDef>(LIAcc) ? LIAcc : LIAcc->getDefiningAccess();
-      auto *NewAccess = MSSAU->createMemoryAccessInBB(
-          NewLoad, DefiningAcc, NewLoad->getParent(),
-          MemorySSA::BeforeTerminator);
-      if (auto *NewDef = dyn_cast<MemoryDef>(NewAccess))
-        MSSAU->insertDef(NewDef, /*RenameUses=*/true);
-      else
-        MSSAU->insertUse(cast<MemoryUse>(NewAccess), /*RenameUses=*/true);
-    }
+bool GVN::performLoopLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
+                             UnavailBlkVect &UnavailableBlocks) {
+  if (!LI)
+    return false;
 
-    // Transfer the old load's AA tags to the new load.
-    AAMDNodes Tags;
-    LI->getAAMetadata(Tags);
-    if (Tags)
-      NewLoad->setAAMetadata(Tags);
+  const Loop *L = LI->getLoopFor(Load->getParent());
+  // TODO: Generalize to other loop blocks that dominate the latch.
+  if (!L || L->getHeader() != Load->getParent())
+    return false;
 
-    if (auto *MD = LI->getMetadata(LLVMContext::MD_invariant_load))
-      NewLoad->setMetadata(LLVMContext::MD_invariant_load, MD);
-    if (auto *InvGroupMD = LI->getMetadata(LLVMContext::MD_invariant_group))
-      NewLoad->setMetadata(LLVMContext::MD_invariant_group, InvGroupMD);
-    if (auto *RangeMD = LI->getMetadata(LLVMContext::MD_range))
-      NewLoad->setMetadata(LLVMContext::MD_range, RangeMD);
+  BasicBlock *Preheader = L->getLoopPreheader();
+  BasicBlock *Latch = L->getLoopLatch();
+  if (!Preheader || !Latch)
+    return false;
 
-    // We do not propagate the old load's debug location, because the new
-    // load now lives in a different BB, and we want to avoid a jumpy line
-    // table.
-    // FIXME: How do we retain source locations without causing poor debugging
-    // behavior?
+  Value *LoadPtr = Load->getPointerOperand();
+  // Must be available in preheader.
+  if (!L->isLoopInvariant(LoadPtr))
+    return false;
 
-    // Add the newly created load.
-    ValuesPerBlock.push_back(AvailableValueInBlock::get(UnavailablePred,
-                                                        NewLoad));
-    MD->invalidateCachedPointerInfo(LoadPtr);
-    LLVM_DEBUG(dbgs() << "GVN INSERTED " << *NewLoad << '\n');
+  // We plan to hoist the load to preheader without introducing a new fault.
+  // In order to do it, we need to prove that we cannot side-exit the loop
+  // once loop header is first entered before execution of the load.
+  if (ICF->isDominatedByICFIFromSameBlock(Load))
+    return false;
+
+  BasicBlock *LoopBlock = nullptr;
+  for (auto *Blocker : UnavailableBlocks) {
+    // Blockers from outside the loop are handled in preheader.
+    if (!L->contains(Blocker))
+      continue;
+
+    // Only allow one loop block. Loop header is not less frequently executed
+    // than each loop block, and likely it is much more frequently executed. But
+    // in case of multiple loop blocks, we need extra information (such as block
+    // frequency info) to understand whether it is profitable to PRE into
+    // multiple loop blocks.
+    if (LoopBlock)
+      return false;
+
+    // Do not sink into inner loops. This may be non-profitable.
+    if (L != LI->getLoopFor(Blocker))
+      return false;
+
+    // Blocks that dominate the latch execute on every single iteration, maybe
+    // except the last one. So PREing into these blocks doesn't make much sense
+    // in most cases. But the blocks that do not necessarily execute on each
+    // iteration are sometimes much colder than the header, and this is when
+    // PRE is potentially profitable.
+    if (DT->dominates(Blocker, Latch))
+      return false;
+
+    // Make sure that the terminator itself doesn't clobber.
+    if (Blocker->getTerminator()->mayWriteToMemory())
+      return false;
+
+    LoopBlock = Blocker;
   }
 
-  // Perform PHI construction.
-  Value *V = ConstructSSAForLoadSet(LI, ValuesPerBlock, *this);
-  LI->replaceAllUsesWith(V);
-  if (isa<PHINode>(V))
-    V->takeName(LI);
-  if (Instruction *I = dyn_cast<Instruction>(V))
-    I->setDebugLoc(LI->getDebugLoc());
-  if (V->getType()->isPtrOrPtrVectorTy())
-    MD->invalidateCachedPointerInfo(V);
-  markInstructionForDeletion(LI);
-  ORE->emit([&]() {
-    return OptimizationRemark(DEBUG_TYPE, "LoadPRE", LI)
-           << "load eliminated by PRE";
-  });
-  ++NumPRELoad;
+  if (!LoopBlock)
+    return false;
+
+  // Make sure the memory at this pointer cannot be freed, therefore we can
+  // safely reload from it after clobber.
+  if (LoadPtr->canBeFreed())
+    return false;
+
+  // TODO: Support critical edge splitting if blocker has more than 1 successor.
+  MapVector<BasicBlock *, Value *> AvailableLoads;
+  AvailableLoads[LoopBlock] = LoadPtr;
+  AvailableLoads[Preheader] = LoadPtr;
+
+  LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOOP LOAD: " << *Load << '\n');
+  eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, AvailableLoads);
+  ++NumPRELoopLoad;
   return true;
 }
 
-static void reportLoadElim(LoadInst *LI, Value *AvailableValue,
+static void reportLoadElim(LoadInst *Load, Value *AvailableValue,
                            OptimizationRemarkEmitter *ORE) {
   using namespace ore;
 
   ORE->emit([&]() {
-    return OptimizationRemark(DEBUG_TYPE, "LoadElim", LI)
-           << "load of type " << NV("Type", LI->getType()) << " eliminated"
+    return OptimizationRemark(DEBUG_TYPE, "LoadElim", Load)
+           << "load of type " << NV("Type", Load->getType()) << " eliminated"
            << setExtraArgs() << " in favor of "
            << NV("InfavorOfValue", AvailableValue);
   });
@@ -1442,17 +1590,17 @@ static void reportLoadElim(LoadInst *LI, Value *AvailableValue,
 
 /// Attempt to eliminate a load whose dependencies are
 /// non-local by performing PHI construction.
-bool GVN::processNonLocalLoad(LoadInst *LI) {
+bool GVN::processNonLocalLoad(LoadInst *Load) {
   // non-local speculations are not allowed under asan.
-  if (LI->getParent()->getParent()->hasFnAttribute(
+  if (Load->getParent()->getParent()->hasFnAttribute(
           Attribute::SanitizeAddress) ||
-      LI->getParent()->getParent()->hasFnAttribute(
+      Load->getParent()->getParent()->hasFnAttribute(
           Attribute::SanitizeHWAddress))
     return false;
 
   // Step 1: Find the non-local dependencies of the load.
   LoadDepVect Deps;
-  MD->getNonLocalPointerDependency(LI, Deps);
+  MD->getNonLocalPointerDependency(Load, Deps);
 
   // If we had to process more than one hundred blocks to find the
   // dependencies, this load isn't worth worrying about.  Optimizing
@@ -1465,14 +1613,15 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
   // clobber in the current block.  Reject this early.
   if (NumDeps == 1 &&
       !Deps[0].getResult().isDef() && !Deps[0].getResult().isClobber()) {
-    LLVM_DEBUG(dbgs() << "GVN: non-local load "; LI->printAsOperand(dbgs());
+    LLVM_DEBUG(dbgs() << "GVN: non-local load "; Load->printAsOperand(dbgs());
                dbgs() << " has unknown dependencies\n";);
     return false;
   }
 
   bool Changed = false;
   // If this load follows a GEP, see if we can PRE the indices before analyzing.
-  if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(LI->getOperand(0))) {
+  if (GetElementPtrInst *GEP =
+          dyn_cast<GetElementPtrInst>(Load->getOperand(0))) {
     for (GetElementPtrInst::op_iterator OI = GEP->idx_begin(),
                                         OE = GEP->idx_end();
          OI != OE; ++OI)
@@ -1483,7 +1632,7 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
   // Step 2: Analyze the availability of the load
   AvailValInBlkVect ValuesPerBlock;
   UnavailBlkVect UnavailableBlocks;
-  AnalyzeLoadAvailability(LI, Deps, ValuesPerBlock, UnavailableBlocks);
+  AnalyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
 
   // If we have no predecessors that produce a known value for this load, exit
   // early.
@@ -1496,36 +1645,36 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
   // load, then it is fully redundant and we can use PHI insertion to compute
   // its value.  Insert PHIs and remove the fully redundant value now.
   if (UnavailableBlocks.empty()) {
-    LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *LI << '\n');
+    LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n');
 
     // Perform PHI construction.
-    Value *V = ConstructSSAForLoadSet(LI, ValuesPerBlock, *this);
-    LI->replaceAllUsesWith(V);
+    Value *V = ConstructSSAForLoadSet(Load, ValuesPerBlock, *this);
+    Load->replaceAllUsesWith(V);
 
     if (isa<PHINode>(V))
-      V->takeName(LI);
+      V->takeName(Load);
     if (Instruction *I = dyn_cast<Instruction>(V))
       // If instruction I has debug info, then we should not update it.
       // Also, if I has a null DebugLoc, then it is still potentially incorrect
-      // to propagate LI's DebugLoc because LI may not post-dominate I.
-      if (LI->getDebugLoc() && LI->getParent() == I->getParent())
-        I->setDebugLoc(LI->getDebugLoc());
+      // to propagate Load's DebugLoc because Load may not post-dominate I.
+      if (Load->getDebugLoc() && Load->getParent() == I->getParent())
+        I->setDebugLoc(Load->getDebugLoc());
     if (V->getType()->isPtrOrPtrVectorTy())
       MD->invalidateCachedPointerInfo(V);
-    markInstructionForDeletion(LI);
+    markInstructionForDeletion(Load);
     ++NumGVNLoad;
-    reportLoadElim(LI, V, ORE);
+    reportLoadElim(Load, V, ORE);
     return true;
   }
 
   // Step 4: Eliminate partial redundancy.
   if (!isPREEnabled() || !isLoadPREEnabled())
     return Changed;
-  if (!isLoadInLoopPREEnabled() && this->LI &&
-      this->LI->getLoopFor(LI->getParent()))
+  if (!isLoadInLoopPREEnabled() && LI && LI->getLoopFor(Load->getParent()))
     return Changed;
 
-  return Changed || PerformLoadPRE(LI, ValuesPerBlock, UnavailableBlocks);
+  return Changed || PerformLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) ||
+         performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks);
 }
 
 static bool impliesEquivalanceIfTrue(CmpInst* Cmp) {
@@ -1589,9 +1738,7 @@ static bool hasUsersIn(Value *V, BasicBlock *BB) {
   return false;
 }
 
-bool GVN::processAssumeIntrinsic(IntrinsicInst *IntrinsicI) {
-  assert(IntrinsicI->getIntrinsicID() == Intrinsic::assume &&
-         "This function can only be called with llvm.assume intrinsic");
+bool GVN::processAssumeIntrinsic(AssumeInst *IntrinsicI) {
   Value *V = IntrinsicI->getArgOperand(0);
 
   if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) {
@@ -1776,7 +1923,7 @@ bool GVN::processLoad(LoadInst *L) {
       MSSAU->removeMemoryAccess(L);
     ++NumGVNLoad;
     reportLoadElim(L, AvailableValue, ORE);
-    // Tell MDA to rexamine the reused pointer since we might have more
+    // Tell MDA to reexamine the reused pointer since we might have more
     // information after forwarding it.
     if (MD && AvailableValue->getType()->isPtrOrPtrVectorTy())
       MD->invalidateCachedPointerInfo(AvailableValue);
@@ -1852,8 +1999,8 @@ bool GVN::ValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
       MD->getNonLocalCallDependency(Call);
 
   // Check to see if the Call has no function local clobber.
-  for (unsigned i = 0; i < deps.size(); i++) {
-    if (deps[i].getResult().isNonFuncLocal())
+  for (const NonLocalDepEntry &D : deps) {
+    if (D.getResult().isNonFuncLocal())
       return true;
   }
   return false;
@@ -2157,6 +2304,9 @@ bool GVN::processInstruction(Instruction *I) {
   if (Value *V = SimplifyInstruction(I, {DL, TLI, DT, AC})) {
     bool Changed = false;
     if (!I->use_empty()) {
+      // Simplification can cause a special instruction to become not special.
+      // For example, devirtualization to a willreturn function.
+      ICF->removeUsersOf(I);
       I->replaceAllUsesWith(V);
       Changed = true;
     }
@@ -2172,16 +2322,15 @@ bool GVN::processInstruction(Instruction *I) {
     }
   }
 
-  if (IntrinsicInst *IntrinsicI = dyn_cast<IntrinsicInst>(I))
-    if (IntrinsicI->getIntrinsicID() == Intrinsic::assume)
-      return processAssumeIntrinsic(IntrinsicI);
+  if (auto *Assume = dyn_cast<AssumeInst>(I))
+    return processAssumeIntrinsic(Assume);
 
-  if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
-    if (processLoad(LI))
+  if (LoadInst *Load = dyn_cast<LoadInst>(I)) {
+    if (processLoad(Load))
       return true;
 
-    unsigned Num = VN.lookupOrAdd(LI);
-    addToLeaderTable(Num, LI, LI->getParent());
+    unsigned Num = VN.lookupOrAdd(Load);
+    addToLeaderTable(Num, Load, Load->getParent());
     return false;
   }
 
@@ -2365,6 +2514,12 @@ bool GVN::processBlock(BasicBlock *BB) {
   ReplaceOperandsWithMap.clear();
   bool ChangedFunction = false;
 
+  // Since we may not have visited the input blocks of the phis, we can't
+  // use our normal hash approach for phis.  Instead, simply look for
+  // obvious duplicates.  The first pass of GVN will tend to create
+  // identical phis, and the second or later passes can eliminate them.
+  ChangedFunction |= EliminateDuplicatePHINodes(BB);
+
   for (BasicBlock::iterator BI = BB->begin(), BE = BB->end();
        BI != BE;) {
     if (!ReplaceOperandsWithMap.empty())
@@ -2447,6 +2602,8 @@ bool GVN::performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
   Instr->setName(Instr->getName() + ".pre");
   Instr->setDebugLoc(Instr->getDebugLoc());
 
+  ICF->insertInstructionTo(Instr, Pred);
+
   unsigned Num = VN.lookupOrAdd(Instr);
   VN.add(Instr, Num);
 
@@ -2735,9 +2892,8 @@ void GVN::verifyRemoved(const Instruction *Inst) const {
 
   // Walk through the value number scope to make sure the instruction isn't
   // ferreted away in it.
-  for (DenseMap<uint32_t, LeaderTableEntry>::const_iterator
-       I = LeaderTable.begin(), E = LeaderTable.end(); I != E; ++I) {
-    const LeaderTableEntry *Node = &I->second;
+  for (const auto &I : LeaderTable) {
+    const LeaderTableEntry *Node = &I.second;
     assert(Node->Val != Inst && "Inst still in value numbering scope!");
 
     while (Node->Next) {
@@ -2795,9 +2951,7 @@ void GVN::addDeadBlock(BasicBlock *BB) {
 
   // For the dead blocks' live successors, update their phi nodes by replacing
   // the operands corresponding to dead blocks with UndefVal.
-  for(SmallSetVector<BasicBlock *, 4>::iterator I = DF.begin(), E = DF.end();
-        I != E; I++) {
-    BasicBlock *B = *I;
+  for (BasicBlock *B : DF) {
     if (DeadBlocks.count(B))
       continue;
 
diff --git a/llvm/lib/Transforms/Scalar/GVNHoist.cpp b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
index 8d0bd5674964..790d71992da4 100644
--- a/llvm/lib/Transforms/Scalar/GVNHoist.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNHoist.cpp
@@ -1243,7 +1243,6 @@ PreservedAnalyses GVNHoistPass::run(Function &F, FunctionAnalysisManager &AM) {
   PreservedAnalyses PA;
   PA.preserve<DominatorTreeAnalysis>();
   PA.preserve<MemorySSAAnalysis>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/GVNSink.cpp b/llvm/lib/Transforms/Scalar/GVNSink.cpp
index aef927ab6558..e612a82fc89a 100644
--- a/llvm/lib/Transforms/Scalar/GVNSink.cpp
+++ b/llvm/lib/Transforms/Scalar/GVNSink.cpp
@@ -912,10 +912,7 @@ PreservedAnalyses GVNSinkPass::run(Function &F, FunctionAnalysisManager &AM) {
   GVNSink G;
   if (!G.run(F))
     return PreservedAnalyses::all();
-
-  PreservedAnalyses PA;
-  PA.preserve<GlobalsAA>();
-  return PA;
+  return PreservedAnalyses::none();
 }
 
 char GVNSinkLegacyPass::ID = 0;
diff --git a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
index ae1fff0fa844..9ee2a2d0bf08 100644
--- a/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/llvm/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -875,7 +875,8 @@ static bool isLoopCounter(PHINode* Phi, Loop *L,
 
   int LatchIdx = Phi->getBasicBlockIndex(L->getLoopLatch());
   Value *IncV = Phi->getIncomingValue(LatchIdx);
-  return (getLoopPhiForCounter(IncV, L) == Phi);
+  return (getLoopPhiForCounter(IncV, L) == Phi &&
+          isa<SCEVAddRecExpr>(SE->getSCEV(IncV)));
 }
 
 /// Search the loop header for a loop counter (anadd rec w/step of one)
@@ -1564,9 +1565,6 @@ bool IndVarSimplify::predicateLoopExits(Loop *L, SCEVExpander &Rewriter) {
   if (!LoopPredication)
     return false;
 
-  if (!SE->hasLoopInvariantBackedgeTakenCount(L))
-    return false;
-
   // Note: ExactBTC is the exact backedge taken count *iff* the loop exits
   // through *explicit* control flow.  We have to eliminate the possibility of
   // implicit exits (see below) before we know it's truly exact.
@@ -1605,8 +1603,8 @@ bool IndVarSimplify::predicateLoopExits(Loop *L, SCEVExpander &Rewriter) {
       return true;
 
     const SCEV *ExitCount = SE->getExitCount(L, ExitingBB);
-    assert(!isa<SCEVCouldNotCompute>(ExactBTC) && "implied by having exact trip count");
-    if (!SE->isLoopInvariant(ExitCount, L) ||
+    if (isa<SCEVCouldNotCompute>(ExitCount) ||
+        !SE->isLoopInvariant(ExitCount, L) ||
         !isSafeToExpand(ExitCount, *SE))
       return true;
 
@@ -1670,7 +1668,7 @@ bool IndVarSimplify::predicateLoopExits(Loop *L, SCEVExpander &Rewriter) {
   for (BasicBlock *BB : L->blocks())
     for (auto &I : *BB)
       // TODO:isGuaranteedToTransfer
-      if (I.mayHaveSideEffects() || I.mayThrow())
+      if (I.mayHaveSideEffects())
         return false;
 
   bool Changed = false;
diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
index 6e09dec198c2..0e5653eeb7d5 100644
--- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp
@@ -360,7 +360,7 @@ void InductiveRangeCheck::extractRangeChecksFromCond(
     return;
 
   // TODO: Do the same for OR, XOR, NOT etc?
-  if (match(Condition, m_And(m_Value(), m_Value()))) {
+  if (match(Condition, m_LogicalAnd(m_Value(), m_Value()))) {
     extractRangeChecksFromCond(L, SE, cast<User>(Condition)->getOperandUse(0),
                                Checks, Visited);
     extractRangeChecksFromCond(L, SE, cast<User>(Condition)->getOperandUse(1),
@@ -1785,8 +1785,11 @@ PreservedAnalyses IRCEPass::run(Function &F, FunctionAnalysisManager &AM) {
     }
     Changed |= CFGChanged;
 
-    if (CFGChanged && !SkipProfitabilityChecks)
-      AM.invalidate<BlockFrequencyAnalysis>(F);
+    if (CFGChanged && !SkipProfitabilityChecks) {
+      PreservedAnalyses PA = PreservedAnalyses::all();
+      PA.abandon<BlockFrequencyAnalysis>();
+      AM.invalidate(F, PA);
+    }
   }
 
   SmallPriorityWorklist<Loop *, 4> Worklist;
@@ -1800,8 +1803,11 @@ PreservedAnalyses IRCEPass::run(Function &F, FunctionAnalysisManager &AM) {
     Loop *L = Worklist.pop_back_val();
     if (IRCE.run(L, LPMAddNewLoop)) {
       Changed = true;
-      if (!SkipProfitabilityChecks)
-        AM.invalidate<BlockFrequencyAnalysis>(F);
+      if (!SkipProfitabilityChecks) {
+        PreservedAnalyses PA = PreservedAnalyses::all();
+        PA.abandon<BlockFrequencyAnalysis>();
+        AM.invalidate(F, PA);
+      }
     }
   }
 
diff --git a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
index 332eb10ac16b..f7d631f5e785 100644
--- a/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
+++ b/llvm/lib/Transforms/Scalar/InferAddressSpaces.cpp
@@ -471,7 +471,7 @@ InferAddressSpacesImpl::collectFlatAddressExpressions(Function &F) const {
     } else if (auto *I2P = dyn_cast<IntToPtrInst>(&I)) {
       if (isNoopPtrIntCastPair(cast<Operator>(I2P), *DL, TTI))
         PushPtrOperand(
-            cast<PtrToIntInst>(I2P->getOperand(0))->getPointerOperand());
+            cast<Operator>(I2P->getOperand(0))->getOperand(0));
     }
   }
 
@@ -508,8 +508,8 @@ static Value *operandWithNewAddressSpaceOrCreateUndef(
     SmallVectorImpl<const Use *> *UndefUsesToFix) {
   Value *Operand = OperandUse.get();
 
-  Type *NewPtrTy =
-      Operand->getType()->getPointerElementType()->getPointerTo(NewAddrSpace);
+  Type *NewPtrTy = PointerType::getWithSamePointeeType(
+      cast<PointerType>(Operand->getType()), NewAddrSpace);
 
   if (Constant *C = dyn_cast<Constant>(Operand))
     return ConstantExpr::getAddrSpaceCast(C, NewPtrTy);
@@ -537,8 +537,8 @@ Value *InferAddressSpacesImpl::cloneInstructionWithNewAddressSpace(
     Instruction *I, unsigned NewAddrSpace,
     const ValueToValueMapTy &ValueWithNewAddrSpace,
     SmallVectorImpl<const Use *> *UndefUsesToFix) const {
-  Type *NewPtrType =
-      I->getType()->getPointerElementType()->getPointerTo(NewAddrSpace);
+  Type *NewPtrType = PointerType::getWithSamePointeeType(
+      cast<PointerType>(I->getType()), NewAddrSpace);
 
   if (I->getOpcode() == Instruction::AddrSpaceCast) {
     Value *Src = I->getOperand(0);
@@ -572,7 +572,8 @@ Value *InferAddressSpacesImpl::cloneInstructionWithNewAddressSpace(
   if (AS != UninitializedAddressSpace) {
     // For the assumed address space, insert an `addrspacecast` to make that
     // explicit.
-    auto *NewPtrTy = I->getType()->getPointerElementType()->getPointerTo(AS);
+    Type *NewPtrTy = PointerType::getWithSamePointeeType(
+        cast<PointerType>(I->getType()), AS);
     auto *NewI = new AddrSpaceCastInst(I, NewPtrTy);
     NewI->insertAfter(I);
     return NewI;
@@ -634,8 +635,10 @@ static Value *cloneConstantExprWithNewAddressSpace(
     ConstantExpr *CE, unsigned NewAddrSpace,
     const ValueToValueMapTy &ValueWithNewAddrSpace, const DataLayout *DL,
     const TargetTransformInfo *TTI) {
-  Type *TargetType =
-    CE->getType()->getPointerElementType()->getPointerTo(NewAddrSpace);
+  Type *TargetType = CE->getType()->isPointerTy()
+                         ? PointerType::getWithSamePointeeType(
+                               cast<PointerType>(CE->getType()), NewAddrSpace)
+                         : CE->getType();
 
   if (CE->getOpcode() == Instruction::AddrSpaceCast) {
     // Because CE is flat, the source address space must be specific.
@@ -953,7 +956,13 @@ static bool handleMemIntrinsicPtrUse(MemIntrinsic *MI, Value *OldV,
     if (Dest == OldV)
       Dest = NewV;
 
-    if (isa<MemCpyInst>(MTI)) {
+    if (isa<MemCpyInlineInst>(MTI)) {
+      MDNode *TBAAStruct = MTI->getMetadata(LLVMContext::MD_tbaa_struct);
+      B.CreateMemCpyInline(Dest, MTI->getDestAlign(), Src,
+                           MTI->getSourceAlign(), MTI->getLength(),
+                           false, // isVolatile
+                           TBAA, TBAAStruct, ScopeMD, NoAliasMD);
+    } else if (isa<MemCpyInst>(MTI)) {
       MDNode *TBAAStruct = MTI->getMetadata(LLVMContext::MD_tbaa_struct);
       B.CreateMemCpy(Dest, MTI->getDestAlign(), Src, MTI->getSourceAlign(),
                      MTI->getLength(),
diff --git a/llvm/lib/Transforms/Scalar/JumpThreading.cpp b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
index 96aef90c1c1a..9dc3b0351346 100644
--- a/llvm/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/llvm/lib/Transforms/Scalar/JumpThreading.cpp
@@ -31,6 +31,7 @@
 #include "llvm/Analysis/LazyValueInfo.h"
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/MemoryLocation.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -370,7 +371,6 @@ PreservedAnalyses JumpThreadingPass::run(Function &F,
   if (!Changed)
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
-  PA.preserve<GlobalsAA>();
   PA.preserve<DominatorTreeAnalysis>();
   PA.preserve<LazyValueAnalysis>();
   return PA;
@@ -462,7 +462,7 @@ bool JumpThreadingPass::runImpl(Function &F, TargetLibraryInfo *TLI_,
         BasicBlock *Succ = BI->getSuccessor(0);
         if (
             // The terminator must be the only non-phi instruction in BB.
-            BB.getFirstNonPHIOrDbg()->isTerminator() &&
+            BB.getFirstNonPHIOrDbg(true)->isTerminator() &&
             // Don't alter Loop headers and latches to ensure another pass can
             // detect and transform nested loops later.
             !LoopHeaders.count(&BB) && !LoopHeaders.count(Succ) &&
@@ -733,23 +733,26 @@ bool JumpThreadingPass::computeValueKnownInPredecessorsImpl(
 
   // Handle some boolean conditions.
   if (I->getType()->getPrimitiveSizeInBits() == 1) {
+    using namespace PatternMatch;
+
     assert(Preference == WantInteger && "One-bit non-integer type?");
     // X | true -> true
     // X & false -> false
-    if (I->getOpcode() == Instruction::Or ||
-        I->getOpcode() == Instruction::And) {
+    Value *Op0, *Op1;
+    if (match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1))) ||
+        match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
       PredValueInfoTy LHSVals, RHSVals;
 
-      computeValueKnownInPredecessorsImpl(I->getOperand(0), BB, LHSVals,
-                                      WantInteger, RecursionSet, CxtI);
-      computeValueKnownInPredecessorsImpl(I->getOperand(1), BB, RHSVals,
-                                          WantInteger, RecursionSet, CxtI);
+      computeValueKnownInPredecessorsImpl(Op0, BB, LHSVals, WantInteger,
+                                          RecursionSet, CxtI);
+      computeValueKnownInPredecessorsImpl(Op1, BB, RHSVals, WantInteger,
+                                          RecursionSet, CxtI);
 
       if (LHSVals.empty() && RHSVals.empty())
         return false;
 
       ConstantInt *InterestingVal;
-      if (I->getOpcode() == Instruction::Or)
+      if (match(I, m_LogicalOr()))
         InterestingVal = ConstantInt::getTrue(I->getContext());
       else
         InterestingVal = ConstantInt::getFalse(I->getContext());
@@ -1105,6 +1108,7 @@ bool JumpThreadingPass::processBlock(BasicBlock *BB) {
     LLVM_DEBUG(dbgs() << "  In block '" << BB->getName()
                       << "' folding undef terminator: " << *BBTerm << '\n');
     BranchInst::Create(BBTerm->getSuccessor(BestSucc), BBTerm);
+    ++NumFolds;
     BBTerm->eraseFromParent();
     DTU->applyUpdatesPermissive(Updates);
     if (FI)
@@ -1150,8 +1154,8 @@ bool JumpThreadingPass::processBlock(BasicBlock *BB) {
       assert(CondBr->isConditional() && "Threading on unconditional terminator");
 
       LazyValueInfo::Tristate Ret =
-        LVI->getPredicateAt(CondCmp->getPredicate(), CondCmp->getOperand(0),
-                            CondConst, CondBr);
+          LVI->getPredicateAt(CondCmp->getPredicate(), CondCmp->getOperand(0),
+                              CondConst, CondBr, /*UseBlockValue=*/false);
       if (Ret != LazyValueInfo::Unknown) {
         unsigned ToRemove = Ret == LazyValueInfo::True ? 1 : 0;
         unsigned ToKeep = Ret == LazyValueInfo::True ? 0 : 1;
@@ -1160,6 +1164,7 @@ bool JumpThreadingPass::processBlock(BasicBlock *BB) {
         BranchInst *UncondBr =
           BranchInst::Create(CondBr->getSuccessor(ToKeep), CondBr);
         UncondBr->setDebugLoc(CondBr->getDebugLoc());
+        ++NumFolds;
         CondBr->eraseFromParent();
         if (CondCmp->use_empty())
           CondCmp->eraseFromParent();
@@ -1275,6 +1280,7 @@ bool JumpThreadingPass::processImpliedCondition(BasicBlock *BB) {
       RemoveSucc->removePredecessor(BB);
       BranchInst *UncondBI = BranchInst::Create(KeepSucc, BI);
       UncondBI->setDebugLoc(BI->getDebugLoc());
+      ++NumFolds;
       BI->eraseFromParent();
       DTU->applyUpdatesPermissive({{DominatorTree::Delete, BB, RemoveSucc}});
       if (HasProfileData)
@@ -1384,10 +1390,14 @@ bool JumpThreadingPass::simplifyPartiallyRedundantLoad(LoadInst *LoadI) {
            "Attempting to CSE volatile or atomic loads");
     // If this is a load on a phi pointer, phi-translate it and search
     // for available load/store to the pointer in predecessors.
-    Value *Ptr = LoadedPtr->DoPHITranslation(LoadBB, PredBB);
-    PredAvailable = FindAvailablePtrLoadStore(
-        Ptr, LoadI->getType(), LoadI->isAtomic(), PredBB, BBIt,
-        DefMaxInstsToScan, AA, &IsLoadCSE, &NumScanedInst);
+    Type *AccessTy = LoadI->getType();
+    const auto &DL = LoadI->getModule()->getDataLayout();
+    MemoryLocation Loc(LoadedPtr->DoPHITranslation(LoadBB, PredBB),
+                       LocationSize::precise(DL.getTypeStoreSize(AccessTy)),
+                       AATags);
+    PredAvailable = findAvailablePtrLoadStore(Loc, AccessTy, LoadI->isAtomic(),
+                                              PredBB, BBIt, DefMaxInstsToScan,
+                                              AA, &IsLoadCSE, &NumScanedInst);
 
     // If PredBB has a single predecessor, continue scanning through the
     // single predecessor.
@@ -1397,8 +1407,8 @@ bool JumpThreadingPass::simplifyPartiallyRedundantLoad(LoadInst *LoadI) {
       SinglePredBB = SinglePredBB->getSinglePredecessor();
       if (SinglePredBB) {
         BBIt = SinglePredBB->end();
-        PredAvailable = FindAvailablePtrLoadStore(
-            Ptr, LoadI->getType(), LoadI->isAtomic(), SinglePredBB, BBIt,
+        PredAvailable = findAvailablePtrLoadStore(
+            Loc, AccessTy, LoadI->isAtomic(), SinglePredBB, BBIt,
             (DefMaxInstsToScan - NumScanedInst), AA, &IsLoadCSE,
             &NumScanedInst);
       }
@@ -1720,6 +1730,7 @@ bool JumpThreadingPass::processThreadableEdges(Value *Cond, BasicBlock *BB,
       // Finally update the terminator.
       Instruction *Term = BB->getTerminator();
       BranchInst::Create(OnlyDest, Term);
+      ++NumFolds;
       Term->eraseFromParent();
       DTU->applyUpdatesPermissive(Updates);
       if (HasProfileData)
@@ -2076,6 +2087,15 @@ JumpThreadingPass::cloneInstructions(BasicBlock::iterator BI,
     ValueMapping[PN] = NewPN;
   }
 
+  // Clone noalias scope declarations in the threaded block. When threading a
+  // loop exit, we would otherwise end up with two idential scope declarations
+  // visible at the same time.
+  SmallVector<MDNode *> NoAliasScopes;
+  DenseMap<MDNode *, MDNode *> ClonedScopes;
+  LLVMContext &Context = PredBB->getContext();
+  identifyNoAliasScopesToClone(BI, BE, NoAliasScopes);
+  cloneNoAliasScopes(NoAliasScopes, ClonedScopes, "thread", Context);
+
   // Clone the non-phi instructions of the source basic block into NewBB,
   // keeping track of the mapping and using it to remap operands in the cloned
   // instructions.
@@ -2084,6 +2104,7 @@ JumpThreadingPass::cloneInstructions(BasicBlock::iterator BI,
     New->setName(BI->getName());
     NewBB->getInstList().push_back(New);
     ValueMapping[&*BI] = New;
+    adaptNoAliasScopes(New, ClonedScopes, Context);
 
     // Remap operands to patch up intra-block references.
     for (unsigned i = 0, e = New->getNumOperands(); i != e; ++i)
@@ -2727,7 +2748,8 @@ void JumpThreadingPass::unfoldSelectInstr(BasicBlock *Pred, BasicBlock *BB,
   PredTerm->removeFromParent();
   NewBB->getInstList().insert(NewBB->end(), PredTerm);
   // Create a conditional branch and update PHI nodes.
-  BranchInst::Create(NewBB, BB, SI->getCondition(), Pred);
+  auto *BI = BranchInst::Create(NewBB, BB, SI->getCondition(), Pred);
+  BI->applyMergedLocation(PredTerm->getDebugLoc(), SI->getDebugLoc());
   SIUse->setIncomingValue(Idx, SI->getFalseValue());
   SIUse->addIncoming(SI->getTrueValue(), NewBB);
 
@@ -2861,11 +2883,14 @@ bool JumpThreadingPass::tryToUnfoldSelectInCurrBB(BasicBlock *BB) {
       continue;
 
     auto isUnfoldCandidate = [BB](SelectInst *SI, Value *V) {
+      using namespace PatternMatch;
+
       // Check if SI is in BB and use V as condition.
       if (SI->getParent() != BB)
         return false;
       Value *Cond = SI->getCondition();
-      return (Cond && Cond == V && Cond->getType()->isIntegerTy(1));
+      bool IsAndOr = match(SI, m_CombineOr(m_LogicalAnd(), m_LogicalOr()));
+      return Cond && Cond == V && Cond->getType()->isIntegerTy(1) && !IsAndOr;
     };
 
     SelectInst *SI = nullptr;
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index d2b4ba296f41..30058df3ded5 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -162,6 +162,7 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
                  OptimizationRemarkEmitter *ORE);
 static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                            const DominatorTree *DT,
+                                           const TargetLibraryInfo *TLI,
                                            const Loop *CurLoop,
                                            const LoopSafetyInfo *SafetyInfo,
                                            OptimizationRemarkEmitter *ORE,
@@ -185,12 +186,17 @@ static void moveInstructionBefore(Instruction &I, Instruction &Dest,
                                   ICFLoopSafetyInfo &SafetyInfo,
                                   MemorySSAUpdater *MSSAU, ScalarEvolution *SE);
 
+static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
+                                function_ref<void(Instruction *)> Fn);
+static SmallVector<SmallSetVector<Value *, 8>, 0>
+collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L);
+
 namespace {
 struct LoopInvariantCodeMotion {
   bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
                  BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI,
                  TargetTransformInfo *TTI, ScalarEvolution *SE, MemorySSA *MSSA,
-                 OptimizationRemarkEmitter *ORE);
+                 OptimizationRemarkEmitter *ORE, bool LoopNestMode = false);
 
   LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
                           unsigned LicmMssaNoAccForPromotionCap)
@@ -203,9 +209,6 @@ private:
 
   std::unique_ptr<AliasSetTracker>
   collectAliasInfoForLoop(Loop *L, LoopInfo *LI, AAResults *AA);
-  std::unique_ptr<AliasSetTracker>
-  collectAliasInfoForLoopWithMSSA(Loop *L, AAResults *AA,
-                                  MemorySSAUpdater *MSSAU);
 };
 
 struct LegacyLICMPass : public LoopPass {
@@ -292,6 +295,33 @@ PreservedAnalyses LICMPass::run(Loop &L, LoopAnalysisManager &AM,
   return PA;
 }
 
+PreservedAnalyses LNICMPass::run(LoopNest &LN, LoopAnalysisManager &AM,
+                                 LoopStandardAnalysisResults &AR,
+                                 LPMUpdater &) {
+  // For the new PM, we also can't use OptimizationRemarkEmitter as an analysis
+  // pass.  Function analyses need to be preserved across loop transformations
+  // but ORE cannot be preserved (see comment before the pass definition).
+  OptimizationRemarkEmitter ORE(LN.getParent());
+
+  LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
+
+  Loop &OutermostLoop = LN.getOutermostLoop();
+  bool Changed = LICM.runOnLoop(&OutermostLoop, &AR.AA, &AR.LI, &AR.DT, AR.BFI,
+                                &AR.TLI, &AR.TTI, &AR.SE, AR.MSSA, &ORE, true);
+
+  if (!Changed)
+    return PreservedAnalyses::all();
+
+  auto PA = getLoopPassPreservedAnalyses();
+
+  PA.preserve<DominatorTreeAnalysis>();
+  PA.preserve<LoopAnalysis>();
+  if (AR.MSSA)
+    PA.preserve<MemorySSAAnalysis>();
+
+  return PA;
+}
+
 char LegacyLICMPass::ID = 0;
 INITIALIZE_PASS_BEGIN(LegacyLICMPass, "licm", "Loop Invariant Code Motion",
                       false, false)
@@ -344,7 +374,8 @@ llvm::SinkAndHoistLICMFlags::SinkAndHoistLICMFlags(
 bool LoopInvariantCodeMotion::runOnLoop(
     Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
     BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
-    ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) {
+    ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE,
+    bool LoopNestMode) {
   bool Changed = false;
 
   assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.");
@@ -359,6 +390,22 @@ bool LoopInvariantCodeMotion::runOnLoop(
   std::unique_ptr<MemorySSAUpdater> MSSAU;
   std::unique_ptr<SinkAndHoistLICMFlags> Flags;
 
+  // Don't sink stores from loops with coroutine suspend instructions.
+  // LICM would sink instructions into the default destination of
+  // the coroutine switch. The default destination of the switch is to
+  // handle the case where the coroutine is suspended, by which point the
+  // coroutine frame may have been destroyed. No instruction can be sunk there.
+  // FIXME: This would unfortunately hurt the performance of coroutines, however
+  // there is currently no general solution for this. Similar issues could also
+  // potentially happen in other passes where instructions are being moved
+  // across that edge.
+  bool HasCoroSuspendInst = llvm::any_of(L->getBlocks(), [](BasicBlock *BB) {
+    return llvm::any_of(*BB, [](Instruction &I) {
+      IntrinsicInst *II = dyn_cast<IntrinsicInst>(&I);
+      return II && II->getIntrinsicID() == Intrinsic::coro_suspend;
+    });
+  });
+
   if (!MSSA) {
     LLVM_DEBUG(dbgs() << "LICM: Using Alias Set Tracker.\n");
     CurAST = collectAliasInfoForLoop(L, LI, AA);
@@ -395,7 +442,7 @@ bool LoopInvariantCodeMotion::runOnLoop(
   if (Preheader)
     Changed |= hoistRegion(DT->getNode(L->getHeader()), AA, LI, DT, BFI, TLI, L,
                            CurAST.get(), MSSAU.get(), SE, &SafetyInfo,
-                           *Flags.get(), ORE);
+                           *Flags.get(), ORE, LoopNestMode);
 
   // Now that all loop invariants have been removed from the loop, promote any
   // memory references to scalars that we can.
@@ -405,7 +452,7 @@ bool LoopInvariantCodeMotion::runOnLoop(
   // preheader for SSA updater, so also avoid sinking when no preheader
   // is available.
   if (!DisablePromotion && Preheader && L->hasDedicatedExits() &&
-      !Flags->tooManyMemoryAccesses()) {
+      !Flags->tooManyMemoryAccesses() && !HasCoroSuspendInst) {
     // Figure out the loop exits and their insertion points
     SmallVector<BasicBlock *, 8> ExitBlocks;
     L->getUniqueExitBlocks(ExitBlocks);
@@ -430,31 +477,43 @@ bool LoopInvariantCodeMotion::runOnLoop(
       PredIteratorCache PIC;
 
       bool Promoted = false;
+      if (CurAST.get()) {
+        // Loop over all of the alias sets in the tracker object.
+        for (AliasSet &AS : *CurAST) {
+          // We can promote this alias set if it has a store, if it is a "Must"
+          // alias set, if the pointer is loop invariant, and if we are not
+          // eliminating any volatile loads or stores.
+          if (AS.isForwardingAliasSet() || !AS.isMod() || !AS.isMustAlias() ||
+              !L->isLoopInvariant(AS.begin()->getValue()))
+            continue;
 
-      // Build an AST using MSSA.
-      if (!CurAST.get())
-        CurAST = collectAliasInfoForLoopWithMSSA(L, AA, MSSAU.get());
-
-      // Loop over all of the alias sets in the tracker object.
-      for (AliasSet &AS : *CurAST) {
-        // We can promote this alias set if it has a store, if it is a "Must"
-        // alias set, if the pointer is loop invariant, and if we are not
-        // eliminating any volatile loads or stores.
-        if (AS.isForwardingAliasSet() || !AS.isMod() || !AS.isMustAlias() ||
-            !L->isLoopInvariant(AS.begin()->getValue()))
-          continue;
-
-        assert(
-            !AS.empty() &&
-            "Must alias set should have at least one pointer element in it!");
+          assert(
+              !AS.empty() &&
+              "Must alias set should have at least one pointer element in it!");
 
-        SmallSetVector<Value *, 8> PointerMustAliases;
-        for (const auto &ASI : AS)
-          PointerMustAliases.insert(ASI.getValue());
+          SmallSetVector<Value *, 8> PointerMustAliases;
+          for (const auto &ASI : AS)
+            PointerMustAliases.insert(ASI.getValue());
 
-        Promoted |= promoteLoopAccessesToScalars(
-            PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC, LI,
-            DT, TLI, L, CurAST.get(), MSSAU.get(), &SafetyInfo, ORE);
+          Promoted |= promoteLoopAccessesToScalars(
+              PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC, LI,
+              DT, TLI, L, CurAST.get(), MSSAU.get(), &SafetyInfo, ORE);
+        }
+      } else {
+        // Promoting one set of accesses may make the pointers for another set
+        // loop invariant, so run this in a loop (with the MaybePromotable set
+        // decreasing in size over time).
+        bool LocalPromoted;
+        do {
+          LocalPromoted = false;
+          for (const SmallSetVector<Value *, 8> &PointerMustAliases :
+               collectPromotionCandidates(MSSA, AA, L)) {
+            LocalPromoted |= promoteLoopAccessesToScalars(
+                PointerMustAliases, ExitBlocks, InsertPts, MSSAInsertPts, PIC,
+                LI, DT, TLI, L, /*AST*/nullptr, MSSAU.get(), &SafetyInfo, ORE);
+          }
+          Promoted |= LocalPromoted;
+        } while (LocalPromoted);
       }
 
       // Once we have promoted values across the loop body we have to
@@ -521,8 +580,8 @@ bool llvm::sinkRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
     for (BasicBlock::iterator II = BB->end(); II != BB->begin();) {
       Instruction &I = *--II;
 
-      // If the instruction is dead, we would try to sink it because it isn't
-      // used in the loop, instead, just delete it.
+      // The instruction is not used in the loop if it is dead.  In this case,
+      // we just delete it instead of sinking it.
       if (isInstructionTriviallyDead(&I, TLI)) {
         LLVM_DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n');
         salvageKnowledge(&I);
@@ -828,7 +887,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                        AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU,
                        ScalarEvolution *SE, ICFLoopSafetyInfo *SafetyInfo,
                        SinkAndHoistLICMFlags &Flags,
-                       OptimizationRemarkEmitter *ORE) {
+                       OptimizationRemarkEmitter *ORE, bool LoopNestMode) {
   // Verify inputs.
   assert(N != nullptr && AA != nullptr && LI != nullptr && DT != nullptr &&
          CurLoop != nullptr && SafetyInfo != nullptr &&
@@ -851,7 +910,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
   for (BasicBlock *BB : Worklist) {
     // Only need to process the contents of this block if it is not part of a
     // subloop (which would already have been processed).
-    if (inSubLoop(BB, CurLoop, LI))
+    if (!LoopNestMode && inSubLoop(BB, CurLoop, LI))
       continue;
 
     for (BasicBlock::iterator II = BB->begin(), E = BB->end(); II != E;) {
@@ -885,7 +944,7 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
                              ORE) &&
           worthSinkOrHoistInst(I, CurLoop->getLoopPreheader(), ORE, BFI) &&
           isSafeToExecuteUnconditionally(
-              I, DT, CurLoop, SafetyInfo, ORE,
+              I, DT, TLI, CurLoop, SafetyInfo, ORE,
               CurLoop->getLoopPreheader()->getTerminator())) {
         hoist(I, DT, CurLoop, CFH.getOrCreateHoistedBlock(BB), SafetyInfo,
               MSSAU, SE, ORE);
@@ -1089,7 +1148,7 @@ bool isHoistableAndSinkableInst(Instruction &I) {
           isa<InsertValueInst>(I) || isa<FreezeInst>(I));
 }
 /// Return true if all of the alias sets within this AST are known not to
-/// contain a Mod, or if MSSA knows thare are no MemoryDefs in the loop.
+/// contain a Mod, or if MSSA knows there are no MemoryDefs in the loop.
 bool isReadOnly(AliasSetTracker *CurAST, const MemorySSAUpdater *MSSAU,
                 const Loop *L) {
   if (CurAST) {
@@ -1314,7 +1373,7 @@ bool llvm::canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT,
               }
               // Any call, while it may not be clobbering SI, it may be a use.
               if (auto *CI = dyn_cast<CallInst>(MD->getMemoryInst())) {
-                // Check if the call may read from the memory locattion written
+                // Check if the call may read from the memory location written
                 // to by SI. Check CI's attributes and arguments; the number of
                 // such checks performed is limited above by NoOfMemAccTooLarge.
                 ModRefInfo MRI = AA->getModRefInfo(CI, MemoryLocation::get(SI));
@@ -1466,25 +1525,23 @@ static Instruction *cloneInstructionInExitBlock(
     }
   }
 
-  // Build LCSSA PHI nodes for any in-loop operands. Note that this is
-  // particularly cheap because we can rip off the PHI node that we're
+  // Build LCSSA PHI nodes for any in-loop operands (if legal).  Note that
+  // this is particularly cheap because we can rip off the PHI node that we're
   // replacing for the number and blocks of the predecessors.
   // OPT: If this shows up in a profile, we can instead finish sinking all
   // invariant instructions, and then walk their operands to re-establish
   // LCSSA. That will eliminate creating PHI nodes just to nuke them when
   // sinking bottom-up.
-  for (User::op_iterator OI = New->op_begin(), OE = New->op_end(); OI != OE;
-       ++OI)
-    if (Instruction *OInst = dyn_cast<Instruction>(*OI))
-      if (Loop *OLoop = LI->getLoopFor(OInst->getParent()))
-        if (!OLoop->contains(&PN)) {
-          PHINode *OpPN =
-              PHINode::Create(OInst->getType(), PN.getNumIncomingValues(),
-                              OInst->getName() + ".lcssa", &ExitBlock.front());
-          for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
-            OpPN->addIncoming(OInst, PN.getIncomingBlock(i));
-          *OI = OpPN;
-        }
+  for (Use &Op : New->operands())
+    if (LI->wouldBeOutOfLoopUseRequiringLCSSA(Op.get(), PN.getParent())) {
+      auto *OInst = cast<Instruction>(Op.get());
+      PHINode *OpPN =
+        PHINode::Create(OInst->getType(), PN.getNumIncomingValues(),
+                        OInst->getName() + ".lcssa", &ExitBlock.front());
+      for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
+        OpPN->addIncoming(OInst, PN.getIncomingBlock(i));
+      Op = OpPN;
+    }
   return New;
 }
 
@@ -1627,17 +1684,8 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
                  BlockFrequencyInfo *BFI, const Loop *CurLoop,
                  ICFLoopSafetyInfo *SafetyInfo, MemorySSAUpdater *MSSAU,
                  OptimizationRemarkEmitter *ORE) {
-  LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n");
-  ORE->emit([&]() {
-    return OptimizationRemark(DEBUG_TYPE, "InstSunk", &I)
-           << "sinking " << ore::NV("Inst", &I);
-  });
   bool Changed = false;
-  if (isa<LoadInst>(I))
-    ++NumMovedLoads;
-  else if (isa<CallInst>(I))
-    ++NumMovedCalls;
-  ++NumSunk;
+  LLVM_DEBUG(dbgs() << "LICM sinking instruction: " << I << "\n");
 
   // Iterate over users to be ready for actual sinking. Replace users via
   // unreachable blocks with undef and make all user PHIs trivially replaceable.
@@ -1689,6 +1737,16 @@ static bool sink(Instruction &I, LoopInfo *LI, DominatorTree *DT,
   if (VisitedUsers.empty())
     return Changed;
 
+  ORE->emit([&]() {
+    return OptimizationRemark(DEBUG_TYPE, "InstSunk", &I)
+           << "sinking " << ore::NV("Inst", &I);
+  });
+  if (isa<LoadInst>(I))
+    ++NumMovedLoads;
+  else if (isa<CallInst>(I))
+    ++NumMovedCalls;
+  ++NumSunk;
+
 #ifndef NDEBUG
   SmallVector<BasicBlock *, 32> ExitBlocks;
   CurLoop->getUniqueExitBlocks(ExitBlocks);
@@ -1752,12 +1810,16 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
   // Conservatively strip all metadata on the instruction unless we were
   // guaranteed to execute I if we entered the loop, in which case the metadata
   // is valid in the loop preheader.
-  if (I.hasMetadataOtherThanDebugLoc() &&
+  // Similarly, If I is a call and it is not guaranteed to execute in the loop,
+  // then moving to the preheader means we should strip attributes on the call
+  // that can cause UB since we may be hoisting above conditions that allowed
+  // inferring those attributes. They may not be valid at the preheader.
+  if ((I.hasMetadataOtherThanDebugLoc() || isa<CallInst>(I)) &&
       // The check on hasMetadataOtherThanDebugLoc is to prevent us from burning
       // time in isGuaranteedToExecute if we don't actually have anything to
       // drop.  It is a compile time optimization, not required for correctness.
       !SafetyInfo->isGuaranteedToExecute(I, DT, CurLoop))
-    I.dropUnknownNonDebugMetadata();
+    I.dropUndefImplyingAttrsAndUnknownMetadata();
 
   if (isa<PHINode>(I))
     // Move the new node to the end of the phi list in the destination block.
@@ -1780,11 +1842,12 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
 /// or if it is a trapping instruction and is guaranteed to execute.
 static bool isSafeToExecuteUnconditionally(Instruction &Inst,
                                            const DominatorTree *DT,
+                                           const TargetLibraryInfo *TLI,
                                            const Loop *CurLoop,
                                            const LoopSafetyInfo *SafetyInfo,
                                            OptimizationRemarkEmitter *ORE,
                                            const Instruction *CtxI) {
-  if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT))
+  if (isSafeToSpeculativelyExecute(&Inst, CtxI, DT, TLI))
     return true;
 
   bool GuaranteedToExecute =
@@ -1821,19 +1884,21 @@ class LoopPromoter : public LoadAndStorePromoter {
   AAMDNodes AATags;
   ICFLoopSafetyInfo &SafetyInfo;
 
+  // We're about to add a use of V in a loop exit block.  Insert an LCSSA phi
+  // (if legal) if doing so would add an out-of-loop use to an instruction
+  // defined in-loop.
   Value *maybeInsertLCSSAPHI(Value *V, BasicBlock *BB) const {
-    if (Instruction *I = dyn_cast<Instruction>(V))
-      if (Loop *L = LI.getLoopFor(I->getParent()))
-        if (!L->contains(BB)) {
-          // We need to create an LCSSA PHI node for the incoming value and
-          // store that.
-          PHINode *PN = PHINode::Create(I->getType(), PredCache.size(BB),
-                                        I->getName() + ".lcssa", &BB->front());
-          for (BasicBlock *Pred : PredCache.get(BB))
-            PN->addIncoming(I, Pred);
-          return PN;
-        }
-    return V;
+    if (!LI.wouldBeOutOfLoopUseRequiringLCSSA(V, BB))
+      return V;
+
+    Instruction *I = cast<Instruction>(V);
+    // We need to create an LCSSA PHI node for the incoming value and
+    // store that.
+    PHINode *PN = PHINode::Create(I->getType(), PredCache.size(BB),
+                                  I->getName() + ".lcssa", &BB->front());
+    for (BasicBlock *Pred : PredCache.get(BB))
+      PN->addIncoming(I, Pred);
+    return PN;
   }
 
 public:
@@ -1911,10 +1976,21 @@ public:
   }
 };
 
+bool isNotCapturedBeforeOrInLoop(const Value *V, const Loop *L,
+                                 DominatorTree *DT) {
+  // We can perform the captured-before check against any instruction in the
+  // loop header, as the loop header is reachable from any instruction inside
+  // the loop.
+  // TODO: ReturnCaptures=true shouldn't be necessary here.
+  return !PointerMayBeCapturedBefore(V, /* ReturnCaptures */ true,
+                                     /* StoreCaptures */ true,
+                                     L->getHeader()->getTerminator(), DT);
+}
 
 /// Return true iff we can prove that a caller of this function can not inspect
 /// the contents of the provided object in a well defined program.
-bool isKnownNonEscaping(Value *Object, const TargetLibraryInfo *TLI) {
+bool isKnownNonEscaping(Value *Object, const Loop *L,
+                        const TargetLibraryInfo *TLI, DominatorTree *DT) {
   if (isa<AllocaInst>(Object))
     // Since the alloca goes out of scope, we know the caller can't retain a
     // reference to it and be well defined.  Thus, we don't need to check for
@@ -1931,7 +2007,7 @@ bool isKnownNonEscaping(Value *Object, const TargetLibraryInfo *TLI) {
   //      weaker condition and handle only AllocLikeFunctions (which are
   //      known to be noalias).  TODO
   return isAllocLikeFn(Object, TLI) &&
-    !PointerMayBeCaptured(Object, true, true);
+         isNotCapturedBeforeOrInLoop(Object, L, DT);
 }
 
 } // namespace
@@ -2017,7 +2093,7 @@ bool llvm::promoteLoopAccessesToScalars(
     // this by proving that the caller can't have a reference to the object
     // after return and thus can't possibly load from the object.
     Value *Object = getUnderlyingObject(SomePtr);
-    if (!isKnownNonEscaping(Object, TLI))
+    if (!isKnownNonEscaping(Object, CurLoop, TLI, DT))
       return false;
     // Subtlety: Alloca's aren't visible to callers, but *are* potentially
     // visible to other threads if captured and used during their lifetimes.
@@ -2054,10 +2130,11 @@ bool llvm::promoteLoopAccessesToScalars(
         // Note that proving a load safe to speculate requires proving
         // sufficient alignment at the target location.  Proving it guaranteed
         // to execute does as well.  Thus we can increase our guaranteed
-        // alignment as well. 
+        // alignment as well.
         if (!DereferenceableInPH || (InstAlignment > Alignment))
-          if (isSafeToExecuteUnconditionally(*Load, DT, CurLoop, SafetyInfo,
-                                             ORE, Preheader->getTerminator())) {
+          if (isSafeToExecuteUnconditionally(*Load, DT, TLI, CurLoop,
+                                             SafetyInfo, ORE,
+                                             Preheader->getTerminator())) {
             DereferenceableInPH = true;
             Alignment = std::max(Alignment, InstAlignment);
           }
@@ -2104,7 +2181,7 @@ bool llvm::promoteLoopAccessesToScalars(
         if (!DereferenceableInPH) {
           DereferenceableInPH = isDereferenceableAndAlignedPointer(
               Store->getPointerOperand(), Store->getValueOperand()->getType(),
-              Store->getAlign(), MDL, Preheader->getTerminator(), DT);
+              Store->getAlign(), MDL, Preheader->getTerminator(), DT, TLI);
         }
       } else
         return false; // Not a load or store.
@@ -2151,7 +2228,7 @@ bool llvm::promoteLoopAccessesToScalars(
       Value *Object = getUnderlyingObject(SomePtr);
       SafeToInsertStore =
           (isAllocLikeFn(Object, TLI) || isa<AllocaInst>(Object)) &&
-          !PointerMayBeCaptured(Object, true, true);
+          isNotCapturedBeforeOrInLoop(Object, CurLoop, DT);
     }
   }
 
@@ -2218,6 +2295,67 @@ bool llvm::promoteLoopAccessesToScalars(
   return true;
 }
 
+static void foreachMemoryAccess(MemorySSA *MSSA, Loop *L,
+                                function_ref<void(Instruction *)> Fn) {
+  for (const BasicBlock *BB : L->blocks())
+    if (const auto *Accesses = MSSA->getBlockAccesses(BB))
+      for (const auto &Access : *Accesses)
+        if (const auto *MUD = dyn_cast<MemoryUseOrDef>(&Access))
+          Fn(MUD->getMemoryInst());
+}
+
+static SmallVector<SmallSetVector<Value *, 8>, 0>
+collectPromotionCandidates(MemorySSA *MSSA, AliasAnalysis *AA, Loop *L) {
+  AliasSetTracker AST(*AA);
+
+  auto IsPotentiallyPromotable = [L](const Instruction *I) {
+    if (const auto *SI = dyn_cast<StoreInst>(I))
+      return L->isLoopInvariant(SI->getPointerOperand());
+    if (const auto *LI = dyn_cast<LoadInst>(I))
+      return L->isLoopInvariant(LI->getPointerOperand());
+    return false;
+  };
+
+  // Populate AST with potentially promotable accesses and remove them from
+  // MaybePromotable, so they will not be checked again on the next iteration.
+  SmallPtrSet<Value *, 16> AttemptingPromotion;
+  foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
+    if (IsPotentiallyPromotable(I)) {
+      AttemptingPromotion.insert(I);
+      AST.add(I);
+    }
+  });
+
+  // We're only interested in must-alias sets that contain a mod.
+  SmallVector<const AliasSet *, 8> Sets;
+  for (AliasSet &AS : AST)
+    if (!AS.isForwardingAliasSet() && AS.isMod() && AS.isMustAlias())
+      Sets.push_back(&AS);
+
+  if (Sets.empty())
+    return {}; // Nothing to promote...
+
+  // Discard any sets for which there is an aliasing non-promotable access.
+  foreachMemoryAccess(MSSA, L, [&](Instruction *I) {
+    if (AttemptingPromotion.contains(I))
+      return;
+
+    llvm::erase_if(Sets, [&](const AliasSet *AS) {
+      return AS->aliasesUnknownInst(I, *AA);
+    });
+  });
+
+  SmallVector<SmallSetVector<Value *, 8>, 0> Result;
+  for (const AliasSet *Set : Sets) {
+    SmallSetVector<Value *, 8> PointerMustAliases;
+    for (const auto &ASI : *Set)
+      PointerMustAliases.insert(ASI.getValue());
+    Result.push_back(std::move(PointerMustAliases));
+  }
+
+  return Result;
+}
+
 /// Returns an owning pointer to an alias set which incorporates aliasing info
 /// from L and all subloops of L.
 std::unique_ptr<AliasSetTracker>
@@ -2238,15 +2376,6 @@ LoopInvariantCodeMotion::collectAliasInfoForLoop(Loop *L, LoopInfo *LI,
   return CurAST;
 }
 
-std::unique_ptr<AliasSetTracker>
-LoopInvariantCodeMotion::collectAliasInfoForLoopWithMSSA(
-    Loop *L, AAResults *AA, MemorySSAUpdater *MSSAU) {
-  auto *MSSA = MSSAU->getMemorySSA();
-  auto CurAST = std::make_unique<AliasSetTracker>(*AA, MSSA, L);
-  CurAST->addAllInstructionsInLoopUsingMSSA();
-  return CurAST;
-}
-
 static bool pointerInvalidatedByLoop(MemoryLocation MemLoc,
                                      AliasSetTracker *CurAST, Loop *CurLoop,
                                      AAResults *AA) {
diff --git a/llvm/lib/Transforms/Scalar/LoopBoundSplit.cpp b/llvm/lib/Transforms/Scalar/LoopBoundSplit.cpp
new file mode 100644
index 000000000000..993b154dc9a8
--- /dev/null
+++ b/llvm/lib/Transforms/Scalar/LoopBoundSplit.cpp
@@ -0,0 +1,439 @@
+//===------- LoopBoundSplit.cpp - Split Loop Bound --------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Scalar/LoopBoundSplit.h"
+#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopIterator.h"
+#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/MemorySSAUpdater.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/IR/PatternMatch.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LoopSimplify.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
+
+#define DEBUG_TYPE "loop-bound-split"
+
+namespace llvm {
+
+using namespace PatternMatch;
+
+namespace {
+struct ConditionInfo {
+  /// Branch instruction with this condition
+  BranchInst *BI;
+  /// ICmp instruction with this condition
+  ICmpInst *ICmp;
+  /// Preciate info
+  ICmpInst::Predicate Pred;
+  /// AddRec llvm value
+  Value *AddRecValue;
+  /// Bound llvm value
+  Value *BoundValue;
+  /// AddRec SCEV
+  const SCEV *AddRecSCEV;
+  /// Bound SCEV
+  const SCEV *BoundSCEV;
+
+  ConditionInfo()
+      : BI(nullptr), ICmp(nullptr), Pred(ICmpInst::BAD_ICMP_PREDICATE),
+        AddRecValue(nullptr), BoundValue(nullptr), AddRecSCEV(nullptr),
+        BoundSCEV(nullptr) {}
+};
+} // namespace
+
+static void analyzeICmp(ScalarEvolution &SE, ICmpInst *ICmp,
+                        ConditionInfo &Cond) {
+  Cond.ICmp = ICmp;
+  if (match(ICmp, m_ICmp(Cond.Pred, m_Value(Cond.AddRecValue),
+                         m_Value(Cond.BoundValue)))) {
+    Cond.AddRecSCEV = SE.getSCEV(Cond.AddRecValue);
+    Cond.BoundSCEV = SE.getSCEV(Cond.BoundValue);
+    // Locate AddRec in LHSSCEV and Bound in RHSSCEV.
+    if (isa<SCEVAddRecExpr>(Cond.BoundSCEV) &&
+        !isa<SCEVAddRecExpr>(Cond.AddRecSCEV)) {
+      std::swap(Cond.AddRecValue, Cond.BoundValue);
+      std::swap(Cond.AddRecSCEV, Cond.BoundSCEV);
+      Cond.Pred = ICmpInst::getSwappedPredicate(Cond.Pred);
+    }
+  }
+}
+
+static bool calculateUpperBound(const Loop &L, ScalarEvolution &SE,
+                                ConditionInfo &Cond, bool IsExitCond) {
+  if (IsExitCond) {
+    const SCEV *ExitCount = SE.getExitCount(&L, Cond.ICmp->getParent());
+    if (isa<SCEVCouldNotCompute>(ExitCount))
+      return false;
+
+    Cond.BoundSCEV = ExitCount;
+    return true;
+  }
+
+  // For non-exit condtion, if pred is LT, keep existing bound.
+  if (Cond.Pred == ICmpInst::ICMP_SLT || Cond.Pred == ICmpInst::ICMP_ULT)
+    return true;
+
+  // For non-exit condition, if pre is LE, try to convert it to LT.
+  //      Range                 Range
+  // AddRec <= Bound  -->  AddRec < Bound + 1
+  if (Cond.Pred != ICmpInst::ICMP_ULE && Cond.Pred != ICmpInst::ICMP_SLE)
+    return false;
+
+  if (IntegerType *BoundSCEVIntType =
+          dyn_cast<IntegerType>(Cond.BoundSCEV->getType())) {
+    unsigned BitWidth = BoundSCEVIntType->getBitWidth();
+    APInt Max = ICmpInst::isSigned(Cond.Pred)
+                    ? APInt::getSignedMaxValue(BitWidth)
+                    : APInt::getMaxValue(BitWidth);
+    const SCEV *MaxSCEV = SE.getConstant(Max);
+    // Check Bound < INT_MAX
+    ICmpInst::Predicate Pred =
+        ICmpInst::isSigned(Cond.Pred) ? ICmpInst::ICMP_SLT : ICmpInst::ICMP_ULT;
+    if (SE.isKnownPredicate(Pred, Cond.BoundSCEV, MaxSCEV)) {
+      const SCEV *BoundPlusOneSCEV =
+          SE.getAddExpr(Cond.BoundSCEV, SE.getOne(BoundSCEVIntType));
+      Cond.BoundSCEV = BoundPlusOneSCEV;
+      Cond.Pred = Pred;
+      return true;
+    }
+  }
+
+  // ToDo: Support ICMP_NE/EQ.
+
+  return false;
+}
+
+static bool hasProcessableCondition(const Loop &L, ScalarEvolution &SE,
+                                    ICmpInst *ICmp, ConditionInfo &Cond,
+                                    bool IsExitCond) {
+  analyzeICmp(SE, ICmp, Cond);
+
+  // The BoundSCEV should be evaluated at loop entry.
+  if (!SE.isAvailableAtLoopEntry(Cond.BoundSCEV, &L))
+    return false;
+
+  const SCEVAddRecExpr *AddRecSCEV = dyn_cast<SCEVAddRecExpr>(Cond.AddRecSCEV);
+  // Allowed AddRec as induction variable.
+  if (!AddRecSCEV)
+    return false;
+
+  if (!AddRecSCEV->isAffine())
+    return false;
+
+  const SCEV *StepRecSCEV = AddRecSCEV->getStepRecurrence(SE);
+  // Allowed constant step.
+  if (!isa<SCEVConstant>(StepRecSCEV))
+    return false;
+
+  ConstantInt *StepCI = cast<SCEVConstant>(StepRecSCEV)->getValue();
+  // Allowed positive step for now.
+  // TODO: Support negative step.
+  if (StepCI->isNegative() || StepCI->isZero())
+    return false;
+
+  // Calculate upper bound.
+  if (!calculateUpperBound(L, SE, Cond, IsExitCond))
+    return false;
+
+  return true;
+}
+
+static bool isProcessableCondBI(const ScalarEvolution &SE,
+                                const BranchInst *BI) {
+  BasicBlock *TrueSucc = nullptr;
+  BasicBlock *FalseSucc = nullptr;
+  ICmpInst::Predicate Pred;
+  Value *LHS, *RHS;
+  if (!match(BI, m_Br(m_ICmp(Pred, m_Value(LHS), m_Value(RHS)),
+                      m_BasicBlock(TrueSucc), m_BasicBlock(FalseSucc))))
+    return false;
+
+  if (!SE.isSCEVable(LHS->getType()))
+    return false;
+  assert(SE.isSCEVable(RHS->getType()) && "Expected RHS's type is SCEVable");
+
+  if (TrueSucc == FalseSucc)
+    return false;
+
+  return true;
+}
+
+static bool canSplitLoopBound(const Loop &L, const DominatorTree &DT,
+                              ScalarEvolution &SE, ConditionInfo &Cond) {
+  // Skip function with optsize.
+  if (L.getHeader()->getParent()->hasOptSize())
+    return false;
+
+  // Split only innermost loop.
+  if (!L.isInnermost())
+    return false;
+
+  // Check loop is in simplified form.
+  if (!L.isLoopSimplifyForm())
+    return false;
+
+  // Check loop is in LCSSA form.
+  if (!L.isLCSSAForm(DT))
+    return false;
+
+  // Skip loop that cannot be cloned.
+  if (!L.isSafeToClone())
+    return false;
+
+  BasicBlock *ExitingBB = L.getExitingBlock();
+  // Assumed only one exiting block.
+  if (!ExitingBB)
+    return false;
+
+  BranchInst *ExitingBI = dyn_cast<BranchInst>(ExitingBB->getTerminator());
+  if (!ExitingBI)
+    return false;
+
+  // Allowed only conditional branch with ICmp.
+  if (!isProcessableCondBI(SE, ExitingBI))
+    return false;
+
+  // Check the condition is processable.
+  ICmpInst *ICmp = cast<ICmpInst>(ExitingBI->getCondition());
+  if (!hasProcessableCondition(L, SE, ICmp, Cond, /*IsExitCond*/ true))
+    return false;
+
+  Cond.BI = ExitingBI;
+  return true;
+}
+
+static bool isProfitableToTransform(const Loop &L, const BranchInst *BI) {
+  // If the conditional branch splits a loop into two halves, we could
+  // generally say it is profitable.
+  //
+  // ToDo: Add more profitable cases here.
+
+  // Check this branch causes diamond CFG.
+  BasicBlock *Succ0 = BI->getSuccessor(0);
+  BasicBlock *Succ1 = BI->getSuccessor(1);
+
+  BasicBlock *Succ0Succ = Succ0->getSingleSuccessor();
+  BasicBlock *Succ1Succ = Succ1->getSingleSuccessor();
+  if (!Succ0Succ || !Succ1Succ || Succ0Succ != Succ1Succ)
+    return false;
+
+  // ToDo: Calculate each successor's instruction cost.
+
+  return true;
+}
+
+static BranchInst *findSplitCandidate(const Loop &L, ScalarEvolution &SE,
+                                      ConditionInfo &ExitingCond,
+                                      ConditionInfo &SplitCandidateCond) {
+  for (auto *BB : L.blocks()) {
+    // Skip condition of backedge.
+    if (L.getLoopLatch() == BB)
+      continue;
+
+    auto *BI = dyn_cast<BranchInst>(BB->getTerminator());
+    if (!BI)
+      continue;
+
+    // Check conditional branch with ICmp.
+    if (!isProcessableCondBI(SE, BI))
+      continue;
+
+    // Skip loop invariant condition.
+    if (L.isLoopInvariant(BI->getCondition()))
+      continue;
+
+    // Check the condition is processable.
+    ICmpInst *ICmp = cast<ICmpInst>(BI->getCondition());
+    if (!hasProcessableCondition(L, SE, ICmp, SplitCandidateCond,
+                                 /*IsExitCond*/ false))
+      continue;
+
+    if (ExitingCond.BoundSCEV->getType() !=
+        SplitCandidateCond.BoundSCEV->getType())
+      continue;
+
+    SplitCandidateCond.BI = BI;
+    return BI;
+  }
+
+  return nullptr;
+}
+
+static bool splitLoopBound(Loop &L, DominatorTree &DT, LoopInfo &LI,
+                           ScalarEvolution &SE, LPMUpdater &U) {
+  ConditionInfo SplitCandidateCond;
+  ConditionInfo ExitingCond;
+
+  // Check we can split this loop's bound.
+  if (!canSplitLoopBound(L, DT, SE, ExitingCond))
+    return false;
+
+  if (!findSplitCandidate(L, SE, ExitingCond, SplitCandidateCond))
+    return false;
+
+  if (!isProfitableToTransform(L, SplitCandidateCond.BI))
+    return false;
+
+  // Now, we have a split candidate. Let's build a form as below.
+  //    +--------------------+
+  //    |     preheader      |
+  //    |  set up newbound   |
+  //    +--------------------+
+  //             |     /----------------\
+  //    +--------v----v------+          |
+  //    |      header        |---\      |
+  //    | with true condition|   |      |
+  //    +--------------------+   |      |
+  //             |               |      |
+  //    +--------v-----------+   |      |
+  //    |     if.then.BB     |   |      |
+  //    +--------------------+   |      |
+  //             |               |      |
+  //    +--------v-----------<---/      |
+  //    |       latch        >----------/
+  //    |   with newbound    |
+  //    +--------------------+
+  //             |
+  //    +--------v-----------+
+  //    |     preheader2     |--------------\
+  //    | if (AddRec i !=    |              |
+  //    |     org bound)     |              |
+  //    +--------------------+              |
+  //             |     /----------------\   |
+  //    +--------v----v------+          |   |
+  //    |      header2       |---\      |   |
+  //    | conditional branch |   |      |   |
+  //    |with false condition|   |      |   |
+  //    +--------------------+   |      |   |
+  //             |               |      |   |
+  //    +--------v-----------+   |      |   |
+  //    |    if.then.BB2     |   |      |   |
+  //    +--------------------+   |      |   |
+  //             |               |      |   |
+  //    +--------v-----------<---/      |   |
+  //    |       latch2       >----------/   |
+  //    |   with org bound   |              |
+  //    +--------v-----------+              |
+  //             |                          |
+  //             |  +---------------+       |
+  //             +-->     exit      <-------/
+  //                +---------------+
+
+  // Let's create post loop.
+  SmallVector<BasicBlock *, 8> PostLoopBlocks;
+  Loop *PostLoop;
+  ValueToValueMapTy VMap;
+  BasicBlock *PreHeader = L.getLoopPreheader();
+  BasicBlock *SplitLoopPH = SplitEdge(PreHeader, L.getHeader(), &DT, &LI);
+  PostLoop = cloneLoopWithPreheader(L.getExitBlock(), SplitLoopPH, &L, VMap,
+                                    ".split", &LI, &DT, PostLoopBlocks);
+  remapInstructionsInBlocks(PostLoopBlocks, VMap);
+
+  // Add conditional branch to check we can skip post-loop in its preheader.
+  BasicBlock *PostLoopPreHeader = PostLoop->getLoopPreheader();
+  IRBuilder<> Builder(PostLoopPreHeader);
+  Instruction *OrigBI = PostLoopPreHeader->getTerminator();
+  ICmpInst::Predicate Pred = ICmpInst::ICMP_NE;
+  Value *Cond =
+      Builder.CreateICmp(Pred, ExitingCond.AddRecValue, ExitingCond.BoundValue);
+  Builder.CreateCondBr(Cond, PostLoop->getHeader(), PostLoop->getExitBlock());
+  OrigBI->eraseFromParent();
+
+  // Create new loop bound and add it into preheader of pre-loop.
+  const SCEV *NewBoundSCEV = ExitingCond.BoundSCEV;
+  const SCEV *SplitBoundSCEV = SplitCandidateCond.BoundSCEV;
+  NewBoundSCEV = ICmpInst::isSigned(ExitingCond.Pred)
+                     ? SE.getSMinExpr(NewBoundSCEV, SplitBoundSCEV)
+                     : SE.getUMinExpr(NewBoundSCEV, SplitBoundSCEV);
+
+  SCEVExpander Expander(
+      SE, L.getHeader()->getParent()->getParent()->getDataLayout(), "split");
+  Instruction *InsertPt = SplitLoopPH->getTerminator();
+  Value *NewBoundValue =
+      Expander.expandCodeFor(NewBoundSCEV, NewBoundSCEV->getType(), InsertPt);
+  NewBoundValue->setName("new.bound");
+
+  // Replace exiting bound value of pre-loop NewBound.
+  ExitingCond.ICmp->setOperand(1, NewBoundValue);
+
+  // Replace IV's start value of post-loop by NewBound.
+  for (PHINode &PN : L.getHeader()->phis()) {
+    // Find PHI with exiting condition from pre-loop.
+    if (SE.isSCEVable(PN.getType()) && isa<SCEVAddRecExpr>(SE.getSCEV(&PN))) {
+      for (Value *Op : PN.incoming_values()) {
+        if (Op == ExitingCond.AddRecValue) {
+          // Find cloned PHI for post-loop.
+          PHINode *PostLoopPN = cast<PHINode>(VMap[&PN]);
+          PostLoopPN->setIncomingValueForBlock(PostLoopPreHeader,
+                                               NewBoundValue);
+        }
+      }
+    }
+  }
+
+  // Replace SplitCandidateCond.BI's condition of pre-loop by True.
+  LLVMContext &Context = PreHeader->getContext();
+  SplitCandidateCond.BI->setCondition(ConstantInt::getTrue(Context));
+
+  // Replace cloned SplitCandidateCond.BI's condition in post-loop by False.
+  BranchInst *ClonedSplitCandidateBI =
+      cast<BranchInst>(VMap[SplitCandidateCond.BI]);
+  ClonedSplitCandidateBI->setCondition(ConstantInt::getFalse(Context));
+
+  // Replace exit branch target of pre-loop by post-loop's preheader.
+  if (L.getExitBlock() == ExitingCond.BI->getSuccessor(0))
+    ExitingCond.BI->setSuccessor(0, PostLoopPreHeader);
+  else
+    ExitingCond.BI->setSuccessor(1, PostLoopPreHeader);
+
+  // Update dominator tree.
+  DT.changeImmediateDominator(PostLoopPreHeader, L.getExitingBlock());
+  DT.changeImmediateDominator(PostLoop->getExitBlock(), PostLoopPreHeader);
+
+  // Invalidate cached SE information.
+  SE.forgetLoop(&L);
+
+  // Canonicalize loops.
+  // TODO: Try to update LCSSA information according to above change.
+  formLCSSA(L, DT, &LI, &SE);
+  simplifyLoop(&L, &DT, &LI, &SE, nullptr, nullptr, true);
+  formLCSSA(*PostLoop, DT, &LI, &SE);
+  simplifyLoop(PostLoop, &DT, &LI, &SE, nullptr, nullptr, true);
+
+  // Add new post-loop to loop pass manager.
+  U.addSiblingLoops(PostLoop);
+
+  return true;
+}
+
+PreservedAnalyses LoopBoundSplitPass::run(Loop &L, LoopAnalysisManager &AM,
+                                          LoopStandardAnalysisResults &AR,
+                                          LPMUpdater &U) {
+  Function &F = *L.getHeader()->getParent();
+  (void)F;
+
+  LLVM_DEBUG(dbgs() << "Spliting bound of loop in " << F.getName() << ": " << L
+                    << "\n");
+
+  if (!splitLoopBound(L, AR.DT, AR.LI, AR.SE, U))
+    return PreservedAnalyses::all();
+
+  assert(AR.DT.verify(DominatorTree::VerificationLevel::Fast));
+  AR.LI.verify(AR.DT);
+
+  return getLoopPassPreservedAnalyses();
+}
+
+} // end namespace llvm
diff --git a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
index 45cdcb2f37dd..a5d7835bd094 100644
--- a/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDataPrefetch.cpp
@@ -13,7 +13,6 @@
 #include "llvm/Transforms/Scalar/LoopDataPrefetch.h"
 #include "llvm/InitializePasses.h"
 
-#define DEBUG_TYPE "loop-data-prefetch"
 #include "llvm/ADT/DepthFirstIterator.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AssumptionCache.h"
@@ -33,6 +32,9 @@
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
+
+#define DEBUG_TYPE "loop-data-prefetch"
+
 using namespace llvm;
 
 // By default, we limit this to creating 16 PHIs (which is a little over half
diff --git a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
index 1266c93316fa..f7e8442fae81 100644
--- a/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDeletion.cpp
@@ -16,11 +16,15 @@
 #include "llvm/Transforms/Scalar/LoopDeletion.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/CFG.h"
 #include "llvm/Analysis/GlobalsModRef.h"
+#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LoopIterator.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/IR/Dominators.h"
+
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Transforms/Scalar.h"
@@ -33,6 +37,11 @@ using namespace llvm;
 
 STATISTIC(NumDeleted, "Number of loops deleted");
 
+static cl::opt<bool> EnableSymbolicExecution(
+    "loop-deletion-enable-symbolic-execution", cl::Hidden, cl::init(true),
+    cl::desc("Break backedge through symbolic execution of 1st iteration "
+             "attempting to prove that the backedge is never taken"));
+
 enum class LoopDeletionResult {
   Unmodified,
   Modified,
@@ -54,7 +63,7 @@ static LoopDeletionResult merge(LoopDeletionResult A, LoopDeletionResult B) {
 static bool isLoopDead(Loop *L, ScalarEvolution &SE,
                        SmallVectorImpl<BasicBlock *> &ExitingBlocks,
                        BasicBlock *ExitBlock, bool &Changed,
-                       BasicBlock *Preheader) {
+                       BasicBlock *Preheader, LoopInfo &LI) {
   // Make sure that all PHI entries coming from the loop are loop invariant.
   // Because the code is in LCSSA form, any values used outside of the loop
   // must pass through a PHI in the exit block, meaning that this check is
@@ -100,6 +109,36 @@ static bool isLoopDead(Loop *L, ScalarEvolution &SE,
           return I.mayHaveSideEffects() && !I.isDroppable();
         }))
       return false;
+
+  // The loop or any of its sub-loops looping infinitely is legal. The loop can
+  // only be considered dead if either
+  // a. the function is mustprogress.
+  // b. all (sub-)loops are mustprogress or have a known trip-count.
+  if (L->getHeader()->getParent()->mustProgress())
+    return true;
+
+  LoopBlocksRPO RPOT(L);
+  RPOT.perform(&LI);
+  // If the loop contains an irreducible cycle, it may loop infinitely.
+  if (containsIrreducibleCFG<const BasicBlock *>(RPOT, LI))
+    return false;
+
+  SmallVector<Loop *, 8> WorkList;
+  WorkList.push_back(L);
+  while (!WorkList.empty()) {
+    Loop *Current = WorkList.pop_back_val();
+    if (hasMustProgress(Current))
+      continue;
+
+    const SCEV *S = SE.getConstantMaxBackedgeTakenCount(Current);
+    if (isa<SCEVCouldNotCompute>(S)) {
+      LLVM_DEBUG(
+          dbgs() << "Could not compute SCEV MaxBackedgeTakenCount and was "
+                    "not required to make progress.\n");
+      return false;
+    }
+    WorkList.append(Current->begin(), Current->end());
+  }
   return true;
 }
 
@@ -114,7 +153,7 @@ static bool isLoopNeverExecuted(Loop *L) {
   // predecessor.
   assert(Preheader && "Needs preheader!");
 
-  if (Preheader == &Preheader->getParent()->getEntryBlock())
+  if (Preheader->isEntryBlock())
     return false;
   // All predecessors of the preheader should have a constant conditional
   // branch, with the loop's preheader as not-taken.
@@ -135,6 +174,213 @@ static bool isLoopNeverExecuted(Loop *L) {
   return true;
 }
 
+static Value *
+getValueOnFirstIteration(Value *V, DenseMap<Value *, Value *> &FirstIterValue,
+                         const SimplifyQuery &SQ) {
+  // Quick hack: do not flood cache with non-instruction values.
+  if (!isa<Instruction>(V))
+    return V;
+  // Do we already know cached result?
+  auto Existing = FirstIterValue.find(V);
+  if (Existing != FirstIterValue.end())
+    return Existing->second;
+  Value *FirstIterV = nullptr;
+  if (auto *BO = dyn_cast<BinaryOperator>(V)) {
+    Value *LHS =
+        getValueOnFirstIteration(BO->getOperand(0), FirstIterValue, SQ);
+    Value *RHS =
+        getValueOnFirstIteration(BO->getOperand(1), FirstIterValue, SQ);
+    FirstIterV = SimplifyBinOp(BO->getOpcode(), LHS, RHS, SQ);
+  }
+  if (!FirstIterV)
+    FirstIterV = V;
+  FirstIterValue[V] = FirstIterV;
+  return FirstIterV;
+}
+
+// Try to prove that one of conditions that dominates the latch must exit on 1st
+// iteration.
+static bool canProveExitOnFirstIteration(Loop *L, DominatorTree &DT,
+                                         LoopInfo &LI) {
+  // Disabled by option.
+  if (!EnableSymbolicExecution)
+    return false;
+
+  BasicBlock *Predecessor = L->getLoopPredecessor();
+  BasicBlock *Latch = L->getLoopLatch();
+
+  if (!Predecessor || !Latch)
+    return false;
+
+  LoopBlocksRPO RPOT(L);
+  RPOT.perform(&LI);
+
+  // For the optimization to be correct, we need RPOT to have a property that
+  // each block is processed after all its predecessors, which may only be
+  // violated for headers of the current loop and all nested loops. Irreducible
+  // CFG provides multiple ways to break this assumption, so we do not want to
+  // deal with it.
+  if (containsIrreducibleCFG<const BasicBlock *>(RPOT, LI))
+    return false;
+
+  BasicBlock *Header = L->getHeader();
+  // Blocks that are reachable on the 1st iteration.
+  SmallPtrSet<BasicBlock *, 4> LiveBlocks;
+  // Edges that are reachable on the 1st iteration.
+  DenseSet<BasicBlockEdge> LiveEdges;
+  LiveBlocks.insert(Header);
+
+  SmallPtrSet<BasicBlock *, 4> Visited;
+  auto MarkLiveEdge = [&](BasicBlock *From, BasicBlock *To) {
+    assert(LiveBlocks.count(From) && "Must be live!");
+    assert((LI.isLoopHeader(To) || !Visited.count(To)) &&
+           "Only canonical backedges are allowed. Irreducible CFG?");
+    assert((LiveBlocks.count(To) || !Visited.count(To)) &&
+           "We already discarded this block as dead!");
+    LiveBlocks.insert(To);
+    LiveEdges.insert({ From, To });
+  };
+
+  auto MarkAllSuccessorsLive = [&](BasicBlock *BB) {
+    for (auto *Succ : successors(BB))
+      MarkLiveEdge(BB, Succ);
+  };
+
+  // Check if there is only one value coming from all live predecessor blocks.
+  // Note that because we iterate in RPOT, we have already visited all its
+  // (non-latch) predecessors.
+  auto GetSoleInputOnFirstIteration = [&](PHINode & PN)->Value * {
+    BasicBlock *BB = PN.getParent();
+    bool HasLivePreds = false;
+    (void)HasLivePreds;
+    if (BB == Header)
+      return PN.getIncomingValueForBlock(Predecessor);
+    Value *OnlyInput = nullptr;
+    for (auto *Pred : predecessors(BB))
+      if (LiveEdges.count({ Pred, BB })) {
+        HasLivePreds = true;
+        Value *Incoming = PN.getIncomingValueForBlock(Pred);
+        // Skip undefs. If they are present, we can assume they are equal to
+        // the non-undef input.
+        if (isa<UndefValue>(Incoming))
+          continue;
+        // Two inputs.
+        if (OnlyInput && OnlyInput != Incoming)
+          return nullptr;
+        OnlyInput = Incoming;
+      }
+
+    assert(HasLivePreds && "No live predecessors?");
+    // If all incoming live value were undefs, return undef.
+    return OnlyInput ? OnlyInput : UndefValue::get(PN.getType());
+  };
+  DenseMap<Value *, Value *> FirstIterValue;
+
+  // Use the following algorithm to prove we never take the latch on the 1st
+  // iteration:
+  // 1. Traverse in topological order, so that whenever we visit a block, all
+  //    its predecessors are already visited.
+  // 2. If we can prove that the block may have only 1 predecessor on the 1st
+  //    iteration, map all its phis onto input from this predecessor.
+  // 3a. If we can prove which successor of out block is taken on the 1st
+  //     iteration, mark this successor live.
+  // 3b. If we cannot prove it, conservatively assume that all successors are
+  //     live.
+  auto &DL = Header->getModule()->getDataLayout();
+  const SimplifyQuery SQ(DL);
+  for (auto *BB : RPOT) {
+    Visited.insert(BB);
+
+    // This block is not reachable on the 1st iterations.
+    if (!LiveBlocks.count(BB))
+      continue;
+
+    // Skip inner loops.
+    if (LI.getLoopFor(BB) != L) {
+      MarkAllSuccessorsLive(BB);
+      continue;
+    }
+
+    // If Phi has only one input from all live input blocks, use it.
+    for (auto &PN : BB->phis()) {
+      if (!PN.getType()->isIntegerTy())
+        continue;
+      auto *Incoming = GetSoleInputOnFirstIteration(PN);
+      if (Incoming && DT.dominates(Incoming, BB->getTerminator())) {
+        Value *FirstIterV =
+            getValueOnFirstIteration(Incoming, FirstIterValue, SQ);
+        FirstIterValue[&PN] = FirstIterV;
+      }
+    }
+
+    using namespace PatternMatch;
+    ICmpInst::Predicate Pred;
+    Value *LHS, *RHS;
+    BasicBlock *IfTrue, *IfFalse;
+    auto *Term = BB->getTerminator();
+    if (match(Term, m_Br(m_ICmp(Pred, m_Value(LHS), m_Value(RHS)),
+                         m_BasicBlock(IfTrue), m_BasicBlock(IfFalse)))) {
+      if (!LHS->getType()->isIntegerTy()) {
+        MarkAllSuccessorsLive(BB);
+        continue;
+      }
+
+      // Can we prove constant true or false for this condition?
+      LHS = getValueOnFirstIteration(LHS, FirstIterValue, SQ);
+      RHS = getValueOnFirstIteration(RHS, FirstIterValue, SQ);
+      auto *KnownCondition = SimplifyICmpInst(Pred, LHS, RHS, SQ);
+      if (!KnownCondition) {
+        // Failed to simplify.
+        MarkAllSuccessorsLive(BB);
+        continue;
+      }
+      if (isa<UndefValue>(KnownCondition)) {
+        // TODO: According to langref, branching by undef is undefined behavior.
+        // It means that, theoretically, we should be able to just continue
+        // without marking any successors as live. However, we are not certain
+        // how correct our compiler is at handling such cases. So we are being
+        // very conservative here.
+        //
+        // If there is a non-loop successor, always assume this branch leaves the
+        // loop. Otherwise, arbitrarily take IfTrue.
+        //
+        // Once we are certain that branching by undef is handled correctly by
+        // other transforms, we should not mark any successors live here.
+        if (L->contains(IfTrue) && L->contains(IfFalse))
+          MarkLiveEdge(BB, IfTrue);
+        continue;
+      }
+      auto *ConstCondition = dyn_cast<ConstantInt>(KnownCondition);
+      if (!ConstCondition) {
+        // Non-constant condition, cannot analyze any further.
+        MarkAllSuccessorsLive(BB);
+        continue;
+      }
+      if (ConstCondition->isAllOnesValue())
+        MarkLiveEdge(BB, IfTrue);
+      else
+        MarkLiveEdge(BB, IfFalse);
+    } else if (SwitchInst *SI = dyn_cast<SwitchInst>(Term)) {
+      auto *SwitchValue = SI->getCondition();
+      auto *SwitchValueOnFirstIter =
+          getValueOnFirstIteration(SwitchValue, FirstIterValue, SQ);
+      auto *ConstSwitchValue = dyn_cast<ConstantInt>(SwitchValueOnFirstIter);
+      if (!ConstSwitchValue) {
+        MarkAllSuccessorsLive(BB);
+        continue;
+      }
+      auto CaseIterator = SI->findCaseValue(ConstSwitchValue);
+      MarkLiveEdge(BB, CaseIterator->getCaseSuccessor());
+    } else {
+      MarkAllSuccessorsLive(BB);
+      continue;
+    }
+  }
+
+  // We can break the latch if it wasn't live.
+  return !LiveEdges.count({ Latch, Header });
+}
+
 /// If we can prove the backedge is untaken, remove it.  This destroys the
 /// loop, but leaves the (now trivially loop invariant) control flow and
 /// side effects (if any) in place.
@@ -148,7 +394,9 @@ breakBackedgeIfNotTaken(Loop *L, DominatorTree &DT, ScalarEvolution &SE,
     return LoopDeletionResult::Unmodified;
 
   auto *BTC = SE.getBackedgeTakenCount(L);
-  if (!BTC->isZero())
+  if (!isa<SCEVCouldNotCompute>(BTC) && SE.isKnownNonZero(BTC))
+    return LoopDeletionResult::Unmodified;
+  if (!BTC->isZero() && !canProveExitOnFirstIteration(L, DT, LI))
     return LoopDeletionResult::Unmodified;
 
   breakLoopBackedge(L, DT, SE, LI, MSSA);
@@ -224,23 +472,12 @@ static LoopDeletionResult deleteLoopIfDead(Loop *L, DominatorTree &DT,
   }
   // Finally, we have to check that the loop really is dead.
   bool Changed = false;
-  if (!isLoopDead(L, SE, ExitingBlocks, ExitBlock, Changed, Preheader)) {
+  if (!isLoopDead(L, SE, ExitingBlocks, ExitBlock, Changed, Preheader, LI)) {
     LLVM_DEBUG(dbgs() << "Loop is not invariant, cannot delete.\n");
     return Changed ? LoopDeletionResult::Modified
                    : LoopDeletionResult::Unmodified;
   }
 
-  // Don't remove loops for which we can't solve the trip count unless the loop
-  // was required to make progress but has been determined to be dead.
-  const SCEV *S = SE.getConstantMaxBackedgeTakenCount(L);
-  if (isa<SCEVCouldNotCompute>(S) &&
-      !L->getHeader()->getParent()->mustProgress() && !hasMustProgress(L)) {
-    LLVM_DEBUG(dbgs() << "Could not compute SCEV MaxBackedgeTakenCount and was "
-                         "not required to make progress.\n");
-    return Changed ? LoopDeletionResult::Modified
-                   : LoopDeletionResult::Unmodified;
-  }
-
   LLVM_DEBUG(dbgs() << "Loop is invariant, delete it!");
   ORE.emit([&]() {
     return OptimizationRemark(DEBUG_TYPE, "Invariant", L->getStartLoc(),
diff --git a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp
index 1bd2529891b7..bac3dc0f3fb9 100644
--- a/llvm/lib/Transforms/Scalar/LoopDistribute.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopDistribute.cpp
@@ -1068,7 +1068,6 @@ PreservedAnalyses LoopDistributePass::run(Function &F,
   PreservedAnalyses PA;
   PA.preserve<LoopAnalysis>();
   PA.preserve<DominatorTreeAnalysis>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
index aaff68436c13..f54289f85ef5 100644
--- a/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFlatten.cpp
@@ -63,7 +63,7 @@ static cl::opt<bool>
     AssumeNoOverflow("loop-flatten-assume-no-overflow", cl::Hidden,
                      cl::init(false),
                      cl::desc("Assume that the product of the two iteration "
-                              "limits will never overflow"));
+                              "trip counts will never overflow"));
 
 static cl::opt<bool>
     WidenIV("loop-flatten-widen-iv", cl::Hidden,
@@ -74,10 +74,12 @@ static cl::opt<bool>
 struct FlattenInfo {
   Loop *OuterLoop = nullptr;
   Loop *InnerLoop = nullptr;
+  // These PHINodes correspond to loop induction variables, which are expected
+  // to start at zero and increment by one on each loop.
   PHINode *InnerInductionPHI = nullptr;
   PHINode *OuterInductionPHI = nullptr;
-  Value *InnerLimit = nullptr;
-  Value *OuterLimit = nullptr;
+  Value *InnerTripCount = nullptr;
+  Value *OuterTripCount = nullptr;
   BinaryOperator *InnerIncrement = nullptr;
   BinaryOperator *OuterIncrement = nullptr;
   BranchInst *InnerBranch = nullptr;
@@ -91,12 +93,12 @@ struct FlattenInfo {
   FlattenInfo(Loop *OL, Loop *IL) : OuterLoop(OL), InnerLoop(IL) {};
 };
 
-// Finds the induction variable, increment and limit for a simple loop that we
-// can flatten.
+// Finds the induction variable, increment and trip count for a simple loop that
+// we can flatten.
 static bool findLoopComponents(
     Loop *L, SmallPtrSetImpl<Instruction *> &IterationInstructions,
-    PHINode *&InductionPHI, Value *&Limit, BinaryOperator *&Increment,
-    BranchInst *&BackBranch, ScalarEvolution *SE) {
+    PHINode *&InductionPHI, Value *&TripCount, BinaryOperator *&Increment,
+    BranchInst *&BackBranch, ScalarEvolution *SE, bool IsWidened) {
   LLVM_DEBUG(dbgs() << "Finding components of loop: " << L->getName() << "\n");
 
   if (!L->isLoopSimplifyForm()) {
@@ -104,6 +106,13 @@ static bool findLoopComponents(
     return false;
   }
 
+  // Currently, to simplify the implementation, the Loop induction variable must
+  // start at zero and increment with a step size of one.
+  if (!L->isCanonical(*SE)) {
+    LLVM_DEBUG(dbgs() << "Loop is not canonical\n");
+    return false;
+  }
+
   // There must be exactly one exiting block, and it must be the same at the
   // latch.
   BasicBlock *Latch = L->getLoopLatch();
@@ -111,33 +120,18 @@ static bool findLoopComponents(
     LLVM_DEBUG(dbgs() << "Exiting and latch block are different\n");
     return false;
   }
-  // Latch block must end in a conditional branch.
-  BackBranch = dyn_cast<BranchInst>(Latch->getTerminator());
-  if (!BackBranch || !BackBranch->isConditional()) {
-    LLVM_DEBUG(dbgs() << "Could not find back-branch\n");
-    return false;
-  }
-  IterationInstructions.insert(BackBranch);
-  LLVM_DEBUG(dbgs() << "Found back branch: "; BackBranch->dump());
-  bool ContinueOnTrue = L->contains(BackBranch->getSuccessor(0));
 
   // Find the induction PHI. If there is no induction PHI, we can't do the
   // transformation. TODO: could other variables trigger this? Do we have to
   // search for the best one?
-  InductionPHI = nullptr;
-  for (PHINode &PHI : L->getHeader()->phis()) {
-    InductionDescriptor ID;
-    if (InductionDescriptor::isInductionPHI(&PHI, L, SE, ID)) {
-      InductionPHI = &PHI;
-      LLVM_DEBUG(dbgs() << "Found induction PHI: "; InductionPHI->dump());
-      break;
-    }
-  }
+  InductionPHI = L->getInductionVariable(*SE);
   if (!InductionPHI) {
     LLVM_DEBUG(dbgs() << "Could not find induction PHI\n");
     return false;
   }
+  LLVM_DEBUG(dbgs() << "Found induction PHI: "; InductionPHI->dump());
 
+  bool ContinueOnTrue = L->contains(Latch->getTerminator()->getSuccessor(0));
   auto IsValidPredicate = [&](ICmpInst::Predicate Pred) {
     if (ContinueOnTrue)
       return Pred == CmpInst::ICMP_NE || Pred == CmpInst::ICMP_ULT;
@@ -145,53 +139,64 @@ static bool findLoopComponents(
       return Pred == CmpInst::ICMP_EQ;
   };
 
-  // Find Compare and make sure it is valid
-  ICmpInst *Compare = dyn_cast<ICmpInst>(BackBranch->getCondition());
+  // Find Compare and make sure it is valid. getLatchCmpInst checks that the
+  // back branch of the latch is conditional.
+  ICmpInst *Compare = L->getLatchCmpInst();
   if (!Compare || !IsValidPredicate(Compare->getUnsignedPredicate()) ||
       Compare->hasNUsesOrMore(2)) {
     LLVM_DEBUG(dbgs() << "Could not find valid comparison\n");
     return false;
   }
+  BackBranch = cast<BranchInst>(Latch->getTerminator());
+  IterationInstructions.insert(BackBranch);
+  LLVM_DEBUG(dbgs() << "Found back branch: "; BackBranch->dump());
   IterationInstructions.insert(Compare);
   LLVM_DEBUG(dbgs() << "Found comparison: "; Compare->dump());
 
-  // Find increment and limit from the compare
-  Increment = nullptr;
-  if (match(Compare->getOperand(0),
-            m_c_Add(m_Specific(InductionPHI), m_ConstantInt<1>()))) {
-    Increment = dyn_cast<BinaryOperator>(Compare->getOperand(0));
-    Limit = Compare->getOperand(1);
-  } else if (Compare->getUnsignedPredicate() == CmpInst::ICMP_NE &&
-             match(Compare->getOperand(1),
-                   m_c_Add(m_Specific(InductionPHI), m_ConstantInt<1>()))) {
-    Increment = dyn_cast<BinaryOperator>(Compare->getOperand(1));
-    Limit = Compare->getOperand(0);
-  }
-  if (!Increment || Increment->hasNUsesOrMore(3)) {
-    LLVM_DEBUG(dbgs() << "Cound not find valid increment\n");
+  // Find increment and trip count.
+  // There are exactly 2 incoming values to the induction phi; one from the
+  // pre-header and one from the latch. The incoming latch value is the
+  // increment variable.
+  Increment =
+      dyn_cast<BinaryOperator>(InductionPHI->getIncomingValueForBlock(Latch));
+  if (Increment->hasNUsesOrMore(3)) {
+    LLVM_DEBUG(dbgs() << "Could not find valid increment\n");
     return false;
   }
+  // The trip count is the RHS of the compare. If this doesn't match the trip
+  // count computed by SCEV then this is either because the trip count variable
+  // has been widened (then leave the trip count as it is), or because it is a
+  // constant and another transformation has changed the compare, e.g.
+  // icmp ult %inc, tripcount -> icmp ult %j, tripcount-1, then we don't flatten
+  // the loop (yet).
+  TripCount = Compare->getOperand(1);
+  const SCEV *SCEVTripCount =
+      SE->getTripCountFromExitCount(SE->getBackedgeTakenCount(L));
+  if (SE->getSCEV(TripCount) != SCEVTripCount) {
+    if (!IsWidened) {
+      LLVM_DEBUG(dbgs() << "Could not find valid trip count\n");
+      return false;
+    }
+    auto TripCountInst = dyn_cast<Instruction>(TripCount);
+    if (!TripCountInst) {
+      LLVM_DEBUG(dbgs() << "Could not find valid extended trip count\n");
+      return false;
+    }
+    if ((!isa<ZExtInst>(TripCountInst) && !isa<SExtInst>(TripCountInst)) ||
+        SE->getSCEV(TripCountInst->getOperand(0)) != SCEVTripCount) {
+      LLVM_DEBUG(dbgs() << "Could not find valid extended trip count\n");
+      return false;
+    }
+  }
   IterationInstructions.insert(Increment);
   LLVM_DEBUG(dbgs() << "Found increment: "; Increment->dump());
-  LLVM_DEBUG(dbgs() << "Found limit: "; Limit->dump());
-
-  assert(InductionPHI->getNumIncomingValues() == 2);
-  assert(InductionPHI->getIncomingValueForBlock(Latch) == Increment &&
-         "PHI value is not increment inst");
-
-  auto *CI = dyn_cast<ConstantInt>(
-      InductionPHI->getIncomingValueForBlock(L->getLoopPreheader()));
-  if (!CI || !CI->isZero()) {
-    LLVM_DEBUG(dbgs() << "PHI value is not zero: "; CI->dump());
-    return false;
-  }
+  LLVM_DEBUG(dbgs() << "Found trip count: "; TripCount->dump());
 
   LLVM_DEBUG(dbgs() << "Successfully found all loop components\n");
   return true;
 }
 
-static bool checkPHIs(struct FlattenInfo &FI,
-                      const TargetTransformInfo *TTI) {
+static bool checkPHIs(FlattenInfo &FI, const TargetTransformInfo *TTI) {
   // All PHIs in the inner and outer headers must either be:
   // - The induction PHI, which we are going to rewrite as one induction in
   //   the new loop. This is already checked by findLoopComponents.
@@ -272,7 +277,7 @@ static bool checkPHIs(struct FlattenInfo &FI,
 }
 
 static bool
-checkOuterLoopInsts(struct FlattenInfo &FI,
+checkOuterLoopInsts(FlattenInfo &FI,
                     SmallPtrSetImpl<Instruction *> &IterationInstructions,
                     const TargetTransformInfo *TTI) {
   // Check for instructions in the outer but not inner loop. If any of these
@@ -280,7 +285,7 @@ checkOuterLoopInsts(struct FlattenInfo &FI,
   // a significant amount of code here which can't be optimised out that it's
   // not profitable (as these instructions would get executed for each
   // iteration of the inner loop).
-  unsigned RepeatedInstrCost = 0;
+  InstructionCost RepeatedInstrCost = 0;
   for (auto *B : FI.OuterLoop->getBlocks()) {
     if (FI.InnerLoop->contains(B))
       continue;
@@ -308,9 +313,10 @@ checkOuterLoopInsts(struct FlattenInfo &FI,
       // Multiplies of the outer iteration variable and inner iteration
       // count will be optimised out.
       if (match(&I, m_c_Mul(m_Specific(FI.OuterInductionPHI),
-                            m_Specific(FI.InnerLimit))))
+                            m_Specific(FI.InnerTripCount))))
         continue;
-      int Cost = TTI->getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency);
+      InstructionCost Cost =
+          TTI->getUserCost(&I, TargetTransformInfo::TCK_SizeAndLatency);
       LLVM_DEBUG(dbgs() << "Cost " << Cost << ": "; I.dump());
       RepeatedInstrCost += Cost;
     }
@@ -329,19 +335,19 @@ checkOuterLoopInsts(struct FlattenInfo &FI,
   return true;
 }
 
-static bool checkIVUsers(struct FlattenInfo &FI) {
+static bool checkIVUsers(FlattenInfo &FI) {
   // We require all uses of both induction variables to match this pattern:
   //
-  //   (OuterPHI * InnerLimit) + InnerPHI
+  //   (OuterPHI * InnerTripCount) + InnerPHI
   //
   // Any uses of the induction variables not matching that pattern would
   // require a div/mod to reconstruct in the flattened loop, so the
   // transformation wouldn't be profitable.
 
-  Value *InnerLimit = FI.InnerLimit;
+  Value *InnerTripCount = FI.InnerTripCount;
   if (FI.Widened &&
-      (isa<SExtInst>(InnerLimit) || isa<ZExtInst>(InnerLimit)))
-    InnerLimit = cast<Instruction>(InnerLimit)->getOperand(0);
+      (isa<SExtInst>(InnerTripCount) || isa<ZExtInst>(InnerTripCount)))
+    InnerTripCount = cast<Instruction>(InnerTripCount)->getOperand(0);
 
   // Check that all uses of the inner loop's induction variable match the
   // expected pattern, recording the uses of the outer IV.
@@ -375,7 +381,7 @@ static bool checkIVUsers(struct FlattenInfo &FI) {
                             m_c_Mul(m_Trunc(m_Specific(FI.OuterInductionPHI)),
                             m_Value(MatchedItCount)));
 
-    if ((IsAdd || IsAddTrunc) && MatchedItCount == InnerLimit) {
+    if ((IsAdd || IsAddTrunc) && MatchedItCount == InnerTripCount) {
       LLVM_DEBUG(dbgs() << "Use is optimisable\n");
       ValidOuterPHIUses.insert(MatchedMul);
       FI.LinearIVUses.insert(U);
@@ -424,9 +430,9 @@ static bool checkIVUsers(struct FlattenInfo &FI) {
 }
 
 // Return an OverflowResult dependant on if overflow of the multiplication of
-// InnerLimit and OuterLimit can be assumed not to happen.
-static OverflowResult checkOverflow(struct FlattenInfo &FI,
-                                    DominatorTree *DT, AssumptionCache *AC) {
+// InnerTripCount and OuterTripCount can be assumed not to happen.
+static OverflowResult checkOverflow(FlattenInfo &FI, DominatorTree *DT,
+                                    AssumptionCache *AC) {
   Function *F = FI.OuterLoop->getHeader()->getParent();
   const DataLayout &DL = F->getParent()->getDataLayout();
 
@@ -437,7 +443,7 @@ static OverflowResult checkOverflow(struct FlattenInfo &FI,
   // Check if the multiply could not overflow due to known ranges of the
   // input values.
   OverflowResult OR = computeOverflowForUnsignedMul(
-      FI.InnerLimit, FI.OuterLimit, DL, AC,
+      FI.InnerTripCount, FI.OuterTripCount, DL, AC,
       FI.OuterLoop->getLoopPreheader()->getTerminator(), DT);
   if (OR != OverflowResult::MayOverflow)
     return OR;
@@ -464,25 +470,27 @@ static OverflowResult checkOverflow(struct FlattenInfo &FI,
   return OverflowResult::MayOverflow;
 }
 
-static bool CanFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
-                               LoopInfo *LI, ScalarEvolution *SE,
-                               AssumptionCache *AC, const TargetTransformInfo *TTI) {
+static bool CanFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
+                               ScalarEvolution *SE, AssumptionCache *AC,
+                               const TargetTransformInfo *TTI) {
   SmallPtrSet<Instruction *, 8> IterationInstructions;
-  if (!findLoopComponents(FI.InnerLoop, IterationInstructions, FI.InnerInductionPHI,
-                          FI.InnerLimit, FI.InnerIncrement, FI.InnerBranch, SE))
+  if (!findLoopComponents(FI.InnerLoop, IterationInstructions,
+                          FI.InnerInductionPHI, FI.InnerTripCount,
+                          FI.InnerIncrement, FI.InnerBranch, SE, FI.Widened))
     return false;
-  if (!findLoopComponents(FI.OuterLoop, IterationInstructions, FI.OuterInductionPHI,
-                          FI.OuterLimit, FI.OuterIncrement, FI.OuterBranch, SE))
+  if (!findLoopComponents(FI.OuterLoop, IterationInstructions,
+                          FI.OuterInductionPHI, FI.OuterTripCount,
+                          FI.OuterIncrement, FI.OuterBranch, SE, FI.Widened))
     return false;
 
-  // Both of the loop limit values must be invariant in the outer loop
+  // Both of the loop trip count values must be invariant in the outer loop
   // (non-instructions are all inherently invariant).
-  if (!FI.OuterLoop->isLoopInvariant(FI.InnerLimit)) {
-    LLVM_DEBUG(dbgs() << "inner loop limit not invariant\n");
+  if (!FI.OuterLoop->isLoopInvariant(FI.InnerTripCount)) {
+    LLVM_DEBUG(dbgs() << "inner loop trip count not invariant\n");
     return false;
   }
-  if (!FI.OuterLoop->isLoopInvariant(FI.OuterLimit)) {
-    LLVM_DEBUG(dbgs() << "outer loop limit not invariant\n");
+  if (!FI.OuterLoop->isLoopInvariant(FI.OuterTripCount)) {
+    LLVM_DEBUG(dbgs() << "outer loop trip count not invariant\n");
     return false;
   }
 
@@ -508,9 +516,8 @@ static bool CanFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
   return true;
 }
 
-static bool DoFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
-                              LoopInfo *LI, ScalarEvolution *SE,
-                              AssumptionCache *AC,
+static bool DoFlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
+                              ScalarEvolution *SE, AssumptionCache *AC,
                               const TargetTransformInfo *TTI) {
   Function *F = FI.OuterLoop->getHeader()->getParent();
   LLVM_DEBUG(dbgs() << "Checks all passed, doing the transformation\n");
@@ -523,9 +530,9 @@ static bool DoFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
     ORE.emit(Remark);
   }
 
-  Value *NewTripCount =
-      BinaryOperator::CreateMul(FI.InnerLimit, FI.OuterLimit, "flatten.tripcount",
-                                FI.OuterLoop->getLoopPreheader()->getTerminator());
+  Value *NewTripCount = BinaryOperator::CreateMul(
+      FI.InnerTripCount, FI.OuterTripCount, "flatten.tripcount",
+      FI.OuterLoop->getLoopPreheader()->getTerminator());
   LLVM_DEBUG(dbgs() << "Created new trip count in preheader: ";
              NewTripCount->dump());
 
@@ -571,9 +578,9 @@ static bool DoFlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
   return true;
 }
 
-static bool CanWidenIV(struct FlattenInfo &FI, DominatorTree *DT,
-                       LoopInfo *LI, ScalarEvolution *SE,
-                       AssumptionCache *AC, const TargetTransformInfo *TTI) {
+static bool CanWidenIV(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
+                       ScalarEvolution *SE, AssumptionCache *AC,
+                       const TargetTransformInfo *TTI) {
   if (!WidenIV) {
     LLVM_DEBUG(dbgs() << "Widening the IVs is disabled\n");
     return false;
@@ -589,7 +596,7 @@ static bool CanWidenIV(struct FlattenInfo &FI, DominatorTree *DT,
 
   // If both induction types are less than the maximum legal integer width,
   // promote both to the widest type available so we know calculating
-  // (OuterLimit * InnerLimit) as the new trip count is safe.
+  // (OuterTripCount * InnerTripCount) as the new trip count is safe.
   if (InnerType != OuterType ||
       InnerType->getScalarSizeInBits() >= MaxLegalSize ||
       MaxLegalType->getScalarSizeInBits() < InnerType->getScalarSizeInBits() * 2) {
@@ -602,28 +609,27 @@ static bool CanWidenIV(struct FlattenInfo &FI, DominatorTree *DT,
   SmallVector<WeakTrackingVH, 4> DeadInsts;
   WideIVs.push_back( {FI.InnerInductionPHI, MaxLegalType, false });
   WideIVs.push_back( {FI.OuterInductionPHI, MaxLegalType, false });
-  unsigned ElimExt;
-  unsigned Widened;
+  unsigned ElimExt = 0;
+  unsigned Widened = 0;
 
-  for (unsigned i = 0; i < WideIVs.size(); i++) {
-    PHINode *WidePhi = createWideIV(WideIVs[i], LI, SE, Rewriter, DT, DeadInsts,
+  for (const auto &WideIV : WideIVs) {
+    PHINode *WidePhi = createWideIV(WideIV, LI, SE, Rewriter, DT, DeadInsts,
                                     ElimExt, Widened, true /* HasGuards */,
                                     true /* UsePostIncrementRanges */);
     if (!WidePhi)
       return false;
     LLVM_DEBUG(dbgs() << "Created wide phi: "; WidePhi->dump());
-    LLVM_DEBUG(dbgs() << "Deleting old phi: "; WideIVs[i].NarrowIV->dump());
-    RecursivelyDeleteDeadPHINode(WideIVs[i].NarrowIV);
+    LLVM_DEBUG(dbgs() << "Deleting old phi: "; WideIV.NarrowIV->dump());
+    RecursivelyDeleteDeadPHINode(WideIV.NarrowIV);
   }
   // After widening, rediscover all the loop components.
-  assert(Widened && "Widenend IV expected");
+  assert(Widened && "Widened IV expected");
   FI.Widened = true;
   return CanFlattenLoopPair(FI, DT, LI, SE, AC, TTI);
 }
 
-static bool FlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
-                            LoopInfo *LI, ScalarEvolution *SE,
-                            AssumptionCache *AC,
+static bool FlattenLoopPair(FlattenInfo &FI, DominatorTree *DT, LoopInfo *LI,
+                            ScalarEvolution *SE, AssumptionCache *AC,
                             const TargetTransformInfo *TTI) {
   LLVM_DEBUG(
       dbgs() << "Loop flattening running on outer loop "
@@ -656,33 +662,35 @@ static bool FlattenLoopPair(struct FlattenInfo &FI, DominatorTree *DT,
   return DoFlattenLoopPair(FI, DT, LI, SE, AC, TTI);
 }
 
-bool Flatten(DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE,
+bool Flatten(LoopNest &LN, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE,
              AssumptionCache *AC, TargetTransformInfo *TTI) {
   bool Changed = false;
-  for (auto *InnerLoop : LI->getLoopsInPreorder()) {
+  for (Loop *InnerLoop : LN.getLoops()) {
     auto *OuterLoop = InnerLoop->getParentLoop();
     if (!OuterLoop)
       continue;
-    struct FlattenInfo FI(OuterLoop, InnerLoop);
+    FlattenInfo FI(OuterLoop, InnerLoop);
     Changed |= FlattenLoopPair(FI, DT, LI, SE, AC, TTI);
   }
   return Changed;
 }
 
-PreservedAnalyses LoopFlattenPass::run(Function &F,
-                                       FunctionAnalysisManager &AM) {
-  auto *DT = &AM.getResult<DominatorTreeAnalysis>(F);
-  auto *LI = &AM.getResult<LoopAnalysis>(F);
-  auto *SE = &AM.getResult<ScalarEvolutionAnalysis>(F);
-  auto *AC = &AM.getResult<AssumptionAnalysis>(F);
-  auto *TTI = &AM.getResult<TargetIRAnalysis>(F);
+PreservedAnalyses LoopFlattenPass::run(LoopNest &LN, LoopAnalysisManager &LAM,
+                                       LoopStandardAnalysisResults &AR,
+                                       LPMUpdater &U) {
+
+  bool Changed = false;
+
+  // The loop flattening pass requires loops to be
+  // in simplified form, and also needs LCSSA. Running
+  // this pass will simplify all loops that contain inner loops,
+  // regardless of whether anything ends up being flattened.
+  Changed |= Flatten(LN, &AR.DT, &AR.LI, &AR.SE, &AR.AC, &AR.TTI);
 
-  if (!Flatten(DT, LI, SE, AC, TTI))
+  if (!Changed)
     return PreservedAnalyses::all();
 
-  PreservedAnalyses PA;
-  PA.preserveSet<CFGAnalyses>();
-  return PA;
+  return PreservedAnalyses::none();
 }
 
 namespace {
@@ -724,5 +732,10 @@ bool LoopFlattenLegacyPass::runOnFunction(Function &F) {
   auto &TTIP = getAnalysis<TargetTransformInfoWrapperPass>();
   auto *TTI = &TTIP.getTTI(F);
   auto *AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
-  return Flatten(DT, LI, SE, AC, TTI);
+  bool Changed = false;
+  for (Loop *L : *LI) {
+    auto LN = LoopNest::getLoopNest(*L, *SE);
+    Changed |= Flatten(*LN, DT, LI, SE, AC, TTI);
+  }
+  return Changed;
 }
diff --git a/llvm/lib/Transforms/Scalar/LoopFuse.cpp b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
index b5f8dfa9aafb..ca19913e37ee 100644
--- a/llvm/lib/Transforms/Scalar/LoopFuse.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopFuse.cpp
@@ -99,6 +99,8 @@ STATISTIC(NonEmptyExitBlock, "Candidate has a non-empty exit block with "
 STATISTIC(NonEmptyGuardBlock, "Candidate has a non-empty guard block with "
                               "instructions that cannot be moved");
 STATISTIC(NotRotated, "Candidate is not rotated");
+STATISTIC(OnlySecondCandidateIsGuarded,
+          "The second candidate is guarded while the first one is not");
 
 enum FusionDependenceAnalysisChoice {
   FUSION_DEPENDENCE_ANALYSIS_SCEV,
@@ -370,11 +372,13 @@ private:
   bool reportInvalidCandidate(llvm::Statistic &Stat) const {
     using namespace ore;
     assert(L && Preheader && "Fusion candidate not initialized properly!");
+#if LLVM_ENABLE_STATS
     ++Stat;
     ORE.emit(OptimizationRemarkAnalysis(DEBUG_TYPE, Stat.getName(),
                                         L->getStartLoc(), Preheader)
              << "[" << Preheader->getParent()->getName() << "]: "
              << "Loop is not a candidate for fusion: " << Stat.getDesc());
+#endif
     return false;
   }
 };
@@ -891,6 +895,14 @@ private:
             continue;
           }
 
+          if (!FC0->GuardBranch && FC1->GuardBranch) {
+            LLVM_DEBUG(dbgs() << "The second candidate is guarded while the "
+                                 "first one is not. Not fusing.\n");
+            reportLoopFusion<OptimizationRemarkMissed>(
+                *FC0, *FC1, OnlySecondCandidateIsGuarded);
+            continue;
+          }
+
           // Ensure that FC0 and FC1 have identical guards.
           // If one (or both) are not guarded, this check is not necessary.
           if (FC0->GuardBranch && FC1->GuardBranch &&
@@ -1523,6 +1535,7 @@ private:
     assert(FC0.Preheader && FC1.Preheader &&
            "Expecting valid fusion candidates");
     using namespace ore;
+#if LLVM_ENABLE_STATS
     ++Stat;
     ORE.emit(RemarkKind(DEBUG_TYPE, Stat.getName(), FC0.L->getStartLoc(),
                         FC0.Preheader)
@@ -1530,6 +1543,7 @@ private:
              << "]: " << NV("Cand1", StringRef(FC0.Preheader->getName()))
              << " and " << NV("Cand2", StringRef(FC1.Preheader->getName()))
              << ": " << Stat.getDesc());
+#endif
   }
 
   /// Fuse two guarded fusion candidates, creating a new fused loop.
diff --git a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index 8064c02e2b39..3d60e205b002 100644
--- a/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -21,7 +21,7 @@
 // TODO List:
 //
 // Future loop memory idioms to recognize:
-//   memcmp, memmove, strlen, etc.
+//   memcmp, strlen, etc.
 // Future floating point idioms to recognize in -ffast-math mode:
 //   fpowi
 // Future integer operation idioms to recognize:
@@ -90,6 +90,7 @@
 #include "llvm/Support/Casting.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/InstructionCost.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BuildLibCalls.h"
@@ -108,9 +109,12 @@ using namespace llvm;
 
 STATISTIC(NumMemSet, "Number of memset's formed from loop stores");
 STATISTIC(NumMemCpy, "Number of memcpy's formed from loop load+stores");
+STATISTIC(NumMemMove, "Number of memmove's formed from loop load+stores");
 STATISTIC(
     NumShiftUntilBitTest,
     "Number of uncountable loops recognized as 'shift until bitttest' idiom");
+STATISTIC(NumShiftUntilZero,
+          "Number of uncountable loops recognized as 'shift until zero' idiom");
 
 bool DisableLIRP::All;
 static cl::opt<bool, true>
@@ -204,6 +208,13 @@ private:
   enum class ForMemset { No, Yes };
   bool processLoopStores(SmallVectorImpl<StoreInst *> &SL, const SCEV *BECount,
                          ForMemset For);
+
+  template <typename MemInst>
+  bool processLoopMemIntrinsic(
+      BasicBlock *BB,
+      bool (LoopIdiomRecognize::*Processor)(MemInst *, const SCEV *),
+      const SCEV *BECount);
+  bool processLoopMemCpy(MemCpyInst *MCI, const SCEV *BECount);
   bool processLoopMemSet(MemSetInst *MSI, const SCEV *BECount);
 
   bool processLoopStridedStore(Value *DestPtr, unsigned StoreSize,
@@ -213,6 +224,13 @@ private:
                                const SCEVAddRecExpr *Ev, const SCEV *BECount,
                                bool NegStride, bool IsLoopMemset = false);
   bool processLoopStoreOfLoopLoad(StoreInst *SI, const SCEV *BECount);
+  bool processLoopStoreOfLoopLoad(Value *DestPtr, Value *SourcePtr,
+                                  unsigned StoreSize, MaybeAlign StoreAlign,
+                                  MaybeAlign LoadAlign, Instruction *TheStore,
+                                  Instruction *TheLoad,
+                                  const SCEVAddRecExpr *StoreEv,
+                                  const SCEVAddRecExpr *LoadEv,
+                                  const SCEV *BECount);
   bool avoidLIRForMultiBlockLoop(bool IsMemset = false,
                                  bool IsLoopMemset = false);
 
@@ -233,6 +251,7 @@ private:
                                 bool IsCntPhiUsedOutsideLoop);
 
   bool recognizeShiftUntilBitTest();
+  bool recognizeShiftUntilZero();
 
   /// @}
 };
@@ -501,7 +520,6 @@ LoopIdiomRecognize::isLegalStore(StoreInst *SI) {
   // are stored.  A store of i32 0x01020304 can never be turned into a memset,
   // but it can be turned into memset_pattern if the target supports it.
   Value *SplatValue = isBytewiseValue(StoredVal, *DL);
-  Constant *PatternValue = nullptr;
 
   // Note: memset and memset_pattern on unordered-atomic is yet not supported
   bool UnorderedAtomic = SI->isUnordered() && !SI->isSimple();
@@ -514,10 +532,11 @@ LoopIdiomRecognize::isLegalStore(StoreInst *SI) {
       CurLoop->isLoopInvariant(SplatValue)) {
     // It looks like we can use SplatValue.
     return LegalStoreKind::Memset;
-  } else if (!UnorderedAtomic && HasMemsetPattern && !DisableLIRP::Memset &&
-             // Don't create memset_pattern16s with address spaces.
-             StorePtr->getType()->getPointerAddressSpace() == 0 &&
-             (PatternValue = getMemSetPatternValue(StoredVal, DL))) {
+  }
+  if (!UnorderedAtomic && HasMemsetPattern && !DisableLIRP::Memset &&
+      // Don't create memset_pattern16s with address spaces.
+      StorePtr->getType()->getPointerAddressSpace() == 0 &&
+      getMemSetPatternValue(StoredVal, DL)) {
     // It looks like we can use PatternValue!
     return LegalStoreKind::MemsetPattern;
   }
@@ -627,22 +646,10 @@ bool LoopIdiomRecognize::runOnLoopBlock(
   for (auto &SI : StoreRefsForMemcpy)
     MadeChange |= processLoopStoreOfLoopLoad(SI, BECount);
 
-  for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
-    Instruction *Inst = &*I++;
-    // Look for memset instructions, which may be optimized to a larger memset.
-    if (MemSetInst *MSI = dyn_cast<MemSetInst>(Inst)) {
-      WeakTrackingVH InstPtr(&*I);
-      if (!processLoopMemSet(MSI, BECount))
-        continue;
-      MadeChange = true;
-
-      // If processing the memset invalidated our iterator, start over from the
-      // top of the block.
-      if (!InstPtr)
-        I = BB->begin();
-      continue;
-    }
-  }
+  MadeChange |= processLoopMemIntrinsic<MemCpyInst>(
+      BB, &LoopIdiomRecognize::processLoopMemCpy, BECount);
+  MadeChange |= processLoopMemIntrinsic<MemSetInst>(
+      BB, &LoopIdiomRecognize::processLoopMemSet, BECount);
 
   return MadeChange;
 }
@@ -791,6 +798,100 @@ bool LoopIdiomRecognize::processLoopStores(SmallVectorImpl<StoreInst *> &SL,
   return Changed;
 }
 
+/// processLoopMemIntrinsic - Template function for calling different processor
+/// functions based on mem instrinsic type.
+template <typename MemInst>
+bool LoopIdiomRecognize::processLoopMemIntrinsic(
+    BasicBlock *BB,
+    bool (LoopIdiomRecognize::*Processor)(MemInst *, const SCEV *),
+    const SCEV *BECount) {
+  bool MadeChange = false;
+  for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E;) {
+    Instruction *Inst = &*I++;
+    // Look for memory instructions, which may be optimized to a larger one.
+    if (MemInst *MI = dyn_cast<MemInst>(Inst)) {
+      WeakTrackingVH InstPtr(&*I);
+      if (!(this->*Processor)(MI, BECount))
+        continue;
+      MadeChange = true;
+
+      // If processing the instruction invalidated our iterator, start over from
+      // the top of the block.
+      if (!InstPtr)
+        I = BB->begin();
+    }
+  }
+  return MadeChange;
+}
+
+/// processLoopMemCpy - See if this memcpy can be promoted to a large memcpy
+bool LoopIdiomRecognize::processLoopMemCpy(MemCpyInst *MCI,
+                                           const SCEV *BECount) {
+  // We can only handle non-volatile memcpys with a constant size.
+  if (MCI->isVolatile() || !isa<ConstantInt>(MCI->getLength()))
+    return false;
+
+  // If we're not allowed to hack on memcpy, we fail.
+  if ((!HasMemcpy && !isa<MemCpyInlineInst>(MCI)) || DisableLIRP::Memcpy)
+    return false;
+
+  Value *Dest = MCI->getDest();
+  Value *Source = MCI->getSource();
+  if (!Dest || !Source)
+    return false;
+
+  // See if the load and store pointer expressions are AddRec like {base,+,1} on
+  // the current loop, which indicates a strided load and store.  If we have
+  // something else, it's a random load or store we can't handle.
+  const SCEVAddRecExpr *StoreEv = dyn_cast<SCEVAddRecExpr>(SE->getSCEV(Dest));
+  if (!StoreEv || StoreEv->getLoop() != CurLoop || !StoreEv->isAffine())
+    return false;
+  const SCEVAddRecExpr *LoadEv = dyn_cast<SCEVAddRecExpr>(SE->getSCEV(Source));
+  if (!LoadEv || LoadEv->getLoop() != CurLoop || !LoadEv->isAffine())
+    return false;
+
+  // Reject memcpys that are so large that they overflow an unsigned.
+  uint64_t SizeInBytes = cast<ConstantInt>(MCI->getLength())->getZExtValue();
+  if ((SizeInBytes >> 32) != 0)
+    return false;
+
+  // Check if the stride matches the size of the memcpy. If so, then we know
+  // that every byte is touched in the loop.
+  const SCEVConstant *StoreStride =
+      dyn_cast<SCEVConstant>(StoreEv->getOperand(1));
+  const SCEVConstant *LoadStride =
+      dyn_cast<SCEVConstant>(LoadEv->getOperand(1));
+  if (!StoreStride || !LoadStride)
+    return false;
+
+  APInt StoreStrideValue = StoreStride->getAPInt();
+  APInt LoadStrideValue = LoadStride->getAPInt();
+  // Huge stride value - give up
+  if (StoreStrideValue.getBitWidth() > 64 || LoadStrideValue.getBitWidth() > 64)
+    return false;
+
+  if (SizeInBytes != StoreStrideValue && SizeInBytes != -StoreStrideValue) {
+    ORE.emit([&]() {
+      return OptimizationRemarkMissed(DEBUG_TYPE, "SizeStrideUnequal", MCI)
+             << ore::NV("Inst", "memcpy") << " in "
+             << ore::NV("Function", MCI->getFunction())
+             << " function will not be hoised: "
+             << ore::NV("Reason", "memcpy size is not equal to stride");
+    });
+    return false;
+  }
+
+  int64_t StoreStrideInt = StoreStrideValue.getSExtValue();
+  int64_t LoadStrideInt = LoadStrideValue.getSExtValue();
+  // Check if the load stride matches the store stride.
+  if (StoreStrideInt != LoadStrideInt)
+    return false;
+
+  return processLoopStoreOfLoopLoad(Dest, Source, (unsigned)SizeInBytes,
+                                    MCI->getDestAlign(), MCI->getSourceAlign(),
+                                    MCI, MCI, StoreEv, LoadEv, BECount);
+}
+
 /// processLoopMemSet - See if this memset can be promoted to a large memset.
 bool LoopIdiomRecognize::processLoopMemSet(MemSetInst *MSI,
                                            const SCEV *BECount) {
@@ -799,7 +900,7 @@ bool LoopIdiomRecognize::processLoopMemSet(MemSetInst *MSI,
     return false;
 
   // If we're not allowed to hack on memset, we fail.
-  if (!HasMemset)
+  if (!HasMemset || DisableLIRP::Memset)
     return false;
 
   Value *Pointer = MSI->getDest();
@@ -1039,9 +1140,11 @@ bool LoopIdiomRecognize::processLoopStridedStore(
   ORE.emit([&]() {
     return OptimizationRemark(DEBUG_TYPE, "ProcessLoopStridedStore",
                               NewCall->getDebugLoc(), Preheader)
-           << "Transformed loop-strided store into a call to "
+           << "Transformed loop-strided store in "
+           << ore::NV("Function", TheStore->getFunction())
+           << " function into a call to "
            << ore::NV("NewFunction", NewCall->getCalledFunction())
-           << "() function";
+           << "() intrinsic";
   });
 
   // Okay, the memset has been formed.  Zap the original store and anything that
@@ -1067,9 +1170,7 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
 
   Value *StorePtr = SI->getPointerOperand();
   const SCEVAddRecExpr *StoreEv = cast<SCEVAddRecExpr>(SE->getSCEV(StorePtr));
-  APInt Stride = getStoreStride(StoreEv);
   unsigned StoreSize = DL->getTypeStoreSize(SI->getValueOperand()->getType());
-  bool NegStride = StoreSize == -Stride;
 
   // The store must be feeding a non-volatile load.
   LoadInst *LI = cast<LoadInst>(SI->getValueOperand());
@@ -1078,8 +1179,24 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
   // See if the pointer expression is an AddRec like {base,+,1} on the current
   // loop, which indicates a strided load.  If we have something else, it's a
   // random load we can't handle.
-  const SCEVAddRecExpr *LoadEv =
-      cast<SCEVAddRecExpr>(SE->getSCEV(LI->getPointerOperand()));
+  Value *LoadPtr = LI->getPointerOperand();
+  const SCEVAddRecExpr *LoadEv = cast<SCEVAddRecExpr>(SE->getSCEV(LoadPtr));
+  return processLoopStoreOfLoopLoad(StorePtr, LoadPtr, StoreSize,
+                                    SI->getAlign(), LI->getAlign(), SI, LI,
+                                    StoreEv, LoadEv, BECount);
+}
+
+bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(
+    Value *DestPtr, Value *SourcePtr, unsigned StoreSize, MaybeAlign StoreAlign,
+    MaybeAlign LoadAlign, Instruction *TheStore, Instruction *TheLoad,
+    const SCEVAddRecExpr *StoreEv, const SCEVAddRecExpr *LoadEv,
+    const SCEV *BECount) {
+
+  // FIXME: until llvm.memcpy.inline supports dynamic sizes, we need to
+  // conservatively bail here, since otherwise we may have to transform
+  // llvm.memcpy.inline into llvm.memcpy which is illegal.
+  if (isa<MemCpyInlineInst>(TheStore))
+    return false;
 
   // The trip count of the loop and the base pointer of the addrec SCEV is
   // guaranteed to be loop invariant, which means that it should dominate the
@@ -1092,9 +1209,12 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
 
   bool Changed = false;
   const SCEV *StrStart = StoreEv->getStart();
-  unsigned StrAS = SI->getPointerAddressSpace();
+  unsigned StrAS = DestPtr->getType()->getPointerAddressSpace();
   Type *IntIdxTy = Builder.getIntNTy(DL->getIndexSizeInBits(StrAS));
 
+  APInt Stride = getStoreStride(StoreEv);
+  bool NegStride = StoreSize == -Stride;
+
   // Handle negative strided loops.
   if (NegStride)
     StrStart = getStartForNegStride(StrStart, BECount, IntIdxTy, StoreSize, SE);
@@ -1117,14 +1237,34 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
   // the return value will read this comment, and leave them alone.
   Changed = true;
 
-  SmallPtrSet<Instruction *, 1> Stores;
-  Stores.insert(SI);
-  if (mayLoopAccessLocation(StoreBasePtr, ModRefInfo::ModRef, CurLoop, BECount,
-                            StoreSize, *AA, Stores))
-    return Changed;
+  SmallPtrSet<Instruction *, 2> Stores;
+  Stores.insert(TheStore);
+
+  bool IsMemCpy = isa<MemCpyInst>(TheStore);
+  const StringRef InstRemark = IsMemCpy ? "memcpy" : "load and store";
+
+  bool UseMemMove =
+      mayLoopAccessLocation(StoreBasePtr, ModRefInfo::ModRef, CurLoop, BECount,
+                            StoreSize, *AA, Stores);
+  if (UseMemMove) {
+    Stores.insert(TheLoad);
+    if (mayLoopAccessLocation(StoreBasePtr, ModRefInfo::ModRef, CurLoop,
+                              BECount, StoreSize, *AA, Stores)) {
+      ORE.emit([&]() {
+        return OptimizationRemarkMissed(DEBUG_TYPE, "LoopMayAccessStore",
+                                        TheStore)
+               << ore::NV("Inst", InstRemark) << " in "
+               << ore::NV("Function", TheStore->getFunction())
+               << " function will not be hoisted: "
+               << ore::NV("Reason", "The loop may access store location");
+      });
+      return Changed;
+    }
+    Stores.erase(TheLoad);
+  }
 
   const SCEV *LdStart = LoadEv->getStart();
-  unsigned LdAS = LI->getPointerAddressSpace();
+  unsigned LdAS = SourcePtr->getType()->getPointerAddressSpace();
 
   // Handle negative strided loops.
   if (NegStride)
@@ -1135,9 +1275,37 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
   Value *LoadBasePtr = Expander.expandCodeFor(
       LdStart, Builder.getInt8PtrTy(LdAS), Preheader->getTerminator());
 
+  // If the store is a memcpy instruction, we must check if it will write to
+  // the load memory locations. So remove it from the ignored stores.
+  if (IsMemCpy)
+    Stores.erase(TheStore);
   if (mayLoopAccessLocation(LoadBasePtr, ModRefInfo::Mod, CurLoop, BECount,
-                            StoreSize, *AA, Stores))
+                            StoreSize, *AA, Stores)) {
+    ORE.emit([&]() {
+      return OptimizationRemarkMissed(DEBUG_TYPE, "LoopMayAccessLoad", TheLoad)
+             << ore::NV("Inst", InstRemark) << " in "
+             << ore::NV("Function", TheStore->getFunction())
+             << " function will not be hoisted: "
+             << ore::NV("Reason", "The loop may access load location");
+    });
     return Changed;
+  }
+  if (UseMemMove) {
+    // Ensure that LoadBasePtr is after StoreBasePtr or before StoreBasePtr for
+    // negative stride. LoadBasePtr shouldn't overlap with StoreBasePtr.
+    int64_t LoadOff = 0, StoreOff = 0;
+    const Value *BP1 = llvm::GetPointerBaseWithConstantOffset(
+        LoadBasePtr->stripPointerCasts(), LoadOff, *DL);
+    const Value *BP2 = llvm::GetPointerBaseWithConstantOffset(
+        StoreBasePtr->stripPointerCasts(), StoreOff, *DL);
+    int64_t LoadSize =
+        DL->getTypeSizeInBits(TheLoad->getType()).getFixedSize() / 8;
+    if (BP1 != BP2 || LoadSize != int64_t(StoreSize))
+      return Changed;
+    if ((!NegStride && LoadOff < StoreOff + int64_t(StoreSize)) ||
+        (NegStride && LoadOff + LoadSize > StoreOff))
+      return Changed;
+  }
 
   if (avoidLIRForMultiBlockLoop())
     return Changed;
@@ -1154,15 +1322,22 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
   // Check whether to generate an unordered atomic memcpy:
   //  If the load or store are atomic, then they must necessarily be unordered
   //  by previous checks.
-  if (!SI->isAtomic() && !LI->isAtomic())
-    NewCall = Builder.CreateMemCpy(StoreBasePtr, SI->getAlign(), LoadBasePtr,
-                                   LI->getAlign(), NumBytes);
-  else {
+  if (!TheStore->isAtomic() && !TheLoad->isAtomic()) {
+    if (UseMemMove)
+      NewCall = Builder.CreateMemMove(StoreBasePtr, StoreAlign, LoadBasePtr,
+                                      LoadAlign, NumBytes);
+    else
+      NewCall = Builder.CreateMemCpy(StoreBasePtr, StoreAlign, LoadBasePtr,
+                                     LoadAlign, NumBytes);
+  } else {
+    // For now don't support unordered atomic memmove.
+    if (UseMemMove)
+      return Changed;
     // We cannot allow unaligned ops for unordered load/store, so reject
     // anything where the alignment isn't at least the element size.
-    const Align StoreAlign = SI->getAlign();
-    const Align LoadAlign = LI->getAlign();
-    if (StoreAlign < StoreSize || LoadAlign < StoreSize)
+    assert((StoreAlign.hasValue() && LoadAlign.hasValue()) &&
+           "Expect unordered load/store to have align.");
+    if (StoreAlign.getValue() < StoreSize || LoadAlign.getValue() < StoreSize)
       return Changed;
 
     // If the element.atomic memcpy is not lowered into explicit
@@ -1176,10 +1351,10 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
     // Note that unordered atomic loads/stores are *required* by the spec to
     // have an alignment but non-atomic loads/stores may not.
     NewCall = Builder.CreateElementUnorderedAtomicMemCpy(
-        StoreBasePtr, StoreAlign, LoadBasePtr, LoadAlign, NumBytes,
-        StoreSize);
+        StoreBasePtr, StoreAlign.getValue(), LoadBasePtr, LoadAlign.getValue(),
+        NumBytes, StoreSize);
   }
-  NewCall->setDebugLoc(SI->getDebugLoc());
+  NewCall->setDebugLoc(TheStore->getDebugLoc());
 
   if (MSSAU) {
     MemoryAccess *NewMemAcc = MSSAU->createMemoryAccessInBB(
@@ -1187,9 +1362,10 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
     MSSAU->insertDef(cast<MemoryDef>(NewMemAcc), true);
   }
 
-  LLVM_DEBUG(dbgs() << "  Formed memcpy: " << *NewCall << "\n"
-                    << "    from load ptr=" << *LoadEv << " at: " << *LI << "\n"
-                    << "    from store ptr=" << *StoreEv << " at: " << *SI
+  LLVM_DEBUG(dbgs() << "  Formed new call: " << *NewCall << "\n"
+                    << "    from load ptr=" << *LoadEv << " at: " << *TheLoad
+                    << "\n"
+                    << "    from store ptr=" << *StoreEv << " at: " << *TheStore
                     << "\n");
 
   ORE.emit([&]() {
@@ -1197,17 +1373,22 @@ bool LoopIdiomRecognize::processLoopStoreOfLoopLoad(StoreInst *SI,
                               NewCall->getDebugLoc(), Preheader)
            << "Formed a call to "
            << ore::NV("NewFunction", NewCall->getCalledFunction())
-           << "() function";
+           << "() intrinsic from " << ore::NV("Inst", InstRemark)
+           << " instruction in " << ore::NV("Function", TheStore->getFunction())
+           << " function";
   });
 
   // Okay, the memcpy has been formed.  Zap the original store and anything that
   // feeds into it.
   if (MSSAU)
-    MSSAU->removeMemoryAccess(SI, true);
-  deleteDeadInstruction(SI);
+    MSSAU->removeMemoryAccess(TheStore, true);
+  deleteDeadInstruction(TheStore);
   if (MSSAU && VerifyMemorySSA)
     MSSAU->getMemorySSA()->verifyMemorySSA();
-  ++NumMemCpy;
+  if (UseMemMove)
+    ++NumMemMove;
+  else
+    ++NumMemCpy;
   ExpCleaner.markResultUsed();
   return true;
 }
@@ -1236,7 +1417,7 @@ bool LoopIdiomRecognize::runOnNoncountableLoop() {
                     << CurLoop->getHeader()->getName() << "\n");
 
   return recognizePopcount() || recognizeAndInsertFFS() ||
-         recognizeShiftUntilBitTest();
+         recognizeShiftUntilBitTest() || recognizeShiftUntilZero();
 }
 
 /// Check if the given conditional branch is based on the comparison between
@@ -1587,9 +1768,8 @@ bool LoopIdiomRecognize::recognizeAndInsertFFS() {
   //  %inc = add nsw %i.0, 1
   //  br i1 %tobool
 
-  const Value *Args[] = {
-      InitX, ZeroCheck ? ConstantInt::getTrue(InitX->getContext())
-                       : ConstantInt::getFalse(InitX->getContext())};
+  const Value *Args[] = {InitX,
+                         ConstantInt::getBool(InitX->getContext(), ZeroCheck)};
 
   // @llvm.dbg doesn't count as they have no semantic effect.
   auto InstWithoutDebugIt = CurLoop->getHeader()->instructionsWithoutDebug();
@@ -1597,7 +1777,7 @@ bool LoopIdiomRecognize::recognizeAndInsertFFS() {
       std::distance(InstWithoutDebugIt.begin(), InstWithoutDebugIt.end());
 
   IntrinsicCostAttributes Attrs(IntrinID, InitX->getType(), Args);
-  int Cost =
+  InstructionCost Cost =
     TTI->getIntrinsicInstrCost(Attrs, TargetTransformInfo::TCK_SizeAndLatency);
   if (HeaderSize != IdiomCanonicalSize &&
       Cost > TargetTransformInfo::TCC_Basic)
@@ -1675,7 +1855,7 @@ static CallInst *createPopcntIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
 static CallInst *createFFSIntrinsic(IRBuilder<> &IRBuilder, Value *Val,
                                     const DebugLoc &DL, bool ZeroCheck,
                                     Intrinsic::ID IID) {
-  Value *Ops[] = {Val, ZeroCheck ? IRBuilder.getTrue() : IRBuilder.getFalse()};
+  Value *Ops[] = {Val, IRBuilder.getInt1(ZeroCheck)};
   Type *Tys[] = {Val->getType()};
 
   Module *M = IRBuilder.GetInsertBlock()->getParent()->getParent();
@@ -1727,6 +1907,7 @@ void LoopIdiomRecognize::transformLoopToCountable(
   IRBuilder<> Builder(PreheaderBr);
   Builder.SetCurrentDebugLocation(DL);
 
+  // If there are no uses of CntPhi crate:
   //   Count = BitWidth - CTLZ(InitX);
   //   NewCount = Count;
   // If there are uses of CntPhi create:
@@ -1735,30 +1916,25 @@ void LoopIdiomRecognize::transformLoopToCountable(
   Value *InitXNext;
   if (IsCntPhiUsedOutsideLoop) {
     if (DefX->getOpcode() == Instruction::AShr)
-      InitXNext =
-          Builder.CreateAShr(InitX, ConstantInt::get(InitX->getType(), 1));
+      InitXNext = Builder.CreateAShr(InitX, 1);
     else if (DefX->getOpcode() == Instruction::LShr)
-      InitXNext =
-          Builder.CreateLShr(InitX, ConstantInt::get(InitX->getType(), 1));
+      InitXNext = Builder.CreateLShr(InitX, 1);
     else if (DefX->getOpcode() == Instruction::Shl) // cttz
-      InitXNext =
-          Builder.CreateShl(InitX, ConstantInt::get(InitX->getType(), 1));
+      InitXNext = Builder.CreateShl(InitX, 1);
     else
       llvm_unreachable("Unexpected opcode!");
   } else
     InitXNext = InitX;
-  Value *FFS = createFFSIntrinsic(Builder, InitXNext, DL, ZeroCheck, IntrinID);
-  Value *Count = Builder.CreateSub(
-      ConstantInt::get(FFS->getType(), FFS->getType()->getIntegerBitWidth()),
-      FFS);
+  Value *Count =
+      createFFSIntrinsic(Builder, InitXNext, DL, ZeroCheck, IntrinID);
+  Type *CountTy = Count->getType();
+  Count = Builder.CreateSub(
+      ConstantInt::get(CountTy, CountTy->getIntegerBitWidth()), Count);
   Value *NewCount = Count;
-  if (IsCntPhiUsedOutsideLoop) {
-    NewCount = Count;
-    Count = Builder.CreateAdd(Count, ConstantInt::get(Count->getType(), 1));
-  }
+  if (IsCntPhiUsedOutsideLoop)
+    Count = Builder.CreateAdd(Count, ConstantInt::get(CountTy, 1));
 
-  NewCount = Builder.CreateZExtOrTrunc(NewCount,
-                                       cast<IntegerType>(CntInst->getType()));
+  NewCount = Builder.CreateZExtOrTrunc(NewCount, CntInst->getType());
 
   Value *CntInitVal = CntPhi->getIncomingValueForBlock(Preheader);
   if (cast<ConstantInt>(CntInst->getOperand(1))->isOne()) {
@@ -1784,14 +1960,12 @@ void LoopIdiomRecognize::transformLoopToCountable(
   BasicBlock *Body = *(CurLoop->block_begin());
   auto *LbBr = cast<BranchInst>(Body->getTerminator());
   ICmpInst *LbCond = cast<ICmpInst>(LbBr->getCondition());
-  Type *Ty = Count->getType();
 
-  PHINode *TcPhi = PHINode::Create(Ty, 2, "tcphi", &Body->front());
+  PHINode *TcPhi = PHINode::Create(CountTy, 2, "tcphi", &Body->front());
 
   Builder.SetInsertPoint(LbCond);
-  Instruction *TcDec = cast<Instruction>(
-      Builder.CreateSub(TcPhi, ConstantInt::get(Ty, 1),
-                        "tcdec", false, true));
+  Instruction *TcDec = cast<Instruction>(Builder.CreateSub(
+      TcPhi, ConstantInt::get(CountTy, 1), "tcdec", false, true));
 
   TcPhi->addIncoming(Count, Preheader);
   TcPhi->addIncoming(TcDec, Body);
@@ -1800,7 +1974,7 @@ void LoopIdiomRecognize::transformLoopToCountable(
       (LbBr->getSuccessor(0) == Body) ? CmpInst::ICMP_NE : CmpInst::ICMP_EQ;
   LbCond->setPredicate(Pred);
   LbCond->setOperand(0, TcDec);
-  LbCond->setOperand(1, ConstantInt::get(Ty, 0));
+  LbCond->setOperand(1, ConstantInt::get(CountTy, 0));
 
   // Step 3: All the references to the original counter outside
   //  the loop are replaced with the NewCount
@@ -2033,6 +2207,9 @@ static bool detectShiftUntilBitTestIdiom(Loop *CurLoop, Value *&BaseX,
   NextX =
       dyn_cast<Instruction>(CurrXPN->getIncomingValueForBlock(LoopHeaderBB));
 
+  assert(CurLoop->isLoopInvariant(BaseX) &&
+         "Expected BaseX to be avaliable in the preheader!");
+
   if (!NextX || !match(NextX, m_Shl(m_Specific(CurrX), m_One()))) {
     // FIXME: support right-shift?
     LLVM_DEBUG(dbgs() << DEBUG_TYPE " Bad recurrence.\n");
@@ -2132,13 +2309,14 @@ bool LoopIdiomRecognize::recognizeShiftUntilBitTest() {
   assert(LoopPreheaderBB && "There is always a loop preheader.");
 
   BasicBlock *SuccessorBB = CurLoop->getExitBlock();
-  assert(LoopPreheaderBB && "There is only a single successor.");
+  assert(SuccessorBB && "There is only a single successor.");
 
   IRBuilder<> Builder(LoopPreheaderBB->getTerminator());
   Builder.SetCurrentDebugLocation(cast<Instruction>(XCurr)->getDebugLoc());
 
   Intrinsic::ID IntrID = Intrinsic::ctlz;
   Type *Ty = X->getType();
+  unsigned Bitwidth = Ty->getScalarSizeInBits();
 
   TargetTransformInfo::TargetCostKind CostKind =
       TargetTransformInfo::TCK_SizeAndLatency;
@@ -2148,7 +2326,7 @@ bool LoopIdiomRecognize::recognizeShiftUntilBitTest() {
   // making the loop countable, even if nothing else changes.
   IntrinsicCostAttributes Attrs(
       IntrID, Ty, {UndefValue::get(Ty), /*is_zero_undef=*/Builder.getTrue()});
-  int Cost = TTI->getIntrinsicInstrCost(Attrs, CostKind);
+  InstructionCost Cost = TTI->getIntrinsicInstrCost(Attrs, CostKind);
   if (Cost > TargetTransformInfo::TCC_Basic) {
     LLVM_DEBUG(dbgs() << DEBUG_TYPE
                " Intrinsic is too costly, not beneficial\n");
@@ -2175,18 +2353,22 @@ bool LoopIdiomRecognize::recognizeShiftUntilBitTest() {
       /*FMFSource=*/nullptr, XMasked->getName() + ".numleadingzeros");
   Value *XMaskedNumActiveBits = Builder.CreateSub(
       ConstantInt::get(Ty, Ty->getScalarSizeInBits()), XMaskedNumLeadingZeros,
-      XMasked->getName() + ".numactivebits");
+      XMasked->getName() + ".numactivebits", /*HasNUW=*/true,
+      /*HasNSW=*/Bitwidth != 2);
   Value *XMaskedLeadingOnePos =
       Builder.CreateAdd(XMaskedNumActiveBits, Constant::getAllOnesValue(Ty),
-                        XMasked->getName() + ".leadingonepos");
+                        XMasked->getName() + ".leadingonepos", /*HasNUW=*/false,
+                        /*HasNSW=*/Bitwidth > 2);
 
   Value *LoopBackedgeTakenCount = Builder.CreateSub(
-      BitPos, XMaskedLeadingOnePos, CurLoop->getName() + ".backedgetakencount");
+      BitPos, XMaskedLeadingOnePos, CurLoop->getName() + ".backedgetakencount",
+      /*HasNUW=*/true, /*HasNSW=*/true);
   // We know loop's backedge-taken count, but what's loop's trip count?
   // Note that while NUW is always safe, while NSW is only for bitwidths != 2.
   Value *LoopTripCount =
-      Builder.CreateNUWAdd(LoopBackedgeTakenCount, ConstantInt::get(Ty, 1),
-                           CurLoop->getName() + ".tripcount");
+      Builder.CreateAdd(LoopBackedgeTakenCount, ConstantInt::get(Ty, 1),
+                        CurLoop->getName() + ".tripcount", /*HasNUW=*/true,
+                        /*HasNSW=*/Bitwidth != 2);
 
   // Step 2: Compute the recurrence's final value without a loop.
 
@@ -2234,8 +2416,9 @@ bool LoopIdiomRecognize::recognizeShiftUntilBitTest() {
   // The induction itself.
   // Note that while NUW is always safe, while NSW is only for bitwidths != 2.
   Builder.SetInsertPoint(LoopHeaderBB->getTerminator());
-  auto *IVNext = Builder.CreateNUWAdd(IV, ConstantInt::get(Ty, 1),
-                                      IV->getName() + ".next");
+  auto *IVNext =
+      Builder.CreateAdd(IV, ConstantInt::get(Ty, 1), IV->getName() + ".next",
+                        /*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
 
   // The loop trip count check.
   auto *IVCheck = Builder.CreateICmpEQ(IVNext, LoopTripCount,
@@ -2259,3 +2442,346 @@ bool LoopIdiomRecognize::recognizeShiftUntilBitTest() {
   ++NumShiftUntilBitTest;
   return MadeChange;
 }
+
+/// Return true if the idiom is detected in the loop.
+///
+/// The core idiom we are trying to detect is:
+/// \code
+///   entry:
+///     <...>
+///     %start = <...>
+///     %extraoffset = <...>
+///     <...>
+///     br label %for.cond
+///
+///   loop:
+///     %iv = phi i8 [ %start, %entry ], [ %iv.next, %for.cond ]
+///     %nbits = add nsw i8 %iv, %extraoffset
+///     %val.shifted = {{l,a}shr,shl} i8 %val, %nbits
+///     %val.shifted.iszero = icmp eq i8 %val.shifted, 0
+///     %iv.next = add i8 %iv, 1
+///     <...>
+///     br i1 %val.shifted.iszero, label %end, label %loop
+///
+///   end:
+///     %iv.res = phi i8 [ %iv, %loop ] <...>
+///     %nbits.res = phi i8 [ %nbits, %loop ] <...>
+///     %val.shifted.res = phi i8 [ %val.shifted, %loop ] <...>
+///     %val.shifted.iszero.res = phi i1 [ %val.shifted.iszero, %loop ] <...>
+///     %iv.next.res = phi i8 [ %iv.next, %loop ] <...>
+///     <...>
+/// \endcode
+static bool detectShiftUntilZeroIdiom(Loop *CurLoop, ScalarEvolution *SE,
+                                      Instruction *&ValShiftedIsZero,
+                                      Intrinsic::ID &IntrinID, Instruction *&IV,
+                                      Value *&Start, Value *&Val,
+                                      const SCEV *&ExtraOffsetExpr,
+                                      bool &InvertedCond) {
+  LLVM_DEBUG(dbgs() << DEBUG_TYPE
+             " Performing shift-until-zero idiom detection.\n");
+
+  // Give up if the loop has multiple blocks or multiple backedges.
+  if (CurLoop->getNumBlocks() != 1 || CurLoop->getNumBackEdges() != 1) {
+    LLVM_DEBUG(dbgs() << DEBUG_TYPE " Bad block/backedge count.\n");
+    return false;
+  }
+
+  Instruction *ValShifted, *NBits, *IVNext;
+  Value *ExtraOffset;
+
+  BasicBlock *LoopHeaderBB = CurLoop->getHeader();
+  BasicBlock *LoopPreheaderBB = CurLoop->getLoopPreheader();
+  assert(LoopPreheaderBB && "There is always a loop preheader.");
+
+  using namespace PatternMatch;
+
+  // Step 1: Check if the loop backedge, condition is in desirable form.
+
+  ICmpInst::Predicate Pred;
+  BasicBlock *TrueBB, *FalseBB;
+  if (!match(LoopHeaderBB->getTerminator(),
+             m_Br(m_Instruction(ValShiftedIsZero), m_BasicBlock(TrueBB),
+                  m_BasicBlock(FalseBB))) ||
+      !match(ValShiftedIsZero,
+             m_ICmp(Pred, m_Instruction(ValShifted), m_Zero())) ||
+      !ICmpInst::isEquality(Pred)) {
+    LLVM_DEBUG(dbgs() << DEBUG_TYPE " Bad backedge structure.\n");
+    return false;
+  }
+
+  // Step 2: Check if the comparison's operand is in desirable form.
+  // FIXME: Val could be a one-input PHI node, which we should look past.
+  if (!match(ValShifted, m_Shift(m_LoopInvariant(m_Value(Val), CurLoop),
+                                 m_Instruction(NBits)))) {
+    LLVM_DEBUG(dbgs() << DEBUG_TYPE " Bad comparisons value computation.\n");
+    return false;
+  }
+  IntrinID = ValShifted->getOpcode() == Instruction::Shl ? Intrinsic::cttz
+                                                         : Intrinsic::ctlz;
+
+  // Step 3: Check if the shift amount is in desirable form.
+
+  if (match(NBits, m_c_Add(m_Instruction(IV),
+                           m_LoopInvariant(m_Value(ExtraOffset), CurLoop))) &&
+      (NBits->hasNoSignedWrap() || NBits->hasNoUnsignedWrap()))
+    ExtraOffsetExpr = SE->getNegativeSCEV(SE->getSCEV(ExtraOffset));
+  else if (match(NBits,
+                 m_Sub(m_Instruction(IV),
+                       m_LoopInvariant(m_Value(ExtraOffset), CurLoop))) &&
+           NBits->hasNoSignedWrap())
+    ExtraOffsetExpr = SE->getSCEV(ExtraOffset);
+  else {
+    IV = NBits;
+    ExtraOffsetExpr = SE->getZero(NBits->getType());
+  }
+
+  // Step 4: Check if the recurrence is in desirable form.
+  auto *IVPN = dyn_cast<PHINode>(IV);
+  if (!IVPN || IVPN->getParent() != LoopHeaderBB) {
+    LLVM_DEBUG(dbgs() << DEBUG_TYPE " Not an expected PHI node.\n");
+    return false;
+  }
+
+  Start = IVPN->getIncomingValueForBlock(LoopPreheaderBB);
+  IVNext = dyn_cast<Instruction>(IVPN->getIncomingValueForBlock(LoopHeaderBB));
+
+  if (!IVNext || !match(IVNext, m_Add(m_Specific(IVPN), m_One()))) {
+    LLVM_DEBUG(dbgs() << DEBUG_TYPE " Bad recurrence.\n");
+    return false;
+  }
+
+  // Step 4: Check if the backedge's destinations are in desirable form.
+
+  assert(ICmpInst::isEquality(Pred) &&
+         "Should only get equality predicates here.");
+
+  // cmp-br is commutative, so canonicalize to a single variant.
+  InvertedCond = Pred != ICmpInst::Predicate::ICMP_EQ;
+  if (InvertedCond) {
+    Pred = ICmpInst::getInversePredicate(Pred);
+    std::swap(TrueBB, FalseBB);
+  }
+
+  // We expect to exit loop when comparison yields true,
+  // so when it yields false we should branch back to loop header.
+  if (FalseBB != LoopHeaderBB) {
+    LLVM_DEBUG(dbgs() << DEBUG_TYPE " Bad backedge flow.\n");
+    return false;
+  }
+
+  // The new, countable, loop will certainly only run a known number of
+  // iterations, It won't be infinite. But the old loop might be infinite
+  // under certain conditions. For logical shifts, the value will become zero
+  // after at most bitwidth(%Val) loop iterations. However, for arithmetic
+  // right-shift, iff the sign bit was set, the value will never become zero,
+  // and the loop may never finish.
+  if (ValShifted->getOpcode() == Instruction::AShr &&
+      !isMustProgress(CurLoop) && !SE->isKnownNonNegative(SE->getSCEV(Val))) {
+    LLVM_DEBUG(dbgs() << DEBUG_TYPE " Can not prove the loop is finite.\n");
+    return false;
+  }
+
+  // Okay, idiom checks out.
+  return true;
+}
+
+/// Look for the following loop:
+/// \code
+///   entry:
+///     <...>
+///     %start = <...>
+///     %extraoffset = <...>
+///     <...>
+///     br label %for.cond
+///
+///   loop:
+///     %iv = phi i8 [ %start, %entry ], [ %iv.next, %for.cond ]
+///     %nbits = add nsw i8 %iv, %extraoffset
+///     %val.shifted = {{l,a}shr,shl} i8 %val, %nbits
+///     %val.shifted.iszero = icmp eq i8 %val.shifted, 0
+///     %iv.next = add i8 %iv, 1
+///     <...>
+///     br i1 %val.shifted.iszero, label %end, label %loop
+///
+///   end:
+///     %iv.res = phi i8 [ %iv, %loop ] <...>
+///     %nbits.res = phi i8 [ %nbits, %loop ] <...>
+///     %val.shifted.res = phi i8 [ %val.shifted, %loop ] <...>
+///     %val.shifted.iszero.res = phi i1 [ %val.shifted.iszero, %loop ] <...>
+///     %iv.next.res = phi i8 [ %iv.next, %loop ] <...>
+///     <...>
+/// \endcode
+///
+/// And transform it into:
+/// \code
+///   entry:
+///     <...>
+///     %start = <...>
+///     %extraoffset = <...>
+///     <...>
+///     %val.numleadingzeros = call i8 @llvm.ct{l,t}z.i8(i8 %val, i1 0)
+///     %val.numactivebits = sub i8 8, %val.numleadingzeros
+///     %extraoffset.neg = sub i8 0, %extraoffset
+///     %tmp = add i8 %val.numactivebits, %extraoffset.neg
+///     %iv.final = call i8 @llvm.smax.i8(i8 %tmp, i8 %start)
+///     %loop.tripcount = sub i8 %iv.final, %start
+///     br label %loop
+///
+///   loop:
+///     %loop.iv = phi i8 [ 0, %entry ], [ %loop.iv.next, %loop ]
+///     %loop.iv.next = add i8 %loop.iv, 1
+///     %loop.ivcheck = icmp eq i8 %loop.iv.next, %loop.tripcount
+///     %iv = add i8 %loop.iv, %start
+///     <...>
+///     br i1 %loop.ivcheck, label %end, label %loop
+///
+///   end:
+///     %iv.res = phi i8 [ %iv.final, %loop ] <...>
+///     <...>
+/// \endcode
+bool LoopIdiomRecognize::recognizeShiftUntilZero() {
+  bool MadeChange = false;
+
+  Instruction *ValShiftedIsZero;
+  Intrinsic::ID IntrID;
+  Instruction *IV;
+  Value *Start, *Val;
+  const SCEV *ExtraOffsetExpr;
+  bool InvertedCond;
+  if (!detectShiftUntilZeroIdiom(CurLoop, SE, ValShiftedIsZero, IntrID, IV,
+                                 Start, Val, ExtraOffsetExpr, InvertedCond)) {
+    LLVM_DEBUG(dbgs() << DEBUG_TYPE
+               " shift-until-zero idiom detection failed.\n");
+    return MadeChange;
+  }
+  LLVM_DEBUG(dbgs() << DEBUG_TYPE " shift-until-zero idiom detected!\n");
+
+  // Ok, it is the idiom we were looking for, we *could* transform this loop,
+  // but is it profitable to transform?
+
+  BasicBlock *LoopHeaderBB = CurLoop->getHeader();
+  BasicBlock *LoopPreheaderBB = CurLoop->getLoopPreheader();
+  assert(LoopPreheaderBB && "There is always a loop preheader.");
+
+  BasicBlock *SuccessorBB = CurLoop->getExitBlock();
+  assert(SuccessorBB && "There is only a single successor.");
+
+  IRBuilder<> Builder(LoopPreheaderBB->getTerminator());
+  Builder.SetCurrentDebugLocation(IV->getDebugLoc());
+
+  Type *Ty = Val->getType();
+  unsigned Bitwidth = Ty->getScalarSizeInBits();
+
+  TargetTransformInfo::TargetCostKind CostKind =
+      TargetTransformInfo::TCK_SizeAndLatency;
+
+  // The rewrite is considered to be unprofitable iff and only iff the
+  // intrinsic we'll use are not cheap. Note that we are okay with *just*
+  // making the loop countable, even if nothing else changes.
+  IntrinsicCostAttributes Attrs(
+      IntrID, Ty, {UndefValue::get(Ty), /*is_zero_undef=*/Builder.getFalse()});
+  InstructionCost Cost = TTI->getIntrinsicInstrCost(Attrs, CostKind);
+  if (Cost > TargetTransformInfo::TCC_Basic) {
+    LLVM_DEBUG(dbgs() << DEBUG_TYPE
+               " Intrinsic is too costly, not beneficial\n");
+    return MadeChange;
+  }
+
+  // Ok, transform appears worthwhile.
+  MadeChange = true;
+
+  bool OffsetIsZero = false;
+  if (auto *ExtraOffsetExprC = dyn_cast<SCEVConstant>(ExtraOffsetExpr))
+    OffsetIsZero = ExtraOffsetExprC->isZero();
+
+  // Step 1: Compute the loop's final IV value / trip count.
+
+  CallInst *ValNumLeadingZeros = Builder.CreateIntrinsic(
+      IntrID, Ty, {Val, /*is_zero_undef=*/Builder.getFalse()},
+      /*FMFSource=*/nullptr, Val->getName() + ".numleadingzeros");
+  Value *ValNumActiveBits = Builder.CreateSub(
+      ConstantInt::get(Ty, Ty->getScalarSizeInBits()), ValNumLeadingZeros,
+      Val->getName() + ".numactivebits", /*HasNUW=*/true,
+      /*HasNSW=*/Bitwidth != 2);
+
+  SCEVExpander Expander(*SE, *DL, "loop-idiom");
+  Expander.setInsertPoint(&*Builder.GetInsertPoint());
+  Value *ExtraOffset = Expander.expandCodeFor(ExtraOffsetExpr);
+
+  Value *ValNumActiveBitsOffset = Builder.CreateAdd(
+      ValNumActiveBits, ExtraOffset, ValNumActiveBits->getName() + ".offset",
+      /*HasNUW=*/OffsetIsZero, /*HasNSW=*/true);
+  Value *IVFinal = Builder.CreateIntrinsic(Intrinsic::smax, {Ty},
+                                           {ValNumActiveBitsOffset, Start},
+                                           /*FMFSource=*/nullptr, "iv.final");
+
+  auto *LoopBackedgeTakenCount = cast<Instruction>(Builder.CreateSub(
+      IVFinal, Start, CurLoop->getName() + ".backedgetakencount",
+      /*HasNUW=*/OffsetIsZero, /*HasNSW=*/true));
+  // FIXME: or when the offset was `add nuw`
+
+  // We know loop's backedge-taken count, but what's loop's trip count?
+  Value *LoopTripCount =
+      Builder.CreateAdd(LoopBackedgeTakenCount, ConstantInt::get(Ty, 1),
+                        CurLoop->getName() + ".tripcount", /*HasNUW=*/true,
+                        /*HasNSW=*/Bitwidth != 2);
+
+  // Step 2: Adjust the successor basic block to recieve the original
+  //         induction variable's final value instead of the orig. IV itself.
+
+  IV->replaceUsesOutsideBlock(IVFinal, LoopHeaderBB);
+
+  // Step 3: Rewrite the loop into a countable form, with canonical IV.
+
+  // The new canonical induction variable.
+  Builder.SetInsertPoint(&LoopHeaderBB->front());
+  auto *CIV = Builder.CreatePHI(Ty, 2, CurLoop->getName() + ".iv");
+
+  // The induction itself.
+  Builder.SetInsertPoint(LoopHeaderBB->getFirstNonPHI());
+  auto *CIVNext =
+      Builder.CreateAdd(CIV, ConstantInt::get(Ty, 1), CIV->getName() + ".next",
+                        /*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
+
+  // The loop trip count check.
+  auto *CIVCheck = Builder.CreateICmpEQ(CIVNext, LoopTripCount,
+                                        CurLoop->getName() + ".ivcheck");
+  auto *NewIVCheck = CIVCheck;
+  if (InvertedCond) {
+    NewIVCheck = Builder.CreateNot(CIVCheck);
+    NewIVCheck->takeName(ValShiftedIsZero);
+  }
+
+  // The original IV, but rebased to be an offset to the CIV.
+  auto *IVDePHId = Builder.CreateAdd(CIV, Start, "", /*HasNUW=*/false,
+                                     /*HasNSW=*/true); // FIXME: what about NUW?
+  IVDePHId->takeName(IV);
+
+  // The loop terminator.
+  Builder.SetInsertPoint(LoopHeaderBB->getTerminator());
+  Builder.CreateCondBr(CIVCheck, SuccessorBB, LoopHeaderBB);
+  LoopHeaderBB->getTerminator()->eraseFromParent();
+
+  // Populate the IV PHI.
+  CIV->addIncoming(ConstantInt::get(Ty, 0), LoopPreheaderBB);
+  CIV->addIncoming(CIVNext, LoopHeaderBB);
+
+  // Step 4: Forget the "non-computable" trip-count SCEV associated with the
+  //   loop. The loop would otherwise not be deleted even if it becomes empty.
+
+  SE->forgetLoop(CurLoop);
+
+  // Step 5: Try to cleanup the loop's body somewhat.
+  IV->replaceAllUsesWith(IVDePHId);
+  IV->eraseFromParent();
+
+  ValShiftedIsZero->replaceAllUsesWith(NewIVCheck);
+  ValShiftedIsZero->eraseFromParent();
+
+  // Other passes will take care of actually deleting the loop if possible.
+
+  LLVM_DEBUG(dbgs() << DEBUG_TYPE " shift-until-zero idiom optimized!\n");
+
+  ++NumShiftUntilZero;
+  return MadeChange;
+}
diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
index d9dbc0deb42a..34545f35b3c3 100644
--- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopNestAnalysis.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ScalarEvolution.h"
@@ -186,12 +187,8 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level,
 // matrix by exchanging the two columns.
 static void interChangeDependencies(CharMatrix &DepMatrix, unsigned FromIndx,
                                     unsigned ToIndx) {
-  unsigned numRows = DepMatrix.size();
-  for (unsigned i = 0; i < numRows; ++i) {
-    char TmpVal = DepMatrix[i][ToIndx];
-    DepMatrix[i][ToIndx] = DepMatrix[i][FromIndx];
-    DepMatrix[i][FromIndx] = TmpVal;
-  }
+  for (unsigned I = 0, E = DepMatrix.size(); I < E; ++I)
+    std::swap(DepMatrix[I][ToIndx], DepMatrix[I][FromIndx]);
 }
 
 // Checks if outermost non '=','S'or'I' dependence in the dependence matrix is
@@ -400,10 +397,8 @@ class LoopInterchangeTransform {
 public:
   LoopInterchangeTransform(Loop *Outer, Loop *Inner, ScalarEvolution *SE,
                            LoopInfo *LI, DominatorTree *DT,
-                           BasicBlock *LoopNestExit,
                            const LoopInterchangeLegality &LIL)
-      : OuterLoop(Outer), InnerLoop(Inner), SE(SE), LI(LI), DT(DT),
-        LoopExit(LoopNestExit), LIL(LIL) {}
+      : OuterLoop(Outer), InnerLoop(Inner), SE(SE), LI(LI), DT(DT), LIL(LIL) {}
 
   /// Interchange OuterLoop and InnerLoop.
   bool transform();
@@ -424,7 +419,6 @@ private:
 
   LoopInfo *LI;
   DominatorTree *DT;
-  BasicBlock *LoopExit;
 
   const LoopInterchangeLegality &LIL;
 };
@@ -449,7 +443,15 @@ struct LoopInterchange {
     return processLoopList(populateWorklist(*L));
   }
 
-  bool isComputableLoopNest(LoopVector LoopList) {
+  bool run(LoopNest &LN) {
+    const auto &LoopList = LN.getLoops();
+    for (unsigned I = 1; I < LoopList.size(); ++I)
+      if (LoopList[I]->getParentLoop() != LoopList[I - 1])
+        return false;
+    return processLoopList(LoopList);
+  }
+
+  bool isComputableLoopNest(ArrayRef<Loop *> LoopList) {
     for (Loop *L : LoopList) {
       const SCEV *ExitCountOuter = SE->getBackedgeTakenCount(L);
       if (isa<SCEVCouldNotCompute>(ExitCountOuter)) {
@@ -468,13 +470,13 @@ struct LoopInterchange {
     return true;
   }
 
-  unsigned selectLoopForInterchange(const LoopVector &LoopList) {
+  unsigned selectLoopForInterchange(ArrayRef<Loop *> LoopList) {
     // TODO: Add a better heuristic to select the loop to be interchanged based
     // on the dependence matrix. Currently we select the innermost loop.
     return LoopList.size() - 1;
   }
 
-  bool processLoopList(LoopVector LoopList) {
+  bool processLoopList(ArrayRef<Loop *> LoopList) {
     bool Changed = false;
     unsigned LoopNestDepth = LoopList.size();
     if (LoopNestDepth < 2) {
@@ -515,14 +517,12 @@ struct LoopInterchange {
 
     unsigned SelecLoopId = selectLoopForInterchange(LoopList);
     // Move the selected loop outwards to the best possible position.
+    Loop *LoopToBeInterchanged = LoopList[SelecLoopId];
     for (unsigned i = SelecLoopId; i > 0; i--) {
-      bool Interchanged =
-          processLoop(LoopList, i, i - 1, LoopNestExit, DependencyMatrix);
+      bool Interchanged = processLoop(LoopToBeInterchanged, LoopList[i - 1], i,
+                                      i - 1, DependencyMatrix);
       if (!Interchanged)
         return Changed;
-      // Loops interchanged reflect the same in LoopList
-      std::swap(LoopList[i - 1], LoopList[i]);
-
       // Update the DependencyMatrix
       interChangeDependencies(DependencyMatrix, i, i - 1);
 #ifdef DUMP_DEP_MATRICIES
@@ -534,14 +534,11 @@ struct LoopInterchange {
     return Changed;
   }
 
-  bool processLoop(LoopVector LoopList, unsigned InnerLoopId,
-                   unsigned OuterLoopId, BasicBlock *LoopNestExit,
+  bool processLoop(Loop *InnerLoop, Loop *OuterLoop, unsigned InnerLoopId,
+                   unsigned OuterLoopId,
                    std::vector<std::vector<char>> &DependencyMatrix) {
-    LLVM_DEBUG(dbgs() << "Processing Inner Loop Id = " << InnerLoopId
+    LLVM_DEBUG(dbgs() << "Processing InnerLoopId = " << InnerLoopId
                       << " and OuterLoopId = " << OuterLoopId << "\n");
-    Loop *InnerLoop = LoopList[InnerLoopId];
-    Loop *OuterLoop = LoopList[OuterLoopId];
-
     LoopInterchangeLegality LIL(OuterLoop, InnerLoop, SE, ORE);
     if (!LIL.canInterchangeLoops(InnerLoopId, OuterLoopId, DependencyMatrix)) {
       LLVM_DEBUG(dbgs() << "Not interchanging loops. Cannot prove legality.\n");
@@ -561,8 +558,7 @@ struct LoopInterchange {
              << "Loop interchanged with enclosing loop.";
     });
 
-    LoopInterchangeTransform LIT(OuterLoop, InnerLoop, SE, LI, DT, LoopNestExit,
-                                 LIL);
+    LoopInterchangeTransform LIT(OuterLoop, InnerLoop, SE, LI, DT, LIL);
     LIT.transform();
     LLVM_DEBUG(dbgs() << "Loops interchanged.\n");
     LoopsInterchanged++;
@@ -618,6 +614,22 @@ bool LoopInterchangeLegality::tightlyNested(Loop *OuterLoop, Loop *InnerLoop) {
       containsUnsafeInstructions(InnerLoopPreHeader))
     return false;
 
+  BasicBlock *InnerLoopExit = InnerLoop->getExitBlock();
+  // Ensure the inner loop exit block flows to the outer loop latch possibly
+  // through empty blocks.
+  const BasicBlock &SuccInner =
+      LoopNest::skipEmptyBlockUntil(InnerLoopExit, OuterLoopLatch);
+  if (&SuccInner != OuterLoopLatch) {
+    LLVM_DEBUG(dbgs() << "Inner loop exit block " << *InnerLoopExit
+                      << " does not lead to the outer loop latch.\n";);
+    return false;
+  }
+  // The inner loop exit block does flow to the outer loop latch and not some
+  // other BBs, now make sure it contains safe instructions, since it will be
+  // moved into the (new) inner loop after interchange.
+  if (containsUnsafeInstructions(InnerLoopExit))
+    return false;
+
   LLVM_DEBUG(dbgs() << "Loops are perfectly nested\n");
   // We have a perfect loop nest.
   return true;
@@ -644,6 +656,65 @@ bool LoopInterchangeLegality::isLoopStructureUnderstood(
       return false;
     }
   }
+
+  // TODO: Handle triangular loops of another form.
+  // e.g. for(int i=0;i<N;i++)
+  //        for(int j=0;j<i;j++)
+  // or,
+  //      for(int i=0;i<N;i++)
+  //        for(int j=0;j*i<N;j++)
+  BasicBlock *InnerLoopLatch = InnerLoop->getLoopLatch();
+  BranchInst *InnerLoopLatchBI =
+      dyn_cast<BranchInst>(InnerLoopLatch->getTerminator());
+  if (!InnerLoopLatchBI->isConditional())
+    return false;
+  if (CmpInst *InnerLoopCmp =
+          dyn_cast<CmpInst>(InnerLoopLatchBI->getCondition())) {
+    Value *Op0 = InnerLoopCmp->getOperand(0);
+    Value *Op1 = InnerLoopCmp->getOperand(1);
+
+    // LHS and RHS of the inner loop exit condition, e.g.,
+    // in "for(int j=0;j<i;j++)", LHS is j and RHS is i.
+    Value *Left = nullptr;
+    Value *Right = nullptr;
+
+    // Check if V only involves inner loop induction variable.
+    // Return true if V is InnerInduction, or a cast from
+    // InnerInduction, or a binary operator that involves
+    // InnerInduction and a constant.
+    std::function<bool(Value *)> IsPathToIndVar;
+    IsPathToIndVar = [&InnerInduction, &IsPathToIndVar](Value *V) -> bool {
+      if (V == InnerInduction)
+        return true;
+      if (isa<Constant>(V))
+        return true;
+      Instruction *I = dyn_cast<Instruction>(V);
+      if (!I)
+        return false;
+      if (isa<CastInst>(I))
+        return IsPathToIndVar(I->getOperand(0));
+      if (isa<BinaryOperator>(I))
+        return IsPathToIndVar(I->getOperand(0)) &&
+               IsPathToIndVar(I->getOperand(1));
+      return false;
+    };
+
+    if (IsPathToIndVar(Op0) && !isa<Constant>(Op0)) {
+      Left = Op0;
+      Right = Op1;
+    } else if (IsPathToIndVar(Op1) && !isa<Constant>(Op1)) {
+      Left = Op1;
+      Right = Op0;
+    }
+
+    if (Left == nullptr)
+      return false;
+
+    const SCEV *S = SE->getSCEV(Right);
+    if (!SE->isLoopInvariant(S, OuterLoop))
+      return false;
+  }
+
   return true;
 }
 
@@ -951,6 +1022,40 @@ static bool areOuterLoopExitPHIsSupported(Loop *OuterLoop, Loop *InnerLoop) {
   return true;
 }
 
+// In case of multi-level nested loops, it may occur that lcssa phis exist in
+// the latch of InnerLoop, i.e., when defs of the incoming values are further
+// inside the loopnest. Sometimes those incoming values are not available
+// after interchange, since the original inner latch will become the new outer
+// latch which may have predecessor paths that do not include those incoming
+// values.
+// TODO: Handle transformation of lcssa phis in the InnerLoop latch in case of
+// multi-level loop nests.
+static bool areInnerLoopLatchPHIsSupported(Loop *OuterLoop, Loop *InnerLoop) {
+  if (InnerLoop->getSubLoops().empty())
+    return true;
+  // If the original outer latch has only one predecessor, then values defined
+  // further inside the looploop, e.g., in the innermost loop, will be available
+  // at the new outer latch after interchange.
+  if (OuterLoop->getLoopLatch()->getUniquePredecessor() != nullptr)
+    return true;
+
+  // The outer latch has more than one predecessors, i.e., the inner
+  // exit and the inner header.
+  // PHI nodes in the inner latch are lcssa phis where the incoming values
+  // are defined further inside the loopnest. Check if those phis are used
+  // in the original inner latch. If that is the case then bail out since
+  // those incoming values may not be available at the new outer latch.
+  BasicBlock *InnerLoopLatch = InnerLoop->getLoopLatch();
+  for (PHINode &PHI : InnerLoopLatch->phis()) {
+    for (auto *U : PHI.users()) {
+      Instruction *UI = cast<Instruction>(U);
+      if (InnerLoopLatch == UI->getParent())
+        return false;
+    }
+  }
+  return true;
+}
+
 bool LoopInterchangeLegality::canInterchangeLoops(unsigned InnerLoopId,
                                                   unsigned OuterLoopId,
                                                   CharMatrix &DepMatrix) {
@@ -986,6 +1091,18 @@ bool LoopInterchangeLegality::canInterchangeLoops(unsigned InnerLoopId,
         return false;
       }
 
+  if (!areInnerLoopLatchPHIsSupported(OuterLoop, InnerLoop)) {
+    LLVM_DEBUG(dbgs() << "Found unsupported PHI nodes in inner loop latch.\n");
+    ORE->emit([&]() {
+      return OptimizationRemarkMissed(DEBUG_TYPE, "UnsupportedInnerLatchPHI",
+                                      InnerLoop->getStartLoc(),
+                                      InnerLoop->getHeader())
+             << "Cannot interchange loops because unsupported PHI nodes found "
+                "in inner loop latch.";
+    });
+    return false;
+  }
+
   // TODO: The loops could not be interchanged due to current limitations in the
   // transform module.
   if (currentLimitations()) {
@@ -1269,9 +1386,7 @@ bool LoopInterchangeTransform::transform() {
         assert(!NewI->mayHaveSideEffects() &&
                "Moving instructions with side-effects may change behavior of "
                "the loop nest!");
-        for (auto UI = WorkList[i]->use_begin(), UE = WorkList[i]->use_end();
-             UI != UE;) {
-          Use &U = *UI++;
+        for (Use &U : llvm::make_early_inc_range(WorkList[i]->uses())) {
           Instruction *UserI = cast<Instruction>(U.getUser());
           if (!InnerLoop->contains(UserI->getParent()) ||
               UserI->getParent() == NewLatch || UserI == InductionPHI)
@@ -1460,6 +1575,13 @@ static void moveLCSSAPhis(BasicBlock *InnerExit, BasicBlock *InnerHeader,
       PHINode *NewPhi = dyn_cast<PHINode>(P.clone());
       NewPhi->setIncomingValue(0, P.getIncomingValue(0));
       NewPhi->setIncomingBlock(0, OuterLatch);
+      // We might have incoming edges from other BBs, i.e., the original outer
+      // header.
+      for (auto *Pred : predecessors(InnerLatch)) {
+        if (Pred == OuterLatch)
+          continue;
+        NewPhi->addIncoming(P.getIncomingValue(0), Pred);
+      }
       NewPhi->insertBefore(InnerLatch->getFirstNonPHI());
       P.setIncomingValue(0, NewPhi);
     }
@@ -1537,9 +1659,12 @@ bool LoopInterchangeTransform::adjustLoopBranches() {
                   InnerLoopPreHeader, DTUpdates, /*MustUpdateOnce=*/false);
   // The outer loop header might or might not branch to the outer latch.
   // We are guaranteed to branch to the inner loop preheader.
-  if (llvm::is_contained(OuterLoopHeaderBI->successors(), OuterLoopLatch))
-    updateSuccessor(OuterLoopHeaderBI, OuterLoopLatch, LoopExit, DTUpdates,
+  if (llvm::is_contained(OuterLoopHeaderBI->successors(), OuterLoopLatch)) {
+    // In this case the outerLoopHeader should branch to the InnerLoopLatch.
+    updateSuccessor(OuterLoopHeaderBI, OuterLoopLatch, InnerLoopLatch,
+                    DTUpdates,
                     /*MustUpdateOnce=*/false);
+  }
   updateSuccessor(OuterLoopHeaderBI, InnerLoopPreHeader,
                   InnerLoopHeaderSuccessor, DTUpdates,
                   /*MustUpdateOnce=*/false);
@@ -1582,15 +1707,19 @@ bool LoopInterchangeTransform::adjustLoopBranches() {
   // replaced by Inners'.
   OuterLoopLatchSuccessor->replacePhiUsesWith(OuterLoopLatch, InnerLoopLatch);
 
+  auto &OuterInnerReductions = LIL.getOuterInnerReductions();
   // Now update the reduction PHIs in the inner and outer loop headers.
   SmallVector<PHINode *, 4> InnerLoopPHIs, OuterLoopPHIs;
-  for (PHINode &PHI : drop_begin(InnerLoopHeader->phis()))
+  for (PHINode &PHI : InnerLoopHeader->phis()) {
+    if (OuterInnerReductions.find(&PHI) == OuterInnerReductions.end())
+      continue;
     InnerLoopPHIs.push_back(cast<PHINode>(&PHI));
-  for (PHINode &PHI : drop_begin(OuterLoopHeader->phis()))
+  }
+  for (PHINode &PHI : OuterLoopHeader->phis()) {
+    if (OuterInnerReductions.find(&PHI) == OuterInnerReductions.end())
+      continue;
     OuterLoopPHIs.push_back(cast<PHINode>(&PHI));
-
-  auto &OuterInnerReductions = LIL.getOuterInnerReductions();
-  (void)OuterInnerReductions;
+  }
 
   // Now move the remaining reduction PHIs from outer to inner loop header and
   // vice versa. The PHI nodes must be part of a reduction across the inner and
@@ -1682,14 +1811,15 @@ Pass *llvm::createLoopInterchangePass() {
   return new LoopInterchangeLegacyPass();
 }
 
-PreservedAnalyses LoopInterchangePass::run(Loop &L, LoopAnalysisManager &AM,
+PreservedAnalyses LoopInterchangePass::run(LoopNest &LN,
+                                           LoopAnalysisManager &AM,
                                            LoopStandardAnalysisResults &AR,
                                            LPMUpdater &U) {
-  Function &F = *L.getHeader()->getParent();
+  Function &F = *LN.getParent();
 
   DependenceInfo DI(&F, &AR.AA, &AR.SE, &AR.LI);
   OptimizationRemarkEmitter ORE(&F);
-  if (!LoopInterchange(&AR.SE, &AR.LI, &DI, &AR.DT, &ORE).run(&L))
+  if (!LoopInterchange(&AR.SE, &AR.LI, &DI, &AR.DT, &ORE).run(LN))
     return PreservedAnalyses::all();
   return getLoopPassPreservedAnalyses();
 }
diff --git a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
index 058612149a94..aaf586173e44 100644
--- a/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -99,12 +99,11 @@ struct StoreToLoadForwardingCandidate {
                                  Loop *L) const {
     Value *LoadPtr = Load->getPointerOperand();
     Value *StorePtr = Store->getPointerOperand();
-    Type *LoadPtrType = LoadPtr->getType();
-    Type *LoadType = LoadPtrType->getPointerElementType();
+    Type *LoadType = getLoadStoreType(Load);
 
-    assert(LoadPtrType->getPointerAddressSpace() ==
+    assert(LoadPtr->getType()->getPointerAddressSpace() ==
                StorePtr->getType()->getPointerAddressSpace() &&
-           LoadType == StorePtr->getType()->getPointerElementType() &&
+           LoadType == getLoadStoreType(Store) &&
            "Should be a known dependence");
 
     // Currently we only support accesses with unit stride.  FIXME: we should be
@@ -664,10 +663,11 @@ public:
     auto *BFI = (PSI && PSI->hasProfileSummary()) ?
                 &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI() :
                 nullptr;
+    auto *SE = &getAnalysis<ScalarEvolutionWrapperPass>().getSE();
 
     // Process each loop nest in the function.
     return eliminateLoadsAcrossLoops(
-        F, LI, DT, BFI, PSI, /*SE*/ nullptr, /*AC*/ nullptr,
+        F, LI, DT, BFI, PSI, SE, /*AC*/ nullptr,
         [&LAA](Loop &L) -> const LoopAccessInfo & { return LAA.getInfo(&L); });
   }
 
diff --git a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
index 3fe8e7259114..f4fce4871331 100644
--- a/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopPassManager.cpp
@@ -14,6 +14,7 @@
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/ScalarEvolutionAliasAnalysis.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/TimeProfiler.h"
 
 using namespace llvm;
@@ -26,10 +27,6 @@ PreservedAnalyses
 PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
             LPMUpdater &>::run(Loop &L, LoopAnalysisManager &AM,
                                LoopStandardAnalysisResults &AR, LPMUpdater &U) {
-
-  if (DebugLogging)
-    dbgs() << "Starting Loop pass manager run.\n";
-
   // Runs loop-nest passes only when the current loop is a top-level one.
   PreservedAnalyses PA = (L.isOutermost() && !LoopNestPasses.empty())
                              ? runWithLoopNestPasses(L, AM, AR, U)
@@ -44,9 +41,6 @@ PassManager<Loop, LoopAnalysisManager, LoopStandardAnalysisResults &,
   // be preserved, but unrolling should invalidate the parent loop's analyses.
   PA.preserveSet<AllAnalysesOn<Loop>>();
 
-  if (DebugLogging)
-    dbgs() << "Finished Loop pass manager run.\n";
-
   return PA;
 }
 
@@ -114,6 +108,11 @@ LoopPassManager::runWithLoopNestPasses(Loop &L, LoopAnalysisManager &AM,
     // Check if the current pass preserved the loop-nest object or not.
     IsLoopNestPtrValid &= PassPA->getChecker<LoopNestAnalysis>().preserved();
 
+    // After running the loop pass, the parent loop might change and we need to
+    // notify the updater, otherwise U.ParentL might gets outdated and triggers
+    // assertion failures in addSiblingLoops and addChildLoops.
+    U.setParentLoop(L.getParentLoop());
+
     // FIXME: Historically, the pass managers all called the LLVM context's
     // yield function here. We don't have a generic way to acquire the
     // context and it isn't yet clear what the right pattern is for yielding
@@ -158,6 +157,11 @@ LoopPassManager::runWithoutLoopNestPasses(Loop &L, LoopAnalysisManager &AM,
     // aggregate preserved set for this pass manager.
     PA.intersect(std::move(*PassPA));
 
+    // After running the loop pass, the parent loop might change and we need to
+    // notify the updater, otherwise U.ParentL might gets outdated and triggers
+    // assertion failures in addSiblingLoops and addChildLoops.
+    U.setParentLoop(L.getParentLoop());
+
     // FIXME: Historically, the pass managers all called the LLVM context's
     // yield function here. We don't have a generic way to acquire the
     // context and it isn't yet clear what the right pattern is for yielding
@@ -281,8 +285,17 @@ PreservedAnalyses FunctionToLoopPassAdaptor::run(Function &F,
     else
       PI.runAfterPass<Loop>(*Pass, *L, PassPA);
 
-    // FIXME: We should verify the set of analyses relevant to Loop passes
-    // are preserved.
+#ifndef NDEBUG
+    // LoopAnalysisResults should always be valid.
+    // Note that we don't LAR.SE.verify() because that can change observed SE
+    // queries. See PR44815.
+    if (VerifyDomInfo)
+      LAR.DT.verify();
+    if (VerifyLoopInfo)
+      LAR.LI.verify(LAR.DT);
+    if (LAR.MSSA && VerifyMemorySSA)
+      LAR.MSSA->verifyMemorySSA();
+#endif
 
     // If the loop hasn't been deleted, we need to handle invalidation here.
     if (!Updater.skipCurrentLoop())
@@ -314,12 +327,6 @@ PreservedAnalyses FunctionToLoopPassAdaptor::run(Function &F,
     PA.preserve<BlockFrequencyAnalysis>();
   if (UseMemorySSA)
     PA.preserve<MemorySSAAnalysis>();
-  // FIXME: What we really want to do here is preserve an AA category, but
-  // that concept doesn't exist yet.
-  PA.preserve<AAManager>();
-  PA.preserve<BasicAA>();
-  PA.preserve<GlobalsAA>();
-  PA.preserve<SCEVAA>();
   return PA;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/LoopRerollPass.cpp b/llvm/lib/Transforms/Scalar/LoopRerollPass.cpp
index b3bae47e96de..56d66b93dd69 100644
--- a/llvm/lib/Transforms/Scalar/LoopRerollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopRerollPass.cpp
@@ -716,9 +716,8 @@ void LoopReroll::DAGRootTracker::collectInLoopUserSet(
       }
 
     // We also want to collect single-user "feeder" values.
-    for (User::op_iterator OI = I->op_begin(),
-         OIE = I->op_end(); OI != OIE; ++OI) {
-      if (Instruction *Op = dyn_cast<Instruction>(*OI))
+    for (Use &U : I->operands()) {
+      if (Instruction *Op = dyn_cast<Instruction>(U))
         if (Op->hasOneUse() && L->contains(Op) && !Exclude.count(Op) &&
             !Final.count(Op))
           Queue.push_back(Op);
@@ -912,6 +911,8 @@ bool LoopReroll::DAGRootTracker::validateRootSet(DAGRootSet &DRS) {
   // Check that the first root is evenly spaced.
   unsigned N = DRS.Roots.size() + 1;
   const SCEV *StepSCEV = SE->getMinusSCEV(SE->getSCEV(DRS.Roots[0]), ADR);
+  if (isa<SCEVCouldNotCompute>(StepSCEV) || StepSCEV->getType()->isPointerTy())
+    return false;
   const SCEV *ScaleSCEV = SE->getConstant(StepSCEV->getType(), N);
   if (ADR->getStepRecurrence(*SE) != SE->getMulExpr(StepSCEV, ScaleSCEV))
     return false;
@@ -1081,6 +1082,12 @@ bool LoopReroll::DAGRootTracker::collectUsedInstructions(SmallInstructionSet &Po
   DenseSet<Instruction*> V;
   collectInLoopUserSet(LoopIncs, Exclude, PossibleRedSet, V);
   for (auto *I : V) {
+    if (I->mayHaveSideEffects()) {
+      LLVM_DEBUG(dbgs() << "LRR: Aborting - "
+                        << "An instruction which does not belong to any root "
+                        << "sets must not have side effects: " << *I);
+      return false;
+    }
     Uses[I].set(IL_All);
   }
 
diff --git a/llvm/lib/Transforms/Scalar/LoopRotation.cpp b/llvm/lib/Transforms/Scalar/LoopRotation.cpp
index ad1cfc68ece0..6d5b19443c76 100644
--- a/llvm/lib/Transforms/Scalar/LoopRotation.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopRotation.cpp
@@ -14,6 +14,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/MemorySSA.h"
 #include "llvm/Analysis/MemorySSAUpdater.h"
@@ -101,6 +102,11 @@ public:
     if (EnableMSSALoopDependency)
       AU.addPreserved<MemorySSAWrapperPass>();
     getLoopAnalysisUsage(AU);
+
+    // Lazy BFI and BPI are marked as preserved here so LoopRotate
+    // can remain part of the same loop pass manager as LICM.
+    AU.addPreserved<LazyBlockFrequencyInfoPass>();
+    AU.addPreserved<LazyBranchProbabilityInfoPass>();
   }
 
   bool runOnLoop(Loop *L, LPPassManager &LPM) override {
diff --git a/llvm/lib/Transforms/Scalar/LoopSink.cpp b/llvm/lib/Transforms/Scalar/LoopSink.cpp
index 47698fdde69f..a01287f587d7 100644
--- a/llvm/lib/Transforms/Scalar/LoopSink.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopSink.cpp
@@ -31,6 +31,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Scalar/LoopSink.h"
+#include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AliasSetTracker.h"
@@ -191,7 +192,7 @@ static bool sinkInstruction(
   for (auto &U : I.uses()) {
     Instruction *UI = cast<Instruction>(U.getUser());
     // We cannot sink I to PHI-uses.
-    if (dyn_cast<PHINode>(UI))
+    if (isa<PHINode>(UI))
       return false;
     // We cannot sink I if it has uses outside of the loop.
     if (!L.contains(LI.getLoopFor(UI->getParent())))
@@ -212,11 +213,9 @@ static bool sinkInstruction(
     return false;
 
   // Return if any of the candidate blocks to sink into is non-cold.
-  if (BBsToSinkInto.size() > 1) {
-    for (auto *BB : BBsToSinkInto)
-      if (!LoopBlockNumber.count(BB))
-        return false;
-  }
+  if (BBsToSinkInto.size() > 1 &&
+      !llvm::set_is_subset(BBsToSinkInto, LoopBlockNumber))
+    return false;
 
   // Copy the final BBs into a vector and sort them using the total ordering
   // of the loop block numbers as iterating the set doesn't give a useful
diff --git a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index 5dec9b542076..5f210380ae5a 100644
--- a/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -77,6 +77,7 @@
 #include "llvm/Analysis/ScalarEvolutionNormalization.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Config/llvm-config.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
@@ -161,9 +162,18 @@ static cl::opt<bool> FilterSameScaledReg(
     cl::desc("Narrow LSR search space by filtering non-optimal formulae"
              " with the same ScaledReg and Scale"));
 
-static cl::opt<bool> EnableBackedgeIndexing(
-  "lsr-backedge-indexing", cl::Hidden, cl::init(true),
-  cl::desc("Enable the generation of cross iteration indexed memops"));
+static cl::opt<TTI::AddressingModeKind> PreferredAddresingMode(
+  "lsr-preferred-addressing-mode", cl::Hidden, cl::init(TTI::AMK_None),
+   cl::desc("A flag that overrides the target's preferred addressing mode."),
+   cl::values(clEnumValN(TTI::AMK_None,
+                         "none",
+                         "Don't prefer any addressing mode"),
+              clEnumValN(TTI::AMK_PreIndexed,
+                         "preindexed",
+                         "Prefer pre-indexed addressing mode"),
+              clEnumValN(TTI::AMK_PostIndexed,
+                         "postindexed",
+                         "Prefer post-indexed addressing mode")));
 
 static cl::opt<unsigned> ComplexityLimit(
   "lsr-complexity-limit", cl::Hidden,
@@ -501,9 +511,16 @@ bool Formula::isCanonical(const Loop &L) const {
 void Formula::canonicalize(const Loop &L) {
   if (isCanonical(L))
     return;
-  // So far we did not need this case. This is easy to implement but it is
-  // useless to maintain dead code. Beside it could hurt compile time.
-  assert(!BaseRegs.empty() && "1*reg => reg, should not be needed.");
+
+  if (BaseRegs.empty()) {
+    // No base reg? Use scale reg with scale = 1 as such.
+    assert(ScaledReg && "Expected 1*reg => reg");
+    assert(Scale == 1 && "Expected 1*reg => reg");
+    BaseRegs.push_back(ScaledReg);
+    Scale = 0;
+    ScaledReg = nullptr;
+    return;
+  }
 
   // Keep the invariant sum in BaseRegs and one of the variant sum in ScaledReg.
   if (!ScaledReg) {
@@ -523,6 +540,7 @@ void Formula::canonicalize(const Loop &L) {
     if (I != BaseRegs.end())
       std::swap(ScaledReg, *I);
   }
+  assert(isCanonical(L) && "Failed to canonicalize?");
 }
 
 /// Get rid of the scale in the formula.
@@ -655,7 +673,7 @@ static bool isMulSExtable(const SCEVMulExpr *M, ScalarEvolution &SE) {
 
 /// Return an expression for LHS /s RHS, if it can be determined and if the
 /// remainder is known to be zero, or null otherwise. If IgnoreSignificantBits
-/// is true, expressions like (X * Y) /s Y are simplified to Y, ignoring that
+/// is true, expressions like (X * Y) /s Y are simplified to X, ignoring that
 /// the multiplication may overflow, which is useful when the result will be
 /// used in a context where the most significant bits are ignored.
 static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
@@ -671,8 +689,11 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
     const APInt &RA = RC->getAPInt();
     // Handle x /s -1 as x * -1, to give ScalarEvolution a chance to do
     // some folding.
-    if (RA.isAllOnesValue())
+    if (RA.isAllOnesValue()) {
+      if (LHS->getType()->isPointerTy())
+        return nullptr;
       return SE.getMulExpr(LHS, RC);
+    }
     // Handle x /s 1 as x.
     if (RA == 1)
       return LHS;
@@ -723,6 +744,21 @@ static const SCEV *getExactSDiv(const SCEV *LHS, const SCEV *RHS,
   // Check for a multiply operand that we can pull RHS out of.
   if (const SCEVMulExpr *Mul = dyn_cast<SCEVMulExpr>(LHS)) {
     if (IgnoreSignificantBits || isMulSExtable(Mul, SE)) {
+      // Handle special case C1*X*Y /s C2*X*Y.
+      if (const SCEVMulExpr *MulRHS = dyn_cast<SCEVMulExpr>(RHS)) {
+        if (IgnoreSignificantBits || isMulSExtable(MulRHS, SE)) {
+          const SCEVConstant *LC = dyn_cast<SCEVConstant>(Mul->getOperand(0));
+          const SCEVConstant *RC =
+              dyn_cast<SCEVConstant>(MulRHS->getOperand(0));
+          if (LC && RC) {
+            SmallVector<const SCEV *, 4> LOps(drop_begin(Mul->operands()));
+            SmallVector<const SCEV *, 4> ROps(drop_begin(MulRHS->operands()));
+            if (LOps == ROps)
+              return getExactSDiv(LC, RC, SE, IgnoreSignificantBits);
+          }
+        }
+      }
+
       SmallVector<const SCEV *, 4> Ops;
       bool Found = false;
       for (const SCEV *S : Mul->operands()) {
@@ -999,9 +1035,9 @@ static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
                                  const LSRUse &LU, const Formula &F);
 
 // Get the cost of the scaling factor used in F for LU.
-static unsigned getScalingFactorCost(const TargetTransformInfo &TTI,
-                                     const LSRUse &LU, const Formula &F,
-                                     const Loop &L);
+static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
+                                            const LSRUse &LU, const Formula &F,
+                                            const Loop &L);
 
 namespace {
 
@@ -1011,11 +1047,13 @@ class Cost {
   ScalarEvolution *SE = nullptr;
   const TargetTransformInfo *TTI = nullptr;
   TargetTransformInfo::LSRCost C;
+  TTI::AddressingModeKind AMK = TTI::AMK_None;
 
 public:
   Cost() = delete;
-  Cost(const Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI) :
-    L(L), SE(&SE), TTI(&TTI) {
+  Cost(const Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI,
+       TTI::AddressingModeKind AMK) :
+    L(L), SE(&SE), TTI(&TTI), AMK(AMK) {
     C.Insns = 0;
     C.NumRegs = 0;
     C.AddRecCost = 0;
@@ -1233,7 +1271,7 @@ void Cost::RateRegister(const Formula &F, const SCEV *Reg,
     // for now LSR only handles innermost loops).
     if (AR->getLoop() != L) {
       // If the AddRec exists, consider it's register free and leave it alone.
-      if (isExistingPhi(AR, *SE) && !TTI->shouldFavorPostInc())
+      if (isExistingPhi(AR, *SE) && AMK != TTI::AMK_PostIndexed)
         return;
 
       // It is bad to allow LSR for current loop to add induction variables
@@ -1254,13 +1292,11 @@ void Cost::RateRegister(const Formula &F, const SCEV *Reg,
 
       // If the step size matches the base offset, we could use pre-indexed
       // addressing.
-      if (TTI->shouldFavorBackedgeIndex(L)) {
+      if (AMK == TTI::AMK_PreIndexed) {
         if (auto *Step = dyn_cast<SCEVConstant>(AR->getStepRecurrence(*SE)))
           if (Step->getAPInt() == F.BaseOffset)
             LoopCost = 0;
-      }
-
-      if (TTI->shouldFavorPostInc()) {
+      } else if (AMK == TTI::AMK_PostIndexed) {
         const SCEV *LoopStep = AR->getStepRecurrence(*SE);
         if (isa<SCEVConstant>(LoopStep)) {
           const SCEV *LoopStart = AR->getStart();
@@ -1350,7 +1386,7 @@ void Cost::RateFormula(const Formula &F,
   C.NumBaseAdds += (F.UnfoldedOffset != 0);
 
   // Accumulate non-free scaling amounts.
-  C.ScaleCost += getScalingFactorCost(*TTI, LU, F, *L);
+  C.ScaleCost += *getScalingFactorCost(*TTI, LU, F, *L).getValue();
 
   // Tally up the non-zero immediates.
   for (const LSRFixup &Fixup : LU.Fixups) {
@@ -1747,9 +1783,9 @@ static bool isAMCompletelyFolded(const TargetTransformInfo &TTI,
                               F.Scale);
 }
 
-static unsigned getScalingFactorCost(const TargetTransformInfo &TTI,
-                                     const LSRUse &LU, const Formula &F,
-                                     const Loop &L) {
+static InstructionCost getScalingFactorCost(const TargetTransformInfo &TTI,
+                                            const LSRUse &LU, const Formula &F,
+                                            const Loop &L) {
   if (!F.Scale)
     return 0;
 
@@ -1762,14 +1798,14 @@ static unsigned getScalingFactorCost(const TargetTransformInfo &TTI,
   switch (LU.Kind) {
   case LSRUse::Address: {
     // Check the scaling factor cost with both the min and max offsets.
-    int ScaleCostMinOffset = TTI.getScalingFactorCost(
+    InstructionCost ScaleCostMinOffset = TTI.getScalingFactorCost(
         LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MinOffset, F.HasBaseReg,
         F.Scale, LU.AccessTy.AddrSpace);
-    int ScaleCostMaxOffset = TTI.getScalingFactorCost(
+    InstructionCost ScaleCostMaxOffset = TTI.getScalingFactorCost(
         LU.AccessTy.MemTy, F.BaseGV, F.BaseOffset + LU.MaxOffset, F.HasBaseReg,
         F.Scale, LU.AccessTy.AddrSpace);
 
-    assert(ScaleCostMinOffset >= 0 && ScaleCostMaxOffset >= 0 &&
+    assert(ScaleCostMinOffset.isValid() && ScaleCostMaxOffset.isValid() &&
            "Legal addressing mode has an illegal cost!");
     return std::max(ScaleCostMinOffset, ScaleCostMaxOffset);
   }
@@ -1908,7 +1944,7 @@ class LSRInstance {
   const TargetTransformInfo &TTI;
   Loop *const L;
   MemorySSAUpdater *MSSAU;
-  bool FavorBackedgeIndex = false;
+  TTI::AddressingModeKind AMK;
   bool Changed = false;
 
   /// This is the insert position that the current loop's induction variable
@@ -1945,6 +1981,9 @@ class LSRInstance {
   /// IV users that belong to profitable IVChains.
   SmallPtrSet<Use*, MaxChains> IVIncSet;
 
+  /// Induction variables that were generated and inserted by the SCEV Expander.
+  SmallVector<llvm::WeakVH, 2> ScalarEvolutionIVs;
+
   void OptimizeShadowIV();
   bool FindIVUserForCond(ICmpInst *Cond, IVStrideUse *&CondUse);
   ICmpInst *OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse);
@@ -2049,6 +2088,9 @@ public:
               TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU);
 
   bool getChanged() const { return Changed; }
+  const SmallVectorImpl<WeakVH> &getScalarEvolutionIVs() const {
+    return ScalarEvolutionIVs;
+  }
 
   void print_factors_and_types(raw_ostream &OS) const;
   void print_fixups(raw_ostream &OS) const;
@@ -2337,6 +2379,7 @@ ICmpInst *LSRInstance::OptimizeMax(ICmpInst *Cond, IVStrideUse* &CondUse) {
     new ICmpInst(Cond, Pred, Cond->getOperand(0), NewRHS, "scmp");
 
   // Delete the max calculation instructions.
+  NewCond->setDebugLoc(Cond->getDebugLoc());
   Cond->replaceAllUsesWith(NewCond);
   CondUse->setUser(NewCond);
   Instruction *Cmp = cast<Instruction>(Sel->getOperand(0));
@@ -2466,7 +2509,7 @@ LSRInstance::OptimizeLoopTermCond() {
     // It's possible for the setcc instruction to be anywhere in the loop, and
     // possible for it to have multiple users.  If it is not immediately before
     // the exiting block branch, move it.
-    if (&*++BasicBlock::iterator(Cond) != TermBr) {
+    if (Cond->getNextNonDebugInstruction() != TermBr) {
       if (Cond->hasOneUse()) {
         Cond->moveBefore(TermBr);
       } else {
@@ -2686,13 +2729,13 @@ void LSRInstance::CollectInterestingTypesAndFactors() {
       if (const SCEVConstant *Factor =
             dyn_cast_or_null<SCEVConstant>(getExactSDiv(NewStride, OldStride,
                                                         SE, true))) {
-        if (Factor->getAPInt().getMinSignedBits() <= 64)
+        if (Factor->getAPInt().getMinSignedBits() <= 64 && !Factor->isZero())
           Factors.insert(Factor->getAPInt().getSExtValue());
       } else if (const SCEVConstant *Factor =
                    dyn_cast_or_null<SCEVConstant>(getExactSDiv(OldStride,
                                                                NewStride,
                                                                SE, true))) {
-        if (Factor->getAPInt().getMinSignedBits() <= 64)
+        if (Factor->getAPInt().getMinSignedBits() <= 64 && !Factor->isZero())
           Factors.insert(Factor->getAPInt().getSExtValue());
       }
     }
@@ -2936,7 +2979,7 @@ void LSRInstance::ChainInstruction(Instruction *UserInst, Instruction *IVOper,
     // The increment must be loop-invariant so it can be kept in a register.
     const SCEV *PrevExpr = SE.getSCEV(PrevIV);
     const SCEV *IncExpr = SE.getMinusSCEV(OperExpr, PrevExpr);
-    if (!SE.isLoopInvariant(IncExpr, L))
+    if (isa<SCEVCouldNotCompute>(IncExpr) || !SE.isLoopInvariant(IncExpr, L))
       continue;
 
     if (Chain.isProfitableIncrement(OperExpr, IncExpr, SE)) {
@@ -3107,7 +3150,7 @@ void LSRInstance::CollectChains() {
 void LSRInstance::FinalizeChain(IVChain &Chain) {
   assert(!Chain.Incs.empty() && "empty IV chains are not allowed");
   LLVM_DEBUG(dbgs() << "Final Chain: " << *Chain.Incs[0].UserInst << "\n");
-  
+
   for (const IVInc &Inc : Chain) {
     LLVM_DEBUG(dbgs() << "        Inc: " << *Inc.UserInst << "\n");
     auto UseI = find(Inc.UserInst->operands(), Inc.IVOperand);
@@ -3289,7 +3332,9 @@ void LSRInstance::CollectFixupsAndInitialFormulae() {
 
         // x == y  -->  x - y == 0
         const SCEV *N = SE.getSCEV(NV);
-        if (SE.isLoopInvariant(N, L) && isSafeToExpand(N, SE)) {
+        if (SE.isLoopInvariant(N, L) && isSafeToExpand(N, SE) &&
+            (!NV->getType()->isPointerTy() ||
+             SE.getPointerBase(N) == SE.getPointerBase(S))) {
           // S is normalized, so normalize N before folding it into S
           // to keep the result normalized.
           N = normalizeForPostIncUse(N, TmpPostIncLoops, SE);
@@ -3420,6 +3465,9 @@ LSRInstance::CollectLoopInvariantFixupsAndFormulae() {
         // Ignore non-instructions.
         if (!UserInst)
           continue;
+        // Don't bother if the instruction is an EHPad.
+        if (UserInst->isEHPad())
+          continue;
         // Ignore instructions in other functions (as can happen with
         // Constants).
         if (UserInst->getParent()->getParent() != L->getHeader()->getParent())
@@ -3575,7 +3623,7 @@ void LSRInstance::GenerateReassociationsImpl(LSRUse &LU, unsigned LUIdx,
   // may generate a post-increment operator. The reason is that the
   // reassociations cause extra base+register formula to be created,
   // and possibly chosen, but the post-increment is more efficient.
-  if (TTI.shouldFavorPostInc() && mayUsePostIncMode(TTI, LU, BaseReg, L, SE))
+  if (AMK == TTI::AMK_PostIndexed && mayUsePostIncMode(TTI, LU, BaseReg, L, SE))
     return;
   SmallVector<const SCEV *, 8> AddOps;
   const SCEV *Remainder = CollectSubexprs(BaseReg, nullptr, AddOps, L, SE);
@@ -3779,8 +3827,7 @@ void LSRInstance::GenerateConstantOffsetsImpl(
     Formula F = Base;
     F.BaseOffset = (uint64_t)Base.BaseOffset - Offset;
 
-    if (isLegalUse(TTI, LU.MinOffset - Offset, LU.MaxOffset - Offset, LU.Kind,
-                   LU.AccessTy, F)) {
+    if (isLegalUse(TTI, LU.MinOffset, LU.MaxOffset, LU.Kind, LU.AccessTy, F)) {
       // Add the offset to the base register.
       const SCEV *NewG = SE.getAddExpr(SE.getConstant(G->getType(), Offset), G);
       // If it cancelled out, drop the base register, otherwise update it.
@@ -3810,7 +3857,7 @@ void LSRInstance::GenerateConstantOffsetsImpl(
   // means that a single pre-indexed access can be generated to become the new
   // base pointer for each iteration of the loop, resulting in no extra add/sub
   // instructions for pointer updating.
-  if (FavorBackedgeIndex && LU.Kind == LSRUse::Address) {
+  if (AMK == TTI::AMK_PreIndexed && LU.Kind == LSRUse::Address) {
     if (auto *GAR = dyn_cast<SCEVAddRecExpr>(G)) {
       if (auto *StepRec =
           dyn_cast<SCEVConstant>(GAR->getStepRecurrence(SE))) {
@@ -3891,6 +3938,7 @@ void LSRInstance::GenerateICmpZeroScales(LSRUse &LU, unsigned LUIdx,
     if (Base.BaseOffset == std::numeric_limits<int64_t>::min() && Factor == -1)
       continue;
     int64_t NewBaseOffset = (uint64_t)Base.BaseOffset * Factor;
+    assert(Factor != 0 && "Zero factor not expected!");
     if (NewBaseOffset / Factor != Base.BaseOffset)
       continue;
     // If the offset will be truncated at this use, check that it is in bounds.
@@ -4031,7 +4079,8 @@ void LSRInstance::GenerateTruncates(LSRUse &LU, unsigned LUIdx, Formula Base) {
   // Determine the integer type for the base formula.
   Type *DstTy = Base.getType();
   if (!DstTy) return;
-  DstTy = SE.getEffectiveSCEVType(DstTy);
+  if (DstTy->isPointerTy())
+    return;
 
   for (Type *SrcTy : Types) {
     if (SrcTy != DstTy && TTI.isTruncateFree(SrcTy, DstTy)) {
@@ -4239,7 +4288,7 @@ void LSRInstance::GenerateCrossUseConstantOffsets() {
           NewF.BaseOffset = (uint64_t)NewF.BaseOffset + Imm;
           if (!isLegalUse(TTI, LU.MinOffset, LU.MaxOffset,
                           LU.Kind, LU.AccessTy, NewF)) {
-            if (TTI.shouldFavorPostInc() &&
+            if (AMK == TTI::AMK_PostIndexed &&
                 mayUsePostIncMode(TTI, LU, OrigReg, this->L, SE))
               continue;
             if (!TTI.isLegalAddImmediate((uint64_t)NewF.UnfoldedOffset + Imm))
@@ -4342,7 +4391,7 @@ void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
       // avoids the need to recompute this information across formulae using the
       // same bad AddRec. Passing LoserRegs is also essential unless we remove
       // the corresponding bad register from the Regs set.
-      Cost CostF(L, SE, TTI);
+      Cost CostF(L, SE, TTI, AMK);
       Regs.clear();
       CostF.RateFormula(F, Regs, VisitedRegs, LU, &LoserRegs);
       if (CostF.isLoser()) {
@@ -4375,7 +4424,7 @@ void LSRInstance::FilterOutUndesirableDedicatedRegisters() {
 
         Formula &Best = LU.Formulae[P.first->second];
 
-        Cost CostBest(L, SE, TTI);
+        Cost CostBest(L, SE, TTI, AMK);
         Regs.clear();
         CostBest.RateFormula(Best, Regs, VisitedRegs, LU);
         if (CostF.isLess(CostBest))
@@ -4495,7 +4544,7 @@ void LSRInstance::NarrowSearchSpaceByDetectingSupersets() {
 /// When there are many registers for expressions like A, A+1, A+2, etc.,
 /// allocate a single register for them.
 void LSRInstance::NarrowSearchSpaceByCollapsingUnrolledCode() {
-  if (EstimateSearchSpaceComplexity() < ComplexityLimit) 
+  if (EstimateSearchSpaceComplexity() < ComplexityLimit)
     return;
 
   LLVM_DEBUG(
@@ -4628,8 +4677,8 @@ void LSRInstance::NarrowSearchSpaceByFilterFormulaWithSameScaledReg() {
 
       // If the new register numbers are the same, choose the Formula with
       // less Cost.
-      Cost CostFA(L, SE, TTI);
-      Cost CostFB(L, SE, TTI);
+      Cost CostFA(L, SE, TTI, AMK);
+      Cost CostFB(L, SE, TTI, AMK);
       Regs.clear();
       CostFA.RateFormula(FA, Regs, VisitedRegs, LU);
       Regs.clear();
@@ -4679,7 +4728,7 @@ void LSRInstance::NarrowSearchSpaceByFilterFormulaWithSameScaledReg() {
 /// If we are over the complexity limit, filter out any post-inc prefering
 /// variables to only post-inc values.
 void LSRInstance::NarrowSearchSpaceByFilterPostInc() {
-  if (!TTI.shouldFavorPostInc())
+  if (AMK != TTI::AMK_PostIndexed)
     return;
   if (EstimateSearchSpaceComplexity() < ComplexityLimit)
     return;
@@ -4970,7 +5019,7 @@ void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
       ReqRegs.insert(S);
 
   SmallPtrSet<const SCEV *, 16> NewRegs;
-  Cost NewCost(L, SE, TTI);
+  Cost NewCost(L, SE, TTI, AMK);
   for (const Formula &F : LU.Formulae) {
     // Ignore formulae which may not be ideal in terms of register reuse of
     // ReqRegs.  The formula should use all required registers before
@@ -4978,7 +5027,7 @@ void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
     // This can sometimes (notably when trying to favour postinc) lead to
     // sub-optimial decisions. There it is best left to the cost modelling to
     // get correct.
-    if (!TTI.shouldFavorPostInc() || LU.Kind != LSRUse::Address) {
+    if (AMK != TTI::AMK_PostIndexed || LU.Kind != LSRUse::Address) {
       int NumReqRegsToFind = std::min(F.getNumRegs(), ReqRegs.size());
       for (const SCEV *Reg : ReqRegs) {
         if ((F.ScaledReg && F.ScaledReg == Reg) ||
@@ -5026,9 +5075,9 @@ void LSRInstance::SolveRecurse(SmallVectorImpl<const Formula *> &Solution,
 /// vector.
 void LSRInstance::Solve(SmallVectorImpl<const Formula *> &Solution) const {
   SmallVector<const Formula *, 8> Workspace;
-  Cost SolutionCost(L, SE, TTI);
+  Cost SolutionCost(L, SE, TTI, AMK);
   SolutionCost.Lose();
-  Cost CurCost(L, SE, TTI);
+  Cost CurCost(L, SE, TTI, AMK);
   SmallPtrSet<const SCEV *, 16> CurRegs;
   DenseSet<const SCEV *> VisitedRegs;
   Workspace.reserve(Uses.size());
@@ -5269,7 +5318,7 @@ Value *LSRInstance::Expand(const LSRUse &LU, const LSRFixup &LF,
   if (F.BaseGV) {
     // Flush the operand list to suppress SCEVExpander hoisting.
     if (!Ops.empty()) {
-      Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), Ty);
+      Value *FullV = Rewriter.expandCodeFor(SE.getAddExpr(Ops), IntTy);
       Ops.clear();
       Ops.push_back(SE.getUnknown(FullV));
     }
@@ -5546,12 +5595,61 @@ void LSRInstance::ImplementSolution(
     GenerateIVChain(Chain, Rewriter, DeadInsts);
     Changed = true;
   }
+
+  for (const WeakVH &IV : Rewriter.getInsertedIVs())
+    if (IV && dyn_cast<Instruction>(&*IV)->getParent())
+      ScalarEvolutionIVs.push_back(IV);
+
   // Clean up after ourselves. This must be done before deleting any
   // instructions.
   Rewriter.clear();
 
   Changed |= RecursivelyDeleteTriviallyDeadInstructionsPermissive(DeadInsts,
                                                                   &TLI, MSSAU);
+
+  // In our cost analysis above, we assume that each addrec consumes exactly
+  // one register, and arrange to have increments inserted just before the
+  // latch to maximimize the chance this is true.  However, if we reused
+  // existing IVs, we now need to move the increments to match our
+  // expectations.  Otherwise, our cost modeling results in us having a
+  // chosen a non-optimal result for the actual schedule.  (And yes, this
+  // scheduling decision does impact later codegen.)
+  for (PHINode &PN : L->getHeader()->phis()) {
+    BinaryOperator *BO = nullptr;
+    Value *Start = nullptr, *Step = nullptr;
+    if (!matchSimpleRecurrence(&PN, BO, Start, Step))
+      continue;
+
+    switch (BO->getOpcode()) {
+    case Instruction::Sub:
+      if (BO->getOperand(0) != &PN)
+        // sub is non-commutative - match handling elsewhere in LSR
+        continue;
+      break;
+    case Instruction::Add:
+      break;
+    default:
+      continue;
+    };
+
+    if (!isa<Constant>(Step))
+      // If not a constant step, might increase register pressure
+      // (We assume constants have been canonicalized to RHS)
+      continue;
+
+    if (BO->getParent() == IVIncInsertPos->getParent())
+      // Only bother moving across blocks.  Isel can handle block local case.
+      continue;
+
+    // Can we legally schedule inc at the desired point?
+    if (!llvm::all_of(BO->uses(),
+                      [&](Use &U) {return DT.dominates(IVIncInsertPos, U);}))
+      continue;
+    BO->moveBefore(IVIncInsertPos);
+    Changed = true;
+  }
+
+
 }
 
 LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
@@ -5559,8 +5657,8 @@ LSRInstance::LSRInstance(Loop *L, IVUsers &IU, ScalarEvolution &SE,
                          const TargetTransformInfo &TTI, AssumptionCache &AC,
                          TargetLibraryInfo &TLI, MemorySSAUpdater *MSSAU)
     : IU(IU), SE(SE), DT(DT), LI(LI), AC(AC), TLI(TLI), TTI(TTI), L(L),
-      MSSAU(MSSAU), FavorBackedgeIndex(EnableBackedgeIndexing &&
-                                       TTI.shouldFavorBackedgeIndex(L)) {
+      MSSAU(MSSAU), AMK(PreferredAddresingMode.getNumOccurrences() > 0 ?
+        PreferredAddresingMode : TTI.getPreferredAddressingMode(L, &SE)) {
   // If LoopSimplify form is not available, stay out of trouble.
   if (!L->isLoopSimplifyForm())
     return;
@@ -5772,61 +5870,389 @@ void LoopStrengthReduce::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addPreserved<MemorySSAWrapperPass>();
 }
 
-using EqualValues = SmallVector<std::tuple<WeakVH, int64_t, DIExpression *>, 4>;
-using EqualValuesMap = DenseMap<DbgValueInst *, EqualValues>;
+struct SCEVDbgValueBuilder {
+  SCEVDbgValueBuilder() = default;
+  SCEVDbgValueBuilder(const SCEVDbgValueBuilder &Base) {
+    Values = Base.Values;
+    Expr = Base.Expr;
+  }
+
+  /// The DIExpression as we translate the SCEV.
+  SmallVector<uint64_t, 6> Expr;
+  /// The location ops of the DIExpression.
+  SmallVector<llvm::ValueAsMetadata *, 2> Values;
+
+  void pushOperator(uint64_t Op) { Expr.push_back(Op); }
+  void pushUInt(uint64_t Operand) { Expr.push_back(Operand); }
+
+  /// Add a DW_OP_LLVM_arg to the expression, followed by the index of the value
+  /// in the set of values referenced by the expression.
+  void pushValue(llvm::Value *V) {
+    Expr.push_back(llvm::dwarf::DW_OP_LLVM_arg);
+    auto *It =
+        std::find(Values.begin(), Values.end(), llvm::ValueAsMetadata::get(V));
+    unsigned ArgIndex = 0;
+    if (It != Values.end()) {
+      ArgIndex = std::distance(Values.begin(), It);
+    } else {
+      ArgIndex = Values.size();
+      Values.push_back(llvm::ValueAsMetadata::get(V));
+    }
+    Expr.push_back(ArgIndex);
+  }
+
+  void pushValue(const SCEVUnknown *U) {
+    llvm::Value *V = cast<SCEVUnknown>(U)->getValue();
+    pushValue(V);
+  }
+
+  void pushConst(const SCEVConstant *C) {
+    Expr.push_back(llvm::dwarf::DW_OP_consts);
+    Expr.push_back(C->getAPInt().getSExtValue());
+  }
+
+  /// Several SCEV types are sequences of the same arithmetic operator applied
+  /// to constants and values that may be extended or truncated.
+  bool pushArithmeticExpr(const llvm::SCEVCommutativeExpr *CommExpr,
+                          uint64_t DwarfOp) {
+    assert((isa<llvm::SCEVAddExpr>(CommExpr) || isa<SCEVMulExpr>(CommExpr)) &&
+           "Expected arithmetic SCEV type");
+    bool Success = true;
+    unsigned EmitOperator = 0;
+    for (auto &Op : CommExpr->operands()) {
+      Success &= pushSCEV(Op);
+
+      if (EmitOperator >= 1)
+        pushOperator(DwarfOp);
+      ++EmitOperator;
+    }
+    return Success;
+  }
+
+  // TODO: Identify and omit noop casts.
+  bool pushCast(const llvm::SCEVCastExpr *C, bool IsSigned) {
+    const llvm::SCEV *Inner = C->getOperand(0);
+    const llvm::Type *Type = C->getType();
+    uint64_t ToWidth = Type->getIntegerBitWidth();
+    bool Success = pushSCEV(Inner);
+    uint64_t CastOps[] = {dwarf::DW_OP_LLVM_convert, ToWidth,
+                          IsSigned ? llvm::dwarf::DW_ATE_signed
+                                   : llvm::dwarf::DW_ATE_unsigned};
+    for (const auto &Op : CastOps)
+      pushOperator(Op);
+    return Success;
+  }
+
+  // TODO: MinMax - although these haven't been encountered in the test suite.
+  bool pushSCEV(const llvm::SCEV *S) {
+    bool Success = true;
+    if (const SCEVConstant *StartInt = dyn_cast<SCEVConstant>(S)) {
+      pushConst(StartInt);
+
+    } else if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(S)) {
+      if(!U->getValue())
+        return false;
+      pushValue(U->getValue());
+
+    } else if (const SCEVMulExpr *MulRec = dyn_cast<SCEVMulExpr>(S)) {
+      Success &= pushArithmeticExpr(MulRec, llvm::dwarf::DW_OP_mul);
+
+    } else if (const SCEVUDivExpr *UDiv = dyn_cast<SCEVUDivExpr>(S)) {
+      Success &= pushSCEV(UDiv->getLHS());
+      Success &= pushSCEV(UDiv->getRHS());
+      pushOperator(llvm::dwarf::DW_OP_div);
+
+    } else if (const SCEVCastExpr *Cast = dyn_cast<SCEVCastExpr>(S)) {
+      // Assert if a new and unknown SCEVCastEXpr type is encountered.
+      assert((isa<SCEVZeroExtendExpr>(Cast) || isa<SCEVTruncateExpr>(Cast) ||
+              isa<SCEVPtrToIntExpr>(Cast) || isa<SCEVSignExtendExpr>(Cast)) &&
+             "Unexpected cast type in SCEV.");
+      Success &= pushCast(Cast, (isa<SCEVSignExtendExpr>(Cast)));
+
+    } else if (const SCEVAddExpr *AddExpr = dyn_cast<SCEVAddExpr>(S)) {
+      Success &= pushArithmeticExpr(AddExpr, llvm::dwarf::DW_OP_plus);
+
+    } else if (isa<SCEVAddRecExpr>(S)) {
+      // Nested SCEVAddRecExpr are generated by nested loops and are currently
+      // unsupported.
+      return false;
+
+    } else {
+      return false;
+    }
+    return Success;
+  }
+
+  void setFinalExpression(llvm::DbgValueInst &DI, const DIExpression *OldExpr) {
+    // Re-state assumption that this dbg.value is not variadic. Any remaining
+    // opcodes in its expression operate on a single value already on the
+    // expression stack. Prepend our operations, which will re-compute and
+    // place that value on the expression stack.
+    assert(!DI.hasArgList());
+    auto *NewExpr =
+        DIExpression::prependOpcodes(OldExpr, Expr, /*StackValue*/ true);
+    DI.setExpression(NewExpr);
+
+    auto ValArrayRef = llvm::ArrayRef<llvm::ValueAsMetadata *>(Values);
+    DI.setRawLocation(llvm::DIArgList::get(DI.getContext(), ValArrayRef));
+  }
+
+  /// If a DVI can be emitted without a DIArgList, omit DW_OP_llvm_arg and the
+  /// location op index 0.
+  void setShortFinalExpression(llvm::DbgValueInst &DI,
+                               const DIExpression *OldExpr) {
+    assert((Expr[0] == llvm::dwarf::DW_OP_LLVM_arg && Expr[1] == 0) &&
+           "Expected DW_OP_llvm_arg and 0.");
+    DI.replaceVariableLocationOp(
+        0u, llvm::MetadataAsValue::get(DI.getContext(), Values[0]));
+
+    // See setFinalExpression: prepend our opcodes on the start of any old
+    // expression opcodes.
+    assert(!DI.hasArgList());
+    llvm::SmallVector<uint64_t, 6> FinalExpr(Expr.begin() + 2, Expr.end());
+    auto *NewExpr =
+        DIExpression::prependOpcodes(OldExpr, FinalExpr, /*StackValue*/ true);
+    DI.setExpression(NewExpr);
+  }
+
+  /// Once the IV and variable SCEV translation is complete, write it to the
+  /// source DVI.
+  void applyExprToDbgValue(llvm::DbgValueInst &DI,
+                           const DIExpression *OldExpr) {
+    assert(!Expr.empty() && "Unexpected empty expression.");
+    // Emit a simpler form if only a single location is referenced.
+    if (Values.size() == 1 && Expr[0] == llvm::dwarf::DW_OP_LLVM_arg &&
+        Expr[1] == 0) {
+      setShortFinalExpression(DI, OldExpr);
+    } else {
+      setFinalExpression(DI, OldExpr);
+    }
+  }
+
+  /// Return true if the combination of arithmetic operator and underlying
+  /// SCEV constant value is an identity function.
+  bool isIdentityFunction(uint64_t Op, const SCEV *S) {
+    if (const SCEVConstant *C = dyn_cast<SCEVConstant>(S)) {
+      int64_t I = C->getAPInt().getSExtValue();
+      switch (Op) {
+      case llvm::dwarf::DW_OP_plus:
+      case llvm::dwarf::DW_OP_minus:
+        return I == 0;
+      case llvm::dwarf::DW_OP_mul:
+      case llvm::dwarf::DW_OP_div:
+        return I == 1;
+      }
+    }
+    return false;
+  }
+
+  /// Convert a SCEV of a value to a DIExpression that is pushed onto the
+  /// builder's expression stack. The stack should already contain an
+  /// expression for the iteration count, so that it can be multiplied by
+  /// the stride and added to the start.
+  /// Components of the expression are omitted if they are an identity function.
+  /// Chain (non-affine) SCEVs are not supported.
+  bool SCEVToValueExpr(const llvm::SCEVAddRecExpr &SAR, ScalarEvolution &SE) {
+    assert(SAR.isAffine() && "Expected affine SCEV");
+    // TODO: Is this check needed?
+    if (isa<SCEVAddRecExpr>(SAR.getStart()))
+      return false;
+
+    const SCEV *Start = SAR.getStart();
+    const SCEV *Stride = SAR.getStepRecurrence(SE);
+
+    // Skip pushing arithmetic noops.
+    if (!isIdentityFunction(llvm::dwarf::DW_OP_mul, Stride)) {
+      if (!pushSCEV(Stride))
+        return false;
+      pushOperator(llvm::dwarf::DW_OP_mul);
+    }
+    if (!isIdentityFunction(llvm::dwarf::DW_OP_plus, Start)) {
+      if (!pushSCEV(Start))
+        return false;
+      pushOperator(llvm::dwarf::DW_OP_plus);
+    }
+    return true;
+  }
+
+  /// Convert a SCEV of a value to a DIExpression that is pushed onto the
+  /// builder's expression stack. The stack should already contain an
+  /// expression for the iteration count, so that it can be multiplied by
+  /// the stride and added to the start.
+  /// Components of the expression are omitted if they are an identity function.
+  bool SCEVToIterCountExpr(const llvm::SCEVAddRecExpr &SAR,
+                           ScalarEvolution &SE) {
+    assert(SAR.isAffine() && "Expected affine SCEV");
+    if (isa<SCEVAddRecExpr>(SAR.getStart())) {
+      LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV. Unsupported nested AddRec: "
+                        << SAR << '\n');
+      return false;
+    }
+    const SCEV *Start = SAR.getStart();
+    const SCEV *Stride = SAR.getStepRecurrence(SE);
+
+    // Skip pushing arithmetic noops.
+    if (!isIdentityFunction(llvm::dwarf::DW_OP_minus, Start)) {
+      if (!pushSCEV(Start))
+        return false;
+      pushOperator(llvm::dwarf::DW_OP_minus);
+    }
+    if (!isIdentityFunction(llvm::dwarf::DW_OP_div, Stride)) {
+      if (!pushSCEV(Stride))
+        return false;
+      pushOperator(llvm::dwarf::DW_OP_div);
+    }
+    return true;
+  }
+};
+
+struct DVIRecoveryRec {
+  DbgValueInst *DVI;
+  DIExpression *Expr;
+  Metadata *LocationOp;
+  const llvm::SCEV *SCEV;
+};
+
+static bool RewriteDVIUsingIterCount(DVIRecoveryRec CachedDVI,
+                                     const SCEVDbgValueBuilder &IterationCount,
+                                     ScalarEvolution &SE) {
+  // LSR may add locations to previously single location-op DVIs which
+  // are currently not supported.
+  if (CachedDVI.DVI->getNumVariableLocationOps() != 1)
+    return false;
+
+  // SCEVs for SSA values are most frquently of the form
+  // {start,+,stride}, but sometimes they are ({start,+,stride} + %a + ..).
+  // This is because %a is a PHI node that is not the IV. However, these
+  // SCEVs have not been observed to result in debuginfo-lossy optimisations,
+  // so its not expected this point will be reached.
+  if (!isa<SCEVAddRecExpr>(CachedDVI.SCEV))
+    return false;
+
+  LLVM_DEBUG(dbgs() << "scev-salvage: Value to salvage SCEV: "
+                    << *CachedDVI.SCEV << '\n');
+
+  const auto *Rec = cast<SCEVAddRecExpr>(CachedDVI.SCEV);
+  if (!Rec->isAffine())
+    return false;
+
+  // Initialise a new builder with the iteration count expression. In
+  // combination with the value's SCEV this enables recovery.
+  SCEVDbgValueBuilder RecoverValue(IterationCount);
+  if (!RecoverValue.SCEVToValueExpr(*Rec, SE))
+    return false;
+
+  LLVM_DEBUG(dbgs() << "scev-salvage: Updating: " << *CachedDVI.DVI << '\n');
+  RecoverValue.applyExprToDbgValue(*CachedDVI.DVI, CachedDVI.Expr);
+  LLVM_DEBUG(dbgs() << "scev-salvage: to: " << *CachedDVI.DVI << '\n');
+  return true;
+}
+
+static bool
+DbgRewriteSalvageableDVIs(llvm::Loop *L, ScalarEvolution &SE,
+                          llvm::PHINode *LSRInductionVar,
+                          SmallVector<DVIRecoveryRec, 2> &DVIToUpdate) {
+  if (DVIToUpdate.empty())
+    return false;
+
+  const llvm::SCEV *SCEVInductionVar = SE.getSCEV(LSRInductionVar);
+  assert(SCEVInductionVar &&
+         "Anticipated a SCEV for the post-LSR induction variable");
+
+  bool Changed = false;
+  if (const SCEVAddRecExpr *IVAddRec =
+          dyn_cast<SCEVAddRecExpr>(SCEVInductionVar)) {
+    SCEVDbgValueBuilder IterCountExpr;
+    IterCountExpr.pushValue(LSRInductionVar);
+    if (!IterCountExpr.SCEVToIterCountExpr(*IVAddRec, SE))
+      return false;
+
+    LLVM_DEBUG(dbgs() << "scev-salvage: IV SCEV: " << *SCEVInductionVar
+                      << '\n');
+
+    // Needn't salvage if the location op hasn't been undef'd by LSR.
+    for (auto &DVIRec : DVIToUpdate) {
+      if (!DVIRec.DVI->isUndef())
+        continue;
+
+      // Some DVIs that were single location-op when cached are now multi-op,
+      // due to LSR optimisations. However, multi-op salvaging is not yet
+      // supported by SCEV salvaging. But, we can attempt a salvage by restoring
+      // the pre-LSR single-op expression.
+      if (DVIRec.DVI->hasArgList()) {
+        llvm::Type *Ty = DVIRec.DVI->getVariableLocationOp(0)->getType();
+        DVIRec.DVI->setRawLocation(
+            llvm::ValueAsMetadata::get(UndefValue::get(Ty)));
+        DVIRec.DVI->setExpression(DVIRec.Expr);
+      }
+
+      Changed |= RewriteDVIUsingIterCount(DVIRec, IterCountExpr, SE);
+    }
+  }
+  return Changed;
+}
 
-static void DbgGatherEqualValues(Loop *L, ScalarEvolution &SE,
-                                 EqualValuesMap &DbgValueToEqualSet) {
+/// Identify and cache salvageable DVI locations and expressions along with the
+/// corresponding SCEV(s). Also ensure that the DVI is not deleted before
+static void
+DbgGatherSalvagableDVI(Loop *L, ScalarEvolution &SE,
+                       SmallVector<DVIRecoveryRec, 2> &SalvageableDVISCEVs,
+                       SmallSet<AssertingVH<DbgValueInst>, 2> &DVIHandles) {
   for (auto &B : L->getBlocks()) {
     for (auto &I : *B) {
       auto DVI = dyn_cast<DbgValueInst>(&I);
       if (!DVI)
         continue;
-      auto V = DVI->getVariableLocation();
-      if (!V || !SE.isSCEVable(V->getType()))
+
+      if (DVI->hasArgList())
         continue;
-      auto DbgValueSCEV = SE.getSCEV(V);
-      EqualValues EqSet;
-      for (PHINode &Phi : L->getHeader()->phis()) {
-        if (V->getType() != Phi.getType())
-          continue;
-        if (!SE.isSCEVable(Phi.getType()))
-          continue;
-        auto PhiSCEV = SE.getSCEV(&Phi);
-        Optional<APInt> Offset =
-                SE.computeConstantDifference(DbgValueSCEV, PhiSCEV);
-        if (Offset && Offset->getMinSignedBits() <= 64)
-          EqSet.emplace_back(std::make_tuple(
-              &Phi, Offset.getValue().getSExtValue(), DVI->getExpression()));
-      }
-      DbgValueToEqualSet[DVI] = std::move(EqSet);
+
+      if (!SE.isSCEVable(DVI->getVariableLocationOp(0)->getType()))
+        continue;
+
+      SalvageableDVISCEVs.push_back(
+          {DVI, DVI->getExpression(), DVI->getRawLocation(),
+           SE.getSCEV(DVI->getVariableLocationOp(0))});
+      DVIHandles.insert(DVI);
     }
   }
 }
 
-static void DbgApplyEqualValues(EqualValuesMap &DbgValueToEqualSet) {
-  for (auto A : DbgValueToEqualSet) {
-    auto DVI = A.first;
-    // Only update those that are now undef.
-    if (!isa_and_nonnull<UndefValue>(DVI->getVariableLocation()))
+/// Ideally pick the PHI IV inserted by ScalarEvolutionExpander. As a fallback
+/// any PHi from the loop header is usable, but may have less chance of
+/// surviving subsequent transforms.
+static llvm::PHINode *GetInductionVariable(const Loop &L, ScalarEvolution &SE,
+                                           const LSRInstance &LSR) {
+  // For now, just pick the first IV generated and inserted. Ideally pick an IV
+  // that is unlikely to be optimised away by subsequent transforms.
+  for (const WeakVH &IV : LSR.getScalarEvolutionIVs()) {
+    if (!IV)
       continue;
-    for (auto EV : A.second) {
-      auto V = std::get<WeakVH>(EV);
-      if (!V)
-        continue;
-      auto DbgDIExpr = std::get<DIExpression *>(EV);
-      auto Offset = std::get<int64_t>(EV);
-      auto &Ctx = DVI->getContext();
-      DVI->setOperand(0, MetadataAsValue::get(Ctx, ValueAsMetadata::get(V)));
-      if (Offset) {
-        SmallVector<uint64_t, 8> Ops;
-        DIExpression::appendOffset(Ops, Offset);
-        DbgDIExpr = DIExpression::prependOpcodes(DbgDIExpr, Ops, true);
-      }
-      DVI->setOperand(2, MetadataAsValue::get(Ctx, DbgDIExpr));
-      break;
+
+    assert(isa<PHINode>(&*IV) && "Expected PhI node.");
+    if (SE.isSCEVable((*IV).getType())) {
+      PHINode *Phi = dyn_cast<PHINode>(&*IV);
+      LLVM_DEBUG(const llvm::SCEV *S = SE.getSCEV(Phi);
+                 dbgs() << "scev-salvage: IV : " << *IV << "with SCEV: " << *S
+                 << "\n");
+      return Phi;
     }
   }
+
+  for (PHINode &Phi : L.getHeader()->phis()) {
+    if (!SE.isSCEVable(Phi.getType()))
+      continue;
+
+    const llvm::SCEV *PhiSCEV = SE.getSCEV(&Phi);
+    if (const llvm::SCEVAddRecExpr *Rec = dyn_cast<SCEVAddRecExpr>(PhiSCEV))
+      if (!Rec->isAffine())
+        continue;
+
+    LLVM_DEBUG(dbgs() << "scev-salvage: Selected IV from loop header: " << Phi
+                      << " with SCEV: " << *PhiSCEV << "\n");
+    return &Phi;
+  }
+  return nullptr;
 }
 
 static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
@@ -5835,19 +6261,21 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
                                AssumptionCache &AC, TargetLibraryInfo &TLI,
                                MemorySSA *MSSA) {
 
+  // Debug preservation - before we start removing anything identify which DVI
+  // meet the salvageable criteria and store their DIExpression and SCEVs.
+  SmallVector<DVIRecoveryRec, 2> SalvageableDVI;
+  SmallSet<AssertingVH<DbgValueInst>, 2> DVIHandles;
+  DbgGatherSalvagableDVI(L, SE, SalvageableDVI, DVIHandles);
+
   bool Changed = false;
   std::unique_ptr<MemorySSAUpdater> MSSAU;
   if (MSSA)
     MSSAU = std::make_unique<MemorySSAUpdater>(MSSA);
 
   // Run the main LSR transformation.
-  Changed |=
-      LSRInstance(L, IU, SE, DT, LI, TTI, AC, TLI, MSSAU.get()).getChanged();
-
-  // Debug preservation - before we start removing anything create equivalence
-  // sets for the llvm.dbg.value intrinsics.
-  EqualValuesMap DbgValueToEqualSet;
-  DbgGatherEqualValues(L, SE, DbgValueToEqualSet);
+  const LSRInstance &Reducer =
+      LSRInstance(L, IU, SE, DT, LI, TTI, AC, TLI, MSSAU.get());
+  Changed |= Reducer.getChanged();
 
   // Remove any extra phis created by processing inner loops.
   Changed |= DeleteDeadPHIs(L->getHeader(), &TLI, MSSAU.get());
@@ -5867,8 +6295,22 @@ static bool ReduceLoopStrength(Loop *L, IVUsers &IU, ScalarEvolution &SE,
     }
   }
 
-  DbgApplyEqualValues(DbgValueToEqualSet);
+  if (SalvageableDVI.empty())
+    return Changed;
+
+  // Obtain relevant IVs and attempt to rewrite the salvageable DVIs with
+  // expressions composed using the derived iteration count.
+  // TODO: Allow for multiple IV references for nested AddRecSCEVs
+  for (auto &L : LI) {
+    if (llvm::PHINode *IV = GetInductionVariable(*L, SE, Reducer))
+      DbgRewriteSalvageableDVIs(L, SE, IV, SalvageableDVI);
+    else {
+      LLVM_DEBUG(dbgs() << "scev-salvage: SCEV salvaging not possible. An IV "
+                           "could not be identified.\n");
+    }
+  }
 
+  DVIHandles.clear();
   return Changed;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
index 495906e1a763..71eb393fcdd7 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/LoopAnalysisManager.h"
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
@@ -41,13 +42,14 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/LCSSA.h"
 #include "llvm/Transforms/Utils/LoopPeel.h"
 #include "llvm/Transforms/Utils/LoopSimplify.h"
 #include "llvm/Transforms/Utils/LoopUtils.h"
 #include "llvm/Transforms/Utils/UnrollLoop.h"
 #include <cassert>
 #include <cstdint>
-#include <vector>
 
 namespace llvm {
 class Instruction;
@@ -424,35 +426,29 @@ tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI,
   return UnrollResult;
 }
 
-static bool tryToUnrollAndJamLoop(Function &F, DominatorTree &DT, LoopInfo &LI,
+static bool tryToUnrollAndJamLoop(LoopNest &LN, DominatorTree &DT, LoopInfo &LI,
                                   ScalarEvolution &SE,
                                   const TargetTransformInfo &TTI,
                                   AssumptionCache &AC, DependenceInfo &DI,
-                                  OptimizationRemarkEmitter &ORE,
-                                  int OptLevel) {
+                                  OptimizationRemarkEmitter &ORE, int OptLevel,
+                                  LPMUpdater &U) {
   bool DidSomething = false;
+  ArrayRef<Loop *> Loops = LN.getLoops();
+  Loop *OutmostLoop = &LN.getOutermostLoop();
 
-  // The loop unroll and jam pass requires loops to be in simplified form, and
-  // also needs LCSSA. Since simplification may add new inner loops, it has to
-  // run before the legality and profitability checks. This means running the
-  // loop unroll and jam pass will simplify all loops, regardless of whether
-  // anything end up being unroll and jammed.
-  for (auto &L : LI) {
-    DidSomething |=
-        simplifyLoop(L, &DT, &LI, &SE, &AC, nullptr, false /* PreserveLCSSA */);
-    DidSomething |= formLCSSARecursively(*L, DT, &LI, &SE);
-  }
-
-  // Add the loop nests in the reverse order of LoopInfo. See method
+  // Add the loop nests in the reverse order of LN. See method
   // declaration.
   SmallPriorityWorklist<Loop *, 4> Worklist;
-  appendLoopsToWorklist(LI, Worklist);
+  appendLoopsToWorklist(Loops, Worklist);
   while (!Worklist.empty()) {
     Loop *L = Worklist.pop_back_val();
+    std::string LoopName = std::string(L->getName());
     LoopUnrollResult Result =
         tryToUnrollAndJamLoop(L, DT, &LI, SE, TTI, AC, DI, ORE, OptLevel);
     if (Result != LoopUnrollResult::Unmodified)
       DidSomething = true;
+    if (L == OutmostLoop && Result == LoopUnrollResult::FullyUnrolled)
+      U.markLoopAsDeleted(*L, LoopName);
   }
 
   return DidSomething;
@@ -460,29 +456,35 @@ static bool tryToUnrollAndJamLoop(Function &F, DominatorTree &DT, LoopInfo &LI,
 
 namespace {
 
-class LoopUnrollAndJam : public FunctionPass {
+class LoopUnrollAndJam : public LoopPass {
 public:
   static char ID; // Pass ID, replacement for typeid
   unsigned OptLevel;
 
-  LoopUnrollAndJam(int OptLevel = 2) : FunctionPass(ID), OptLevel(OptLevel) {
+  LoopUnrollAndJam(int OptLevel = 2) : LoopPass(ID), OptLevel(OptLevel) {
     initializeLoopUnrollAndJamPass(*PassRegistry::getPassRegistry());
   }
 
-  bool runOnFunction(Function &F) override {
-    if (skipFunction(F))
+  bool runOnLoop(Loop *L, LPPassManager &LPM) override {
+    if (skipLoop(L))
       return false;
 
-    auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
-    LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
-    ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
-    const TargetTransformInfo &TTI =
-        getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
-    auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+    auto *F = L->getHeader()->getParent();
+    auto &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
+    auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
     auto &DI = getAnalysis<DependenceAnalysisWrapperPass>().getDI();
+    auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+    auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(*F);
     auto &ORE = getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE();
+    auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(*F);
 
-    return tryToUnrollAndJamLoop(F, DT, LI, SE, TTI, AC, DI, ORE, OptLevel);
+    LoopUnrollResult Result =
+        tryToUnrollAndJamLoop(L, DT, LI, SE, TTI, AC, DI, ORE, OptLevel);
+
+    if (Result == LoopUnrollResult::FullyUnrolled)
+      LPM.markLoopAsDeleted(*L);
+
+    return Result != LoopUnrollResult::Unmodified;
   }
 
   /// This transformation requires natural loop information & requires that
@@ -495,6 +497,7 @@ public:
     AU.addRequired<AssumptionCacheTracker>();
     AU.addRequired<DependenceAnalysisWrapperPass>();
     AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
+    getLoopAnalysisUsage(AU);
   }
 };
 
@@ -505,7 +508,10 @@ char LoopUnrollAndJam::ID = 0;
 INITIALIZE_PASS_BEGIN(LoopUnrollAndJam, "loop-unroll-and-jam",
                       "Unroll and Jam loops", false, false)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopPass)
 INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
+INITIALIZE_PASS_DEPENDENCY(LCSSAWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
@@ -518,19 +524,20 @@ Pass *llvm::createLoopUnrollAndJamPass(int OptLevel) {
   return new LoopUnrollAndJam(OptLevel);
 }
 
-PreservedAnalyses LoopUnrollAndJamPass::run(Function &F,
-                                            FunctionAnalysisManager &AM) {
-  ScalarEvolution &SE = AM.getResult<ScalarEvolutionAnalysis>(F);
-  LoopInfo &LI = AM.getResult<LoopAnalysis>(F);
-  TargetTransformInfo &TTI = AM.getResult<TargetIRAnalysis>(F);
-  AssumptionCache &AC = AM.getResult<AssumptionAnalysis>(F);
-  DominatorTree &DT = AM.getResult<DominatorTreeAnalysis>(F);
-  DependenceInfo &DI = AM.getResult<DependenceAnalysis>(F);
-  OptimizationRemarkEmitter &ORE =
-      AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
+PreservedAnalyses LoopUnrollAndJamPass::run(LoopNest &LN,
+                                            LoopAnalysisManager &AM,
+                                            LoopStandardAnalysisResults &AR,
+                                            LPMUpdater &U) {
+  Function &F = *LN.getParent();
+
+  DependenceInfo DI(&F, &AR.AA, &AR.SE, &AR.LI);
+  OptimizationRemarkEmitter ORE(&F);
 
-  if (!tryToUnrollAndJamLoop(F, DT, LI, SE, TTI, AC, DI, ORE, OptLevel))
+  if (!tryToUnrollAndJamLoop(LN, AR.DT, AR.LI, AR.SE, AR.TTI, AR.AC, DI, ORE,
+                             OptLevel, U))
     return PreservedAnalyses::all();
 
-  return getLoopPassPreservedAnalyses();
+  auto PA = getLoopPassPreservedAnalyses();
+  PA.preserve<LoopNestAnalysis>();
+  return PA;
 }
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 1b974576a3cc..49501f324a49 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -356,19 +356,19 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
 
   SmallSetVector<BasicBlock *, 16> BBWorklist;
   SmallSetVector<std::pair<BasicBlock *, BasicBlock *>, 4> ExitWorklist;
-  DenseMap<Value *, Constant *> SimplifiedValues;
-  SmallVector<std::pair<Value *, Constant *>, 4> SimplifiedInputValues;
+  DenseMap<Value *, Value *> SimplifiedValues;
+  SmallVector<std::pair<Value *, Value *>, 4> SimplifiedInputValues;
 
   // The estimated cost of the unrolled form of the loop. We try to estimate
   // this by simplifying as much as we can while computing the estimate.
-  unsigned UnrolledCost = 0;
+  InstructionCost UnrolledCost = 0;
 
   // We also track the estimated dynamic (that is, actually executed) cost in
   // the rolled form. This helps identify cases when the savings from unrolling
   // aren't just exposing dead control flows, but actual reduced dynamic
   // instructions due to the simplifications which we expect to occur after
   // unrolling.
-  unsigned RolledDynamicCost = 0;
+  InstructionCost RolledDynamicCost = 0;
 
   // We track the simplification of each instruction in each iteration. We use
   // this to recursively merge costs into the unrolled cost on-demand so that
@@ -498,11 +498,9 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
 
       Value *V = PHI->getIncomingValueForBlock(
           Iteration == 0 ? L->getLoopPreheader() : L->getLoopLatch());
-      Constant *C = dyn_cast<Constant>(V);
-      if (Iteration != 0 && !C)
-        C = SimplifiedValues.lookup(V);
-      if (C)
-        SimplifiedInputValues.push_back({PHI, C});
+      if (Iteration != 0 && SimplifiedValues.count(V))
+        V = SimplifiedValues.lookup(V);
+      SimplifiedInputValues.push_back({PHI, V});
     }
 
     // Now clear and re-populate the map for the next iteration.
@@ -571,13 +569,18 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
 
       Instruction *TI = BB->getTerminator();
 
+      auto getSimplifiedConstant = [&](Value *V) -> Constant * {
+        if (SimplifiedValues.count(V))
+          V = SimplifiedValues.lookup(V);
+        return dyn_cast<Constant>(V);
+      };
+
       // Add in the live successors by first checking whether we have terminator
       // that may be simplified based on the values simplified by this call.
       BasicBlock *KnownSucc = nullptr;
       if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
         if (BI->isConditional()) {
-          if (Constant *SimpleCond =
-                  SimplifiedValues.lookup(BI->getCondition())) {
+          if (auto *SimpleCond = getSimplifiedConstant(BI->getCondition())) {
             // Just take the first successor if condition is undef
             if (isa<UndefValue>(SimpleCond))
               KnownSucc = BI->getSuccessor(0);
@@ -587,8 +590,7 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
           }
         }
       } else if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
-        if (Constant *SimpleCond =
-                SimplifiedValues.lookup(SI->getCondition())) {
+        if (auto *SimpleCond = getSimplifiedConstant(SI->getCondition())) {
           // Just take the first successor if condition is undef
           if (isa<UndefValue>(SimpleCond))
             KnownSucc = SI->getSuccessor(0);
@@ -639,10 +641,15 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
     }
   }
 
+  assert(UnrolledCost.isValid() && RolledDynamicCost.isValid() &&
+         "All instructions must have a valid cost, whether the "
+         "loop is rolled or unrolled.");
+
   LLVM_DEBUG(dbgs() << "Analysis finished:\n"
                     << "UnrolledCost: " << UnrolledCost << ", "
                     << "RolledDynamicCost: " << RolledDynamicCost << "\n");
-  return {{UnrolledCost, RolledDynamicCost}};
+  return {{unsigned(*UnrolledCost.getValue()),
+           unsigned(*RolledDynamicCost.getValue())}};
 }
 
 /// ApproximateLoopSize - Approximate the size of the loop.
@@ -727,13 +734,24 @@ static unsigned getFullUnrollBoostingFactor(const EstimatedUnrollCost &Cost,
     return MaxPercentThresholdBoost;
 }
 
-// Returns loop size estimation for unrolled loop.
-static uint64_t getUnrolledLoopSize(
-    unsigned LoopSize,
-    TargetTransformInfo::UnrollingPreferences &UP) {
-  assert(LoopSize >= UP.BEInsns && "LoopSize should not be less than BEInsns!");
-  return (uint64_t)(LoopSize - UP.BEInsns) * UP.Count + UP.BEInsns;
-}
+// Produce an estimate of the unrolled cost of the specified loop.  This
+// is used to a) produce a cost estimate for partial unrolling and b) to
+// cheaply estimate cost for full unrolling when we don't want to symbolically
+// evaluate all iterations.
+class UnrollCostEstimator {
+  const unsigned LoopSize;
+
+public:
+  UnrollCostEstimator(Loop &L, unsigned LoopSize) : LoopSize(LoopSize) {}
+
+  // Returns loop size estimation for unrolled loop, given the unrolling
+  // configuration specified by UP.
+  uint64_t getUnrolledLoopSize(TargetTransformInfo::UnrollingPreferences &UP) {
+    assert(LoopSize >= UP.BEInsns &&
+           "LoopSize should not be less than BEInsns!");
+    return (uint64_t)(LoopSize - UP.BEInsns) * UP.Count + UP.BEInsns;
+  }
+};
 
 // Returns true if unroll count was set explicitly.
 // Calculates unroll count and writes it to UP.Count.
@@ -746,11 +764,25 @@ static uint64_t getUnrolledLoopSize(
 bool llvm::computeUnrollCount(
     Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI,
     ScalarEvolution &SE, const SmallPtrSetImpl<const Value *> &EphValues,
-    OptimizationRemarkEmitter *ORE, unsigned &TripCount, unsigned MaxTripCount,
-    bool MaxOrZero, unsigned &TripMultiple, unsigned LoopSize,
+    OptimizationRemarkEmitter *ORE, unsigned TripCount, unsigned MaxTripCount,
+    bool MaxOrZero, unsigned TripMultiple, unsigned LoopSize,
     TargetTransformInfo::UnrollingPreferences &UP,
     TargetTransformInfo::PeelingPreferences &PP, bool &UseUpperBound) {
 
+  UnrollCostEstimator UCE(*L, LoopSize);
+
+  // Use an explicit peel count that has been specified for testing. In this
+  // case it's not permitted to also specify an explicit unroll count.
+  if (PP.PeelCount) {
+    if (UnrollCount.getNumOccurrences() > 0) {
+      report_fatal_error("Cannot specify both explicit peel count and "
+                         "explicit unroll count");
+    }
+    UP.Count = 1;
+    UP.Runtime = false;
+    return true;
+  }
+
   // Check for explicit Count.
   // 1st priority is unroll count set by "unroll-count" option.
   bool UserUnrollCount = UnrollCount.getNumOccurrences() > 0;
@@ -758,7 +790,7 @@ bool llvm::computeUnrollCount(
     UP.Count = UnrollCount;
     UP.AllowExpensiveTripCount = true;
     UP.Force = true;
-    if (UP.AllowRemainder && getUnrolledLoopSize(LoopSize, UP) < UP.Threshold)
+    if (UP.AllowRemainder && UCE.getUnrolledLoopSize(UP) < UP.Threshold)
       return true;
   }
 
@@ -770,13 +802,13 @@ bool llvm::computeUnrollCount(
     UP.AllowExpensiveTripCount = true;
     UP.Force = true;
     if ((UP.AllowRemainder || (TripMultiple % PragmaCount == 0)) &&
-        getUnrolledLoopSize(LoopSize, UP) < PragmaUnrollThreshold)
+        UCE.getUnrolledLoopSize(UP) < PragmaUnrollThreshold)
       return true;
   }
   bool PragmaFullUnroll = hasUnrollFullPragma(L);
   if (PragmaFullUnroll && TripCount != 0) {
     UP.Count = TripCount;
-    if (getUnrolledLoopSize(LoopSize, UP) < PragmaUnrollThreshold)
+    if (UCE.getUnrolledLoopSize(UP) < PragmaUnrollThreshold)
       return false;
   }
 
@@ -797,8 +829,6 @@ bool llvm::computeUnrollCount(
   // Full unroll makes sense only when TripCount or its upper bound could be
   // statically calculated.
   // Also we need to check if we exceed FullUnrollMaxCount.
-  // If using the upper bound to unroll, TripMultiple should be set to 1 because
-  // we do not know when loop may exit.
 
   // We can unroll by the upper bound amount if it's generally allowed or if
   // we know that the loop is executed either the upper bound or zero times.
@@ -825,10 +855,8 @@ bool llvm::computeUnrollCount(
   if (FullUnrollTripCount && FullUnrollTripCount <= UP.FullUnrollMaxCount) {
     // When computing the unrolled size, note that BEInsns are not replicated
     // like the rest of the loop body.
-    if (getUnrolledLoopSize(LoopSize, UP) < UP.Threshold) {
+    if (UCE.getUnrolledLoopSize(UP) < UP.Threshold) {
       UseUpperBound = (FullUnrollMaxTripCount == FullUnrollTripCount);
-      TripCount = FullUnrollTripCount;
-      TripMultiple = UP.UpperBound ? 1 : TripMultiple;
       return ExplicitUnroll;
     } else {
       // The loop isn't that small, but we still can fully unroll it if that
@@ -842,8 +870,6 @@ bool llvm::computeUnrollCount(
             getFullUnrollBoostingFactor(*Cost, UP.MaxPercentThresholdBoost);
         if (Cost->UnrolledCost < UP.Threshold * Boost / 100) {
           UseUpperBound = (FullUnrollMaxTripCount == FullUnrollTripCount);
-          TripCount = FullUnrollTripCount;
-          TripMultiple = UP.UpperBound ? 1 : TripMultiple;
           return ExplicitUnroll;
         }
       }
@@ -872,7 +898,7 @@ bool llvm::computeUnrollCount(
       UP.Count = TripCount;
     if (UP.PartialThreshold != NoThreshold) {
       // Reduce unroll count to be modulo of TripCount for partial unrolling.
-      if (getUnrolledLoopSize(LoopSize, UP) > UP.PartialThreshold)
+      if (UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold)
         UP.Count =
             (std::max(UP.PartialThreshold, UP.BEInsns + 1) - UP.BEInsns) /
             (LoopSize - UP.BEInsns);
@@ -887,7 +913,7 @@ bool llvm::computeUnrollCount(
         // remainder loop is allowed.
         UP.Count = UP.DefaultUnrollRuntimeCount;
         while (UP.Count != 0 &&
-               getUnrolledLoopSize(LoopSize, UP) > UP.PartialThreshold)
+               UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold)
           UP.Count >>= 1;
       }
       if (UP.Count < 2) {
@@ -971,7 +997,7 @@ bool llvm::computeUnrollCount(
   // Reduce unroll count to be the largest power-of-two factor of
   // the original count which satisfies the threshold limit.
   while (UP.Count != 0 &&
-         getUnrolledLoopSize(LoopSize, UP) > UP.PartialThreshold)
+         UCE.getUnrolledLoopSize(UP) > UP.PartialThreshold)
     UP.Count >>= 1;
 
 #ifndef NDEBUG
@@ -1088,18 +1114,29 @@ static LoopUnrollResult tryToUnrollLoop(
     return LoopUnrollResult::Unmodified;
   }
 
-  // Find trip count and trip multiple if count is not available
+  // Find the smallest exact trip count for any exit. This is an upper bound
+  // on the loop trip count, but an exit at an earlier iteration is still
+  // possible. An unroll by the smallest exact trip count guarantees that all
+  // brnaches relating to at least one exit can be eliminated. This is unlike
+  // the max trip count, which only guarantees that the backedge can be broken.
   unsigned TripCount = 0;
   unsigned TripMultiple = 1;
-  // If there are multiple exiting blocks but one of them is the latch, use the
-  // latch for the trip count estimation. Otherwise insist on a single exiting
-  // block for the trip count estimation.
-  BasicBlock *ExitingBlock = L->getLoopLatch();
-  if (!ExitingBlock || !L->isLoopExiting(ExitingBlock))
-    ExitingBlock = L->getExitingBlock();
-  if (ExitingBlock) {
-    TripCount = SE.getSmallConstantTripCount(L, ExitingBlock);
-    TripMultiple = SE.getSmallConstantTripMultiple(L, ExitingBlock);
+  SmallVector<BasicBlock *, 8> ExitingBlocks;
+  L->getExitingBlocks(ExitingBlocks);
+  for (BasicBlock *ExitingBlock : ExitingBlocks)
+    if (unsigned TC = SE.getSmallConstantTripCount(L, ExitingBlock))
+      if (!TripCount || TC < TripCount)
+        TripCount = TripMultiple = TC;
+
+  if (!TripCount) {
+    // If no exact trip count is known, determine the trip multiple of either
+    // the loop latch or the single exiting block.
+    // TODO: Relax for multiple exits.
+    BasicBlock *ExitingBlock = L->getLoopLatch();
+    if (!ExitingBlock || !L->isLoopExiting(ExitingBlock))
+      ExitingBlock = L->getExitingBlock();
+    if (ExitingBlock)
+      TripMultiple = SE.getSmallConstantTripMultiple(L, ExitingBlock);
   }
 
   // If the loop contains a convergent operation, the prelude we'd add
@@ -1134,9 +1171,35 @@ static LoopUnrollResult tryToUnrollLoop(
       TripMultiple, LoopSize, UP, PP, UseUpperBound);
   if (!UP.Count)
     return LoopUnrollResult::Unmodified;
-  // Unroll factor (Count) must be less or equal to TripCount.
-  if (TripCount && UP.Count > TripCount)
-    UP.Count = TripCount;
+
+  if (PP.PeelCount) {
+    assert(UP.Count == 1 && "Cannot perform peel and unroll in the same step");
+    LLVM_DEBUG(dbgs() << "PEELING loop %" << L->getHeader()->getName()
+                      << " with iteration count " << PP.PeelCount << "!\n");
+    ORE.emit([&]() {
+      return OptimizationRemark(DEBUG_TYPE, "Peeled", L->getStartLoc(),
+                                L->getHeader())
+             << " peeled loop by " << ore::NV("PeelCount", PP.PeelCount)
+             << " iterations";
+    });
+
+    if (peelLoop(L, PP.PeelCount, LI, &SE, &DT, &AC, PreserveLCSSA)) {
+      simplifyLoopAfterUnroll(L, true, LI, &SE, &DT, &AC, &TTI);
+      // If the loop was peeled, we already "used up" the profile information
+      // we had, so we don't want to unroll or peel again.
+      if (PP.PeelProfiledIterations)
+        L->setLoopAlreadyUnrolled();
+      return LoopUnrollResult::PartiallyUnrolled;
+    }
+    return LoopUnrollResult::Unmodified;
+  }
+
+  // At this point, UP.Runtime indicates that run-time unrolling is allowed.
+  // However, we only want to actually perform it if we don't know the trip
+  // count and the unroll count doesn't divide the known trip multiple.
+  // TODO: This decision should probably be pushed up into
+  // computeUnrollCount().
+  UP.Runtime &= TripCount == 0 && TripMultiple % UP.Count != 0;
 
   // Save loop properties before it is transformed.
   MDNode *OrigLoopID = L->getLoopID();
@@ -1145,9 +1208,8 @@ static LoopUnrollResult tryToUnrollLoop(
   Loop *RemainderLoop = nullptr;
   LoopUnrollResult UnrollResult = UnrollLoop(
       L,
-      {UP.Count, TripCount, UP.Force, UP.Runtime, UP.AllowExpensiveTripCount,
-       UseUpperBound, MaxOrZero, TripMultiple, PP.PeelCount, UP.UnrollRemainder,
-       ForgetAllSCEV},
+      {UP.Count, UP.Force, UP.Runtime, UP.AllowExpensiveTripCount,
+       UP.UnrollRemainder, ForgetAllSCEV},
       LI, &SE, &DT, &AC, &TTI, &ORE, PreserveLCSSA, &RemainderLoop);
   if (UnrollResult == LoopUnrollResult::Unmodified)
     return LoopUnrollResult::Unmodified;
@@ -1175,10 +1237,7 @@ static LoopUnrollResult tryToUnrollLoop(
 
   // If loop has an unroll count pragma or unrolled by explicitly set count
   // mark loop as unrolled to prevent unrolling beyond that requested.
-  // If the loop was peeled, we already "used up" the profile information
-  // we had, so we don't want to unroll or peel again.
-  if (UnrollResult != LoopUnrollResult::FullyUnrolled &&
-      (IsCountSetExplicitly || (PP.PeelProfiledIterations && PP.PeelCount)))
+  if (UnrollResult != LoopUnrollResult::FullyUnrolled && IsCountSetExplicitly)
     L->setLoopAlreadyUnrolled();
 
   return UnrollResult;
diff --git a/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
index 18717394d384..9a854ff80246 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
@@ -26,6 +26,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
@@ -309,9 +310,8 @@ bool LUAnalysisCache::countLoop(const Loop *L, const TargetTransformInfo &TTI,
     // consideration code simplification opportunities and code that can
     // be shared by the resultant unswitched loops.
     CodeMetrics Metrics;
-    for (Loop::block_iterator I = L->block_begin(), E = L->block_end(); I != E;
-         ++I)
-      Metrics.analyzeBasicBlock(*I, TTI, EphValues);
+    for (BasicBlock *BB : L->blocks())
+      Metrics.analyzeBasicBlock(BB, TTI, EphValues);
 
     Props.SizeEstimation = Metrics.NumInsts;
     Props.CanBeUnswitchedCount = MaxSize / (Props.SizeEstimation);
@@ -387,8 +387,8 @@ void LUAnalysisCache::cloneData(const Loop *NewLoop, const Loop *OldLoop,
   // Clone unswitched values info:
   // for new loop switches we clone info about values that was
   // already unswitched and has redundant successors.
-  for (UnswitchedValsIt I = Insts.begin(); I != Insts.end(); ++I) {
-    const SwitchInst *OldInst = I->first;
+  for (const auto &I : Insts) {
+    const SwitchInst *OldInst = I.first;
     Value *NewI = VMap.lookup(OldInst);
     const SwitchInst *NewInst = cast_or_null<SwitchInst>(NewI);
     assert(NewInst && "All instructions that are in SrcBB must be in VMap.");
@@ -640,145 +640,6 @@ static bool equalityPropUnSafe(Value &LoopCond) {
   return false;
 }
 
-/// Check if the loop header has a conditional branch that is not
-/// loop-invariant, because it involves load instructions. If all paths from
-/// either the true or false successor to the header or loop exists do not
-/// modify the memory feeding the condition, perform 'partial unswitching'. That
-/// is, duplicate the instructions feeding the condition in the pre-header. Then
-/// unswitch on the duplicated condition. The condition is now known in the
-/// unswitched version for the 'invariant' path through the original loop.
-///
-/// If the branch condition of the header is partially invariant, return a pair
-/// containing the instructions to duplicate and a boolean Constant to update
-/// the condition in the loops created for the true or false successors.
-static std::pair<SmallVector<Instruction *, 4>, Constant *>
-hasPartialIVCondition(Loop *L, MemorySSA &MSSA, AAResults *AA) {
-  SmallVector<Instruction *, 4> ToDuplicate;
-
-  auto *TI = dyn_cast<BranchInst>(L->getHeader()->getTerminator());
-  if (!TI || !TI->isConditional())
-    return {};
-
-  auto *CondI = dyn_cast<CmpInst>(TI->getCondition());
-  // The case with the condition outside the loop should already be handled
-  // earlier.
-  if (!CondI || !L->contains(CondI))
-    return {};
-
-  ToDuplicate.push_back(CondI);
-
-  SmallVector<Value *, 4> WorkList;
-  WorkList.append(CondI->op_begin(), CondI->op_end());
-
-  SmallVector<MemoryAccess *, 4> AccessesToCheck;
-  SmallVector<MemoryLocation, 4> AccessedLocs;
-  while (!WorkList.empty()) {
-    Instruction *I = dyn_cast<Instruction>(WorkList.pop_back_val());
-    if (!I || !L->contains(I))
-      continue;
-
-    // TODO: support additional instructions.
-    if (!isa<LoadInst>(I) && !isa<GetElementPtrInst>(I))
-      return {};
-
-    // Do not duplicate volatile and atomic loads.
-    if (auto *LI = dyn_cast<LoadInst>(I))
-      if (LI->isVolatile() || LI->isAtomic())
-        return {};
-
-    ToDuplicate.push_back(I);
-    if (MemoryAccess *MA = MSSA.getMemoryAccess(I)) {
-      if (auto *MemUse = dyn_cast_or_null<MemoryUse>(MA)) {
-        // Queue the defining access to check for alias checks.
-        AccessesToCheck.push_back(MemUse->getDefiningAccess());
-        AccessedLocs.push_back(MemoryLocation::get(I));
-      } else {
-        // MemoryDefs may clobber the location or may be atomic memory
-        // operations. Bail out.
-        return {};
-      }
-    }
-    WorkList.append(I->op_begin(), I->op_end());
-  }
-
-  if (ToDuplicate.size() <= 1)
-    return {};
-
-  auto HasNoClobbersOnPath =
-      [L, AA, &AccessedLocs](BasicBlock *Succ, BasicBlock *Header,
-                             SmallVector<MemoryAccess *, 4> AccessesToCheck) {
-        // First, collect all blocks in the loop that are on a patch from Succ
-        // to the header.
-        SmallVector<BasicBlock *, 4> WorkList;
-        WorkList.push_back(Succ);
-        WorkList.push_back(Header);
-        SmallPtrSet<BasicBlock *, 4> Seen;
-        Seen.insert(Header);
-        while (!WorkList.empty()) {
-          BasicBlock *Current = WorkList.pop_back_val();
-          if (!L->contains(Current))
-            continue;
-          const auto &SeenIns = Seen.insert(Current);
-          if (!SeenIns.second)
-            continue;
-
-          WorkList.append(succ_begin(Current), succ_end(Current));
-        }
-
-        // Require at least 2 blocks on a path through the loop. This skips
-        // paths that directly exit the loop.
-        if (Seen.size() < 2)
-          return false;
-
-        // Next, check if there are any MemoryDefs that are on the path through
-        // the loop (in the Seen set) and they may-alias any of the locations in
-        // AccessedLocs. If that is the case, they may modify the condition and
-        // partial unswitching is not possible.
-        SmallPtrSet<MemoryAccess *, 4> SeenAccesses;
-        while (!AccessesToCheck.empty()) {
-          MemoryAccess *Current = AccessesToCheck.pop_back_val();
-          auto SeenI = SeenAccesses.insert(Current);
-          if (!SeenI.second || !Seen.contains(Current->getBlock()))
-            continue;
-
-          // Bail out if exceeded the threshold.
-          if (SeenAccesses.size() >= MSSAThreshold)
-            return false;
-
-          // MemoryUse are read-only accesses.
-          if (isa<MemoryUse>(Current))
-            continue;
-
-          // For a MemoryDef, check if is aliases any of the location feeding
-          // the original condition.
-          if (auto *CurrentDef = dyn_cast<MemoryDef>(Current)) {
-            if (any_of(AccessedLocs, [AA, CurrentDef](MemoryLocation &Loc) {
-                  return isModSet(
-                      AA->getModRefInfo(CurrentDef->getMemoryInst(), Loc));
-                }))
-              return false;
-          }
-
-          for (Use &U : Current->uses())
-            AccessesToCheck.push_back(cast<MemoryAccess>(U.getUser()));
-        }
-
-        return true;
-      };
-
-  // If we branch to the same successor, partial unswitching will not be
-  // beneficial.
-  if (TI->getSuccessor(0) == TI->getSuccessor(1))
-    return {};
-
-  if (HasNoClobbersOnPath(TI->getSuccessor(0), L->getHeader(), AccessesToCheck))
-    return {ToDuplicate, ConstantInt::getTrue(TI->getContext())};
-  if (HasNoClobbersOnPath(TI->getSuccessor(1), L->getHeader(), AccessesToCheck))
-    return {ToDuplicate, ConstantInt::getFalse(TI->getContext())};
-
-  return {};
-}
-
 /// Do actual work and unswitch loop if possible and profitable.
 bool LoopUnswitch::processCurrentLoop() {
   bool Changed = false;
@@ -986,17 +847,57 @@ bool LoopUnswitch::processCurrentLoop() {
   // metadata, to avoid unswitching the same loop multiple times.
   if (MSSA &&
       !findOptionMDForLoop(CurrentLoop, "llvm.loop.unswitch.partial.disable")) {
-    auto ToDuplicate = hasPartialIVCondition(CurrentLoop, *MSSA, AA);
-    if (!ToDuplicate.first.empty()) {
+    if (auto Info =
+            hasPartialIVCondition(*CurrentLoop, MSSAThreshold, *MSSA, *AA)) {
+      assert(!Info->InstToDuplicate.empty() &&
+             "need at least a partially invariant condition");
       LLVM_DEBUG(dbgs() << "loop-unswitch: Found partially invariant condition "
-                        << *ToDuplicate.first[0] << "\n");
-      ++NumBranches;
-      unswitchIfProfitable(ToDuplicate.first[0], ToDuplicate.second,
-                           CurrentLoop->getHeader()->getTerminator(),
-                           ToDuplicate.first);
+                        << *Info->InstToDuplicate[0] << "\n");
 
-      RedoLoop = false;
-      return true;
+      Instruction *TI = CurrentLoop->getHeader()->getTerminator();
+      Value *LoopCond = Info->InstToDuplicate[0];
+
+      // If the partially unswitched path is a no-op and has a single exit
+      // block, we do not need to do full unswitching. Instead, we can directly
+      // branch to the exit.
+      // TODO: Instead of duplicating the checks, we could also just directly
+      // branch to the exit from the conditional branch in the loop.
+      if (Info->PathIsNoop) {
+        if (HasBranchDivergence &&
+            getAnalysis<LegacyDivergenceAnalysis>().isDivergent(LoopCond)) {
+          LLVM_DEBUG(dbgs() << "NOT unswitching loop %"
+                            << CurrentLoop->getHeader()->getName()
+                            << " at non-trivial condition '"
+                            << *Info->KnownValue << "' == " << *LoopCond << "\n"
+                            << ". Condition is divergent.\n");
+          return false;
+        }
+
+        ++NumBranches;
+
+        BasicBlock *TrueDest = LoopHeader;
+        BasicBlock *FalseDest = Info->ExitForPath;
+        if (Info->KnownValue->isOneValue())
+          std::swap(TrueDest, FalseDest);
+
+        auto *OldBr =
+            cast<BranchInst>(CurrentLoop->getLoopPreheader()->getTerminator());
+        emitPreheaderBranchOnCondition(LoopCond, Info->KnownValue, TrueDest,
+                                       FalseDest, OldBr, TI,
+                                       Info->InstToDuplicate);
+        delete OldBr;
+        RedoLoop = false;
+        return true;
+      }
+
+      // Otherwise, the path is not a no-op. Run regular unswitching.
+      if (unswitchIfProfitable(LoopCond, Info->KnownValue,
+                               CurrentLoop->getHeader()->getTerminator(),
+                               Info->InstToDuplicate)) {
+        ++NumBranches;
+        RedoLoop = false;
+        return true;
+      }
     }
   }
 
@@ -1026,9 +927,9 @@ static bool isTrivialLoopExitBlockHelper(Loop *L, BasicBlock *BB,
   }
 
   // Otherwise, this is an unvisited intra-loop node.  Check all successors.
-  for (succ_iterator SI = succ_begin(BB), E = succ_end(BB); SI != E; ++SI) {
+  for (BasicBlock *Succ : successors(BB)) {
     // Check to see if the successor is a trivial loop exit.
-    if (!isTrivialLoopExitBlockHelper(L, *SI, ExitBB, Visited))
+    if (!isTrivialLoopExitBlockHelper(L, Succ, ExitBB, Visited))
       return false;
   }
 
@@ -1114,12 +1015,16 @@ void LoopUnswitch::emitPreheaderBranchOnCondition(
 
         Loop *L = LI->getLoopFor(I->getParent());
         auto *DefiningAccess = MemA->getDefiningAccess();
-        // If the defining access is a MemoryPhi in the header, get the incoming
-        // value for the pre-header as defining access.
-        if (DefiningAccess->getBlock() == I->getParent()) {
+        // Get the first defining access before the loop.
+        while (L->contains(DefiningAccess->getBlock())) {
+          // If the defining access is a MemoryPhi, get the incoming
+          // value for the pre-header as defining access.
           if (auto *MemPhi = dyn_cast<MemoryPhi>(DefiningAccess)) {
             DefiningAccess =
                 MemPhi->getIncomingValueForBlock(L->getLoopPreheader());
+          } else {
+            DefiningAccess =
+                cast<MemoryDef>(DefiningAccess)->getDefiningAccess();
           }
         }
         MSSAU->createMemoryAccessInBB(New, DefiningAccess, New->getParent(),
@@ -1518,9 +1423,7 @@ void LoopUnswitch::unswitchNontrivialCondition(
       PHINode *PN = PHINode::Create(LPad->getType(), 0, "",
                                     &*ExitSucc->getFirstInsertionPt());
 
-      for (pred_iterator I = pred_begin(ExitSucc), E = pred_end(ExitSucc);
-           I != E; ++I) {
-        BasicBlock *BB = *I;
+      for (BasicBlock *BB : predecessors(ExitSucc)) {
         LandingPadInst *LPI = BB->getLandingPadInst();
         LPI->replaceAllUsesWith(PN);
         PN->addIncoming(LPI, BB);
@@ -1533,9 +1436,8 @@ void LoopUnswitch::unswitchNontrivialCondition(
     for (Instruction &I : *NewBlocks[NBI]) {
       RemapInstruction(&I, VMap,
                        RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
-      if (auto *II = dyn_cast<IntrinsicInst>(&I))
-        if (II->getIntrinsicID() == Intrinsic::assume)
-          AC->registerAssumption(II);
+      if (auto *II = dyn_cast<AssumeInst>(&I))
+        AC->registerAssumption(II);
     }
   }
 
diff --git a/llvm/lib/Transforms/Scalar/LowerAtomic.cpp b/llvm/lib/Transforms/Scalar/LowerAtomic.cpp
index d1f67b355b19..4063e4fe0472 100644
--- a/llvm/lib/Transforms/Scalar/LowerAtomic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerAtomic.cpp
@@ -40,7 +40,7 @@ static bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
   return true;
 }
 
-static bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
+bool llvm::lowerAtomicRMWInst(AtomicRMWInst *RMWI) {
   IRBuilder<> Builder(RMWI);
   Value *Ptr = RMWI->getPointerOperand();
   Value *Val = RMWI->getValOperand();
@@ -123,7 +123,7 @@ static bool runOnBasicBlock(BasicBlock &BB) {
     else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(&Inst))
       Changed |= LowerAtomicCmpXchgInst(CXI);
     else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(&Inst))
-      Changed |= LowerAtomicRMWInst(RMWI);
+      Changed |= lowerAtomicRMWInst(RMWI);
     else if (LoadInst *LI = dyn_cast<LoadInst>(&Inst)) {
       if (LI->isAtomic())
         LowerLoadInst(LI);
diff --git a/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp
index bfe8db83b027..bd3001988369 100644
--- a/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerConstantIntrinsics.cpp
@@ -15,12 +15,14 @@
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/GlobalsModRef.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -43,14 +45,15 @@ STATISTIC(ObjectSizeIntrinsicsHandled,
           "Number of 'objectsize' intrinsic calls handled");
 
 static Value *lowerIsConstantIntrinsic(IntrinsicInst *II) {
-  Value *Op = II->getOperand(0);
-
-  return isa<Constant>(Op) ? ConstantInt::getTrue(II->getType())
-                           : ConstantInt::getFalse(II->getType());
+  if (auto *C = dyn_cast<Constant>(II->getOperand(0)))
+    if (C->isManifestConstant())
+      return ConstantInt::getTrue(II->getType());
+  return ConstantInt::getFalse(II->getType());
 }
 
 static bool replaceConditionalBranchesOnConstant(Instruction *II,
-                                                 Value *NewValue) {
+                                                 Value *NewValue,
+                                                 DomTreeUpdater *DTU) {
   bool HasDeadBlocks = false;
   SmallSetVector<Instruction *, 8> Worklist;
   replaceAndRecursivelySimplify(II, NewValue, nullptr, nullptr, nullptr,
@@ -78,6 +81,8 @@ static bool replaceConditionalBranchesOnConstant(Instruction *II,
       Other->removePredecessor(Source);
       BI->eraseFromParent();
       BranchInst::Create(Target, Source);
+      if (DTU)
+        DTU->applyUpdates({{DominatorTree::Delete, Source, Other}});
       if (pred_empty(Other))
         HasDeadBlocks = true;
     }
@@ -85,7 +90,12 @@ static bool replaceConditionalBranchesOnConstant(Instruction *II,
   return HasDeadBlocks;
 }
 
-static bool lowerConstantIntrinsics(Function &F, const TargetLibraryInfo *TLI) {
+static bool lowerConstantIntrinsics(Function &F, const TargetLibraryInfo *TLI,
+                                    DominatorTree *DT) {
+  Optional<DomTreeUpdater> DTU;
+  if (DT)
+    DTU.emplace(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+
   bool HasDeadBlocks = false;
   const auto &DL = F.getParent()->getDataLayout();
   SmallVector<WeakTrackingVH, 8> Worklist;
@@ -128,19 +138,20 @@ static bool lowerConstantIntrinsics(Function &F, const TargetLibraryInfo *TLI) {
       ObjectSizeIntrinsicsHandled++;
       break;
     }
-    HasDeadBlocks |= replaceConditionalBranchesOnConstant(II, NewValue);
+    HasDeadBlocks |= replaceConditionalBranchesOnConstant(
+        II, NewValue, DTU.hasValue() ? DTU.getPointer() : nullptr);
   }
   if (HasDeadBlocks)
-    removeUnreachableBlocks(F);
+    removeUnreachableBlocks(F, DTU.hasValue() ? DTU.getPointer() : nullptr);
   return !Worklist.empty();
 }
 
 PreservedAnalyses
 LowerConstantIntrinsicsPass::run(Function &F, FunctionAnalysisManager &AM) {
-  if (lowerConstantIntrinsics(F,
-                              AM.getCachedResult<TargetLibraryAnalysis>(F))) {
+  if (lowerConstantIntrinsics(F, AM.getCachedResult<TargetLibraryAnalysis>(F),
+                              AM.getCachedResult<DominatorTreeAnalysis>(F))) {
     PreservedAnalyses PA;
-    PA.preserve<GlobalsAA>();
+    PA.preserve<DominatorTreeAnalysis>();
     return PA;
   }
 
@@ -163,18 +174,25 @@ public:
   bool runOnFunction(Function &F) override {
     auto *TLIP = getAnalysisIfAvailable<TargetLibraryInfoWrapperPass>();
     const TargetLibraryInfo *TLI = TLIP ? &TLIP->getTLI(F) : nullptr;
-    return lowerConstantIntrinsics(F, TLI);
+    DominatorTree *DT = nullptr;
+    if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
+      DT = &DTWP->getDomTree();
+    return lowerConstantIntrinsics(F, TLI, DT);
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addPreserved<GlobalsAAWrapperPass>();
+    AU.addPreserved<DominatorTreeWrapperPass>();
   }
 };
 } // namespace
 
 char LowerConstantIntrinsics::ID = 0;
-INITIALIZE_PASS(LowerConstantIntrinsics, "lower-constant-intrinsics",
-                "Lower constant intrinsics", false, false)
+INITIALIZE_PASS_BEGIN(LowerConstantIntrinsics, "lower-constant-intrinsics",
+                      "Lower constant intrinsics", false, false)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
+INITIALIZE_PASS_END(LowerConstantIntrinsics, "lower-constant-intrinsics",
+                    "Lower constant intrinsics", false, false)
 
 FunctionPass *llvm::createLowerConstantIntrinsicsPass() {
   return new LowerConstantIntrinsics();
diff --git a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
index da13075dfee2..ead8082f3036 100644
--- a/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerExpectIntrinsic.cpp
@@ -24,6 +24,7 @@
 #include "llvm/IR/Metadata.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Transforms/Scalar.h"
 
@@ -41,15 +42,15 @@ STATISTIC(ExpectIntrinsicsHandled,
 // only be used in extreme cases, we could make this ratio higher. As it stands,
 // programmers may be using __builtin_expect() / llvm.expect to annotate that a
 // branch is likely or unlikely to be taken.
-//
-// There is a known dependency on this ratio in CodeGenPrepare when transforming
-// 'select' instructions. It may be worthwhile to hoist these values to some
-// shared space, so they can be used directly by other passes.
 
-cl::opt<uint32_t> llvm::LikelyBranchWeight(
+// WARNING: these values are internal implementation detail of the pass.
+// They should not be exposed to the outside of the pass, front-end codegen
+// should emit @llvm.expect intrinsics instead of using these weights directly.
+// Transforms should use TargetTransformInfo's getPredictableBranchThreshold().
+static cl::opt<uint32_t> LikelyBranchWeight(
     "likely-branch-weight", cl::Hidden, cl::init(2000),
     cl::desc("Weight of the branch likely to be taken (default = 2000)"));
-cl::opt<uint32_t> llvm::UnlikelyBranchWeight(
+static cl::opt<uint32_t> UnlikelyBranchWeight(
     "unlikely-branch-weight", cl::Hidden, cl::init(1),
     cl::desc("Weight of the branch unlikely to be taken (default = 1)"));
 
diff --git a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
index 8e251ca940a3..42c183a6408e 100644
--- a/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
+++ b/llvm/lib/Transforms/Scalar/LowerMatrixIntrinsics.cpp
@@ -34,6 +34,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/MatrixBuilder.h"
 #include "llvm/IR/PatternMatch.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Pass.h"
@@ -50,11 +51,6 @@ using namespace PatternMatch;
 
 #define DEBUG_TYPE "lower-matrix-intrinsics"
 
-static cl::opt<bool> EnableShapePropagation(
-    "matrix-propagate-shape", cl::init(true), cl::Hidden,
-    cl::desc("Enable/disable shape propagation from matrix intrinsics to other "
-             "instructions."));
-
 static cl::opt<bool>
     FuseMatrix("fuse-matrix", cl::init(true), cl::Hidden,
                cl::desc("Enable/disable fusing matrix instructions."));
@@ -200,11 +196,16 @@ class LowerMatrixIntrinsics {
     unsigned NumLoads = 0;
     /// Number of compute operations emitted to generate this matrix.
     unsigned NumComputeOps = 0;
+    /// Most of the time transposes can be fused with matrix multiplies or can
+    /// be folded away via algebraic simplifications.  This is the number of
+    /// transposes that we failed to make "free" via such optimizations.
+    unsigned NumExposedTransposes = 0;
 
     OpInfoTy &operator+=(const OpInfoTy &RHS) {
       NumStores += RHS.NumStores;
       NumLoads += RHS.NumLoads;
       NumComputeOps += RHS.NumComputeOps;
+      NumExposedTransposes += RHS.NumExposedTransposes;
       return *this;
     }
   };
@@ -309,6 +310,11 @@ class LowerMatrixIntrinsics {
       return *this;
     }
 
+    MatrixTy &addNumExposedTransposes(unsigned N) {
+      OpInfo.NumExposedTransposes += N;
+      return *this;
+    }
+
     MatrixTy &addNumComputeOps(unsigned N) {
       OpInfo.NumComputeOps += N;
       return *this;
@@ -384,8 +390,10 @@ class LowerMatrixIntrinsics {
   /// the result value of the instruction, with the only exceptions being store
   /// instructions and the matrix_column_major_store intrinsics. For those, the
   /// shape information indicates that those instructions should be lowered
-  /// using shape information as well.
-  DenseMap<Value *, ShapeInfo> ShapeMap;
+  /// using shape information as well.  A ValueMap is used so that when
+  /// sub-passes like optimizeTransposes performs RAUW the map stays
+  /// up-to-date.
+  ValueMap<Value *, ShapeInfo> ShapeMap;
 
   /// List of instructions to remove. While lowering, we are not replacing all
   /// users of a lowered instruction, if shape information is available and
@@ -395,6 +403,18 @@ class LowerMatrixIntrinsics {
   /// Map from instructions to their produced column matrix.
   MapVector<Value *, MatrixTy> Inst2ColumnMatrix;
 
+private:
+  static FastMathFlags getFastMathFlags(Instruction *Inst) {
+    FastMathFlags FMF;
+
+    if (isa<FPMathOperator>(*Inst))
+      FMF = Inst->getFastMathFlags();
+
+    FMF.setAllowContract(AllowContractEnabled || FMF.allowContract());
+
+    return FMF;
+  }
+
 public:
   LowerMatrixIntrinsics(Function &F, TargetTransformInfo &TTI,
                         AliasAnalysis *AA, DominatorTree *DT, LoopInfo *LI,
@@ -408,12 +428,18 @@ public:
                      cast<FixedVectorType>(VT)->getNumElements());
   }
 
-  //
+  /// Is this the minimal version executed in the backend pipelines.
+  bool isMinimal() const {
+    return !DT;
+  }
+
   /// Return the estimated number of vector ops required for an operation on
   /// \p VT * N.
   unsigned getNumOps(Type *ST, unsigned N) {
     return std::ceil((ST->getPrimitiveSizeInBits() * N).getFixedSize() /
-                     double(TTI.getRegisterBitWidth(true)));
+                     double(TTI.getRegisterBitWidth(
+                                   TargetTransformInfo::RGK_FixedWidthVector)
+                                .getFixedSize()));
   }
 
   /// Return the set of vectors that a matrix value is lowered to.
@@ -657,34 +683,161 @@ public:
     return NewWorkList;
   }
 
-  bool Visit() {
-    if (EnableShapePropagation) {
-      SmallVector<Instruction *, 32> WorkList;
+  /// Try moving transposes in order to fold them away or into multiplies.
+  void optimizeTransposes() {
+    auto ReplaceAllUsesWith = [this](Instruction &Old, Value *New) {
+      // We need to remove Old from the ShapeMap otherwise RAUW will replace it
+      // with New. We should only add New it it supportsShapeInfo so we insert
+      // it conditionally instead.
+      auto S = ShapeMap.find(&Old);
+      if (S != ShapeMap.end()) {
+        ShapeMap.erase(S);
+        if (supportsShapeInfo(New))
+          ShapeMap.insert({New, S->second});
+      }
+      Old.replaceAllUsesWith(New);
+    };
 
-      // Initially only the shape of matrix intrinsics is known.
-      // Initialize the work list with ops carrying shape information.
-      for (BasicBlock &BB : Func)
-        for (Instruction &Inst : BB) {
-          IntrinsicInst *II = dyn_cast<IntrinsicInst>(&Inst);
-          if (!II)
-            continue;
+    // First sink all transposes inside matmuls, hoping that we end up with NN,
+    // NT or TN variants.
+    for (BasicBlock &BB : reverse(Func)) {
+      for (auto II = BB.rbegin(); II != BB.rend();) {
+        Instruction &I = *II;
+        // We may remove II.  By default continue on the next/prev instruction.
+        ++II;
+        // If we were to erase II, move again.
+        auto EraseFromParent = [&II](Value *V) {
+          auto *Inst = cast<Instruction>(V);
+          if (Inst->use_empty()) {
+            if (Inst == &*II) {
+              ++II;
+            }
+            Inst->eraseFromParent();
+          }
+        };
 
-          switch (II->getIntrinsicID()) {
-          case Intrinsic::matrix_multiply:
-          case Intrinsic::matrix_transpose:
-          case Intrinsic::matrix_column_major_load:
-          case Intrinsic::matrix_column_major_store:
-            WorkList.push_back(&Inst);
-            break;
-          default:
-            break;
+        // If we're creating a new instruction, continue from there.
+        Instruction *NewInst = nullptr;
+
+        IRBuilder<> IB(&I);
+        MatrixBuilder<IRBuilder<>> Builder(IB);
+
+        Value *TA, *TAMA, *TAMB;
+        ConstantInt *R, *K, *C;
+        if (match(&I, m_Intrinsic<Intrinsic::matrix_transpose>(m_Value(TA)))) {
+
+          // Transpose of a transpose is a nop
+          Value *TATA;
+          if (match(TA,
+                    m_Intrinsic<Intrinsic::matrix_transpose>(m_Value(TATA)))) {
+            ReplaceAllUsesWith(I, TATA);
+            EraseFromParent(&I);
+            EraseFromParent(TA);
           }
+
+          // (A * B)^t -> B^t * A^t
+          // RxK KxC      CxK   KxR
+          else if (match(TA, m_Intrinsic<Intrinsic::matrix_multiply>(
+                                 m_Value(TAMA), m_Value(TAMB), m_ConstantInt(R),
+                                 m_ConstantInt(K), m_ConstantInt(C)))) {
+            Value *T0 = Builder.CreateMatrixTranspose(TAMB, K->getZExtValue(),
+                                                      C->getZExtValue(),
+                                                      TAMB->getName() + "_t");
+            // We are being run after shape prop, add shape for newly created
+            // instructions so that we lower them later.
+            setShapeInfo(T0, {C, K});
+            Value *T1 = Builder.CreateMatrixTranspose(TAMA, R->getZExtValue(),
+                                                      K->getZExtValue(),
+                                                      TAMA->getName() + "_t");
+            setShapeInfo(T1, {K, R});
+            NewInst = Builder.CreateMatrixMultiply(T0, T1, C->getZExtValue(),
+                                                   K->getZExtValue(),
+                                                   R->getZExtValue(), "mmul");
+            ReplaceAllUsesWith(I, NewInst);
+            EraseFromParent(&I);
+            EraseFromParent(TA);
+          }
+        }
+
+        // If we replaced I with a new instruction, continue from there.
+        if (NewInst)
+          II = std::next(BasicBlock::reverse_iterator(NewInst));
+      }
+    }
+
+    // If we have a TT matmul, lift the transpose.  We may be able to fold into
+    // consuming multiply.
+    for (BasicBlock &BB : Func) {
+      for (BasicBlock::iterator II = BB.begin(); II != BB.end();) {
+        Instruction *I = &*II;
+        // We may remove I.
+        ++II;
+        Value *A, *B, *AT, *BT;
+        ConstantInt *R, *K, *C;
+        // A^t * B ^t -> (B * A)^t
+        if (match(&*I, m_Intrinsic<Intrinsic::matrix_multiply>(
+                           m_Value(A), m_Value(B), m_ConstantInt(R),
+                           m_ConstantInt(K), m_ConstantInt(C))) &&
+            match(A, m_Intrinsic<Intrinsic::matrix_transpose>(m_Value(AT))) &&
+            match(B, m_Intrinsic<Intrinsic::matrix_transpose>(m_Value((BT))))) {
+          IRBuilder<> IB(&*I);
+          MatrixBuilder<IRBuilder<>> Builder(IB);
+          Value *M = Builder.CreateMatrixMultiply(
+              BT, AT, C->getZExtValue(), K->getZExtValue(), R->getZExtValue());
+          setShapeInfo(M, {C, R});
+          Instruction *NewInst = Builder.CreateMatrixTranspose(
+              M, C->getZExtValue(), R->getZExtValue());
+          ReplaceAllUsesWith(*I, NewInst);
+          if (I->use_empty())
+            I->eraseFromParent();
+          if (A->use_empty())
+            cast<Instruction>(A)->eraseFromParent();
+          if (A != B && B->use_empty())
+            cast<Instruction>(B)->eraseFromParent();
+        }
+      }
+    }
+  }
+
+  bool Visit() {
+    SmallVector<Instruction *, 32> WorkList;
+
+    // Initially only the shape of matrix intrinsics is known.
+    // Initialize the work list with ops carrying shape information.
+    for (BasicBlock &BB : Func)
+      for (Instruction &Inst : BB) {
+        IntrinsicInst *II = dyn_cast<IntrinsicInst>(&Inst);
+        if (!II)
+          continue;
+
+        switch (II->getIntrinsicID()) {
+        case Intrinsic::matrix_multiply:
+        case Intrinsic::matrix_transpose:
+        case Intrinsic::matrix_column_major_load:
+        case Intrinsic::matrix_column_major_store:
+          WorkList.push_back(&Inst);
+          break;
+        default:
+          break;
         }
-      // Propagate shapes until nothing changes any longer.
-      while (!WorkList.empty()) {
-        WorkList = propagateShapeForward(WorkList);
-        WorkList = propagateShapeBackward(WorkList);
       }
+
+    // Avoid unnecessary work if there are no matrix intrinsics in the function.
+    if (WorkList.empty())
+      return false;
+
+    // Propagate shapes until nothing changes any longer.
+    while (!WorkList.empty()) {
+      WorkList = propagateShapeForward(WorkList);
+      WorkList = propagateShapeBackward(WorkList);
+    }
+
+    if (!isMinimal()) {
+      optimizeTransposes();
+      LLVM_DEBUG({
+        dbgs() << "Dump after matrix transpose optimization:\n";
+        Func.dump();
+      });
     }
 
     bool Changed = false;
@@ -736,8 +889,33 @@ public:
       RemarkGen.emitRemarks();
     }
 
-    for (Instruction *Inst : reverse(ToRemove))
+    // Delete the instructions backwards, as it has a reduced likelihood of
+    // having to update as many def-use and use-def chains.
+    //
+    // Because we add to ToRemove during fusion we can't guarantee that defs
+    // are before uses.  Change uses to undef temporarily as these should get
+    // removed as well.
+    //
+    // For verification, we keep track of where we changed uses to undefs in
+    // UndefedInsts and then check that we in fact remove them.
+    SmallSet<Instruction *, 16> UndefedInsts;
+    for (auto *Inst : reverse(ToRemove)) {
+      for (auto I = Inst->use_begin(), E = Inst->use_end(); I != E;) {
+        Use &U = *I++;
+        if (auto *Undefed = dyn_cast<Instruction>(U.getUser()))
+          UndefedInsts.insert(Undefed);
+        U.set(UndefValue::get(Inst->getType()));
+      }
       Inst->eraseFromParent();
+      UndefedInsts.erase(Inst);
+    }
+    if (!UndefedInsts.empty()) {
+      // If we didn't remove all undefed instructions, it's a hard error.
+      dbgs() << "Undefed but present instructions:\n";
+      for (auto *I : UndefedInsts)
+        dbgs() << *I << "\n";
+      llvm_unreachable("Undefed but instruction not removed");
+    }
 
     return Changed;
   }
@@ -797,15 +975,16 @@ public:
   /// vectors.
   MatrixTy loadMatrix(Type *Ty, Value *Ptr, MaybeAlign MAlign, Value *Stride,
                       bool IsVolatile, ShapeInfo Shape, IRBuilder<> &Builder) {
-    auto VType = cast<VectorType>(Ty);
-    Value *EltPtr = createElementPtr(Ptr, VType->getElementType(), Builder);
+    auto *VType = cast<VectorType>(Ty);
+    Type *EltTy = VType->getElementType();
+    Type *VecTy = FixedVectorType::get(EltTy, Shape.getStride());
+    Value *EltPtr = createElementPtr(Ptr, EltTy, Builder);
     MatrixTy Result;
     for (unsigned I = 0, E = Shape.getNumVectors(); I < E; ++I) {
       Value *GEP = computeVectorAddr(EltPtr, Builder.getInt64(I), Stride,
-                                     Shape.getStride(), VType->getElementType(),
-                                     Builder);
+                                     Shape.getStride(), EltTy, Builder);
       Value *Vector = Builder.CreateAlignedLoad(
-          GEP, getAlignForIndex(I, Stride, VType->getElementType(), MAlign),
+          VecTy, GEP, getAlignForIndex(I, Stride, EltTy, MAlign),
           IsVolatile, "col.load");
 
       Result.addVector(Vector);
@@ -987,18 +1166,19 @@ public:
   }
 
   /// Cache \p Matrix as result of \p Inst and update the uses of \p Inst. For
-  /// users with shape information, there's nothing to do: the will use the
+  /// users with shape information, there's nothing to do: they will use the
   /// cached value when they are lowered. For other users, \p Matrix is
   /// flattened and the uses are updated to use it. Also marks \p Inst for
   /// deletion.
   void finalizeLowering(Instruction *Inst, MatrixTy Matrix,
                         IRBuilder<> &Builder) {
-    Inst2ColumnMatrix.insert(std::make_pair(Inst, Matrix));
+    auto inserted = Inst2ColumnMatrix.insert(std::make_pair(Inst, Matrix));
+    (void)inserted;
+    assert(inserted.second && "multiple matrix lowering mapping");
 
     ToRemove.push_back(Inst);
     Value *Flattened = nullptr;
-    for (auto I = Inst->use_begin(), E = Inst->use_end(); I != E;) {
-      Use &U = *I++;
+    for (Use &U : llvm::make_early_inc_range(Inst->uses())) {
       if (ShapeMap.find(U.getUser()) == ShapeMap.end()) {
         if (!Flattened)
           Flattened = Matrix.embedInVector(Builder);
@@ -1009,11 +1189,17 @@ public:
 
   /// Compute \p Result += \p A * \p B for input matrices with left-associating
   /// addition.
+  ///
+  /// We can fold a transpose into the operand that is used to extract scalars.
+  /// This is the first operands with row-major and the second with
+  /// column-major.  If \p IsScalarMatrixTransposed we assume the appropriate
+  /// operand is transposed.
   void emitMatrixMultiply(MatrixTy &Result, const MatrixTy &A,
-                          const MatrixTy &B, bool AllowContraction,
-                          IRBuilder<> &Builder, bool isTiled) {
+                          const MatrixTy &B, IRBuilder<> &Builder, bool IsTiled,
+                          bool IsScalarMatrixTransposed, FastMathFlags FMF) {
     const unsigned VF = std::max<unsigned>(
-        TTI.getRegisterBitWidth(true) /
+        TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+                .getFixedSize() /
             Result.getElementType()->getPrimitiveSizeInBits().getFixedSize(),
         1U);
     unsigned R = Result.getNumRows();
@@ -1025,6 +1211,9 @@ public:
            Result.isColumnMajor() == A.isColumnMajor() &&
            "operands must agree on matrix layout");
     unsigned NumComputeOps = 0;
+
+    Builder.setFastMathFlags(FMF);
+
     if (A.isColumnMajor()) {
       // Multiply columns from the first operand with scalars from the second
       // operand. Then move along the K axes and accumulate the columns.  With
@@ -1039,15 +1228,17 @@ public:
           while (I + BlockSize > R)
             BlockSize /= 2;
 
-          Value *Sum = isTiled ? Result.extractVector(I, J, BlockSize, Builder)
+          Value *Sum = IsTiled ? Result.extractVector(I, J, BlockSize, Builder)
                                : nullptr;
           for (unsigned K = 0; K < M; ++K) {
             Value *L = A.extractVector(I, K, BlockSize, Builder);
-            Value *RH = Builder.CreateExtractElement(B.getColumn(J), K);
+            Value *RH = Builder.CreateExtractElement(
+                B.getColumn(IsScalarMatrixTransposed ? K : J),
+                IsScalarMatrixTransposed ? J : K);
             Value *Splat = Builder.CreateVectorSplat(BlockSize, RH, "splat");
-            Sum = createMulAdd(isSumZero && K == 0 ? nullptr : Sum, L, Splat,
-                               Result.getElementType()->isFloatingPointTy(),
-                               Builder, AllowContraction, NumComputeOps);
+            Sum =
+                createMulAdd(isSumZero && K == 0 ? nullptr : Sum, L, Splat,
+                             IsFP, Builder, FMF.allowContract(), NumComputeOps);
           }
           Result.setVector(J,
                            insertVector(Result.getVector(J), I, Sum, Builder));
@@ -1068,10 +1259,13 @@ public:
           Value *Sum = nullptr;
           for (unsigned K = 0; K < M; ++K) {
             Value *R = B.extractVector(K, J, BlockSize, Builder);
-            Value *LH = Builder.CreateExtractElement(A.getVector(I), K);
+            Value *LH = Builder.CreateExtractElement(
+                A.getVector(IsScalarMatrixTransposed ? K : I),
+                IsScalarMatrixTransposed ? I : K);
             Value *Splat = Builder.CreateVectorSplat(BlockSize, LH, "splat");
-            Sum = createMulAdd(isSumZero && K == 0 ? nullptr : Sum, Splat, R,
-                               IsFP, Builder, AllowContraction, NumComputeOps);
+            Sum =
+                createMulAdd(isSumZero && K == 0 ? nullptr : Sum, Splat, R,
+                             IsFP, Builder, FMF.allowContract(), NumComputeOps);
           }
           Result.setVector(I,
                            insertVector(Result.getVector(I), J, Sum, Builder));
@@ -1089,10 +1283,8 @@ public:
     MemoryLocation StoreLoc = MemoryLocation::get(Store);
     MemoryLocation LoadLoc = MemoryLocation::get(Load);
 
-    AliasResult LdAliased = AA->alias(LoadLoc, StoreLoc);
-
     // If we can statically determine noalias we're good.
-    if (!LdAliased)
+    if (AA->isNoAlias(LoadLoc, StoreLoc))
       return Load->getPointerOperand();
 
     // Create code to check if the memory locations of the Load and Store
@@ -1179,10 +1371,11 @@ public:
     const unsigned M = LShape.NumColumns;
     auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
 
-    const unsigned VF =
-        std::max<unsigned>(TTI.getRegisterBitWidth(true) /
-                               EltType->getPrimitiveSizeInBits().getFixedSize(),
-                           1U);
+    const unsigned VF = std::max<unsigned>(
+        TTI.getRegisterBitWidth(TargetTransformInfo::RGK_FixedWidthVector)
+                .getFixedSize() /
+            EltType->getPrimitiveSizeInBits().getFixedSize(),
+        1U);
 
     // Cost model for tiling
     //
@@ -1210,8 +1403,7 @@ public:
   }
 
   void createTiledLoops(CallInst *MatMul, Value *LPtr, ShapeInfo LShape,
-                        Value *RPtr, ShapeInfo RShape, StoreInst *Store,
-                        bool AllowContract) {
+                        Value *RPtr, ShapeInfo RShape, StoreInst *Store) {
     auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
 
     // Create the main tiling loop nest.
@@ -1247,7 +1439,8 @@ public:
                             {TileSize, TileSize}, EltType, Builder);
     MatrixTy B = loadMatrix(RPtr, {}, false, RShape, TI.CurrentK, TI.CurrentCol,
                             {TileSize, TileSize}, EltType, Builder);
-    emitMatrixMultiply(TileResult, A, B, AllowContract, Builder, true);
+    emitMatrixMultiply(TileResult, A, B, Builder, true, false,
+                       getFastMathFlags(MatMul));
     // Store result after the inner loop is done.
     Builder.SetInsertPoint(TI.RowLoopLatch->getTerminator());
     storeMatrix(TileResult, Store->getPointerOperand(), Store->getAlign(),
@@ -1286,11 +1479,8 @@ public:
     Value *BPtr = getNonAliasingPointer(LoadOp1, Store, MatMul);
     Value *CPtr = Store->getPointerOperand();
 
-    bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) &&
-                                                  MatMul->hasAllowContract());
     if (TileUseLoops && (R % TileSize == 0 && C % TileSize == 0))
-      createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store,
-                       AllowContract);
+      createTiledLoops(MatMul, APtr, LShape, BPtr, RShape, Store);
     else {
       IRBuilder<> Builder(Store);
       for (unsigned J = 0; J < C; J += TileSize)
@@ -1309,7 +1499,8 @@ public:
                 loadMatrix(BPtr, LoadOp1->getAlign(), LoadOp1->isVolatile(),
                            RShape, Builder.getInt64(K), Builder.getInt64(J),
                            {TileM, TileC}, EltType, Builder);
-            emitMatrixMultiply(Res, A, B, AllowContract, Builder, true);
+            emitMatrixMultiply(Res, A, B, Builder, true, false,
+                               getFastMathFlags(MatMul));
           }
           storeMatrix(Res, CPtr, Store->getAlign(), Store->isVolatile(), {R, M},
                       Builder.getInt64(I), Builder.getInt64(J), EltType,
@@ -1326,7 +1517,7 @@ public:
       FusedInsts.insert(LoadOp0);
       LoadOp0->eraseFromParent();
     }
-    if (LoadOp1->hasNUses(0)) {
+    if (LoadOp1 != LoadOp0 && LoadOp1->hasNUses(0)) {
       FusedInsts.insert(LoadOp1);
       LoadOp1->eraseFromParent();
     }
@@ -1334,29 +1525,97 @@ public:
 
   /// Try to lower matrix multiply chains by fusing operations.
   ///
-  /// Currently we only lower {ld, ld} -> matmul -> st chains.
-  //
-  /// No need to return a MatrixTy object for the result of the operation, since
-  /// the single store user will be lowered as part of this. Instructions that
-  /// are completely eliminated by fusion are added to \p FusedInsts.
+  /// Call finalizeLowering on lowered instructions.  Instructions that are
+  /// completely eliminated by fusion are added to \p FusedInsts.
   void LowerMatrixMultiplyFused(CallInst *MatMul,
                                 SmallPtrSetImpl<Instruction *> &FusedInsts) {
-    if (!FuseMatrix || !MatMul->hasOneUse() ||
-        MatrixLayout != MatrixLayoutTy::ColumnMajor || !DT)
+    if (!FuseMatrix || !DT)
       return;
 
     assert(AA && LI && "Analyses should be available");
 
-    auto *LoadOp0 = dyn_cast<LoadInst>(MatMul->getOperand(0));
-    auto *LoadOp1 = dyn_cast<LoadInst>(MatMul->getOperand(1));
+    Value *A = MatMul->getArgOperand(0);
+    Value *B = MatMul->getArgOperand(1);
+
+    // We can fold the transpose into the operand that is used to fetch scalars.
+    Value *T;
+    if (MatrixLayout == MatrixLayoutTy::ColumnMajor
+            ? match(B, m_Intrinsic<Intrinsic::matrix_transpose>(m_Value(T)))
+            : match(A, m_Intrinsic<Intrinsic::matrix_transpose>(m_Value(T)))) {
+      IRBuilder<> Builder(MatMul);
+      auto *EltType = cast<VectorType>(MatMul->getType())->getElementType();
+      ShapeInfo LShape(MatMul->getArgOperand(2), MatMul->getArgOperand(3));
+      ShapeInfo RShape(MatMul->getArgOperand(3), MatMul->getArgOperand(4));
+      const unsigned R = LShape.NumRows;
+      const unsigned M = LShape.NumColumns;
+      const unsigned C = RShape.NumColumns;
+
+      MatrixTy MA;
+      MatrixTy MB;
+
+      Value *Transpose;
+      if (MatrixLayout == MatrixLayoutTy::ColumnMajor) {
+        MA = getMatrix(A, ShapeInfo(R, M), Builder);
+        MB = getMatrix(T, ShapeInfo(C, M), Builder);
+        Transpose = B;
+      } else {
+        MA = getMatrix(T, ShapeInfo(R, M), Builder);
+        MB = getMatrix(B, ShapeInfo(C, M), Builder);
+        Transpose = A;
+      }
+
+      // Initialize the output
+      MatrixTy Result(R, C, EltType);
+
+      emitMatrixMultiply(Result, MA, MB, Builder, false, true,
+                         getFastMathFlags(MatMul));
+
+      FusedInsts.insert(MatMul);
+      if (Transpose->hasOneUse()) {
+        FusedInsts.insert(cast<Instruction>(Transpose));
+        ToRemove.push_back(cast<Instruction>(Transpose));
+        // TODO: add a fake entry for the folded instruction so that this is
+        // included in the expression in the remark.
+        Inst2ColumnMatrix[Transpose] = MatrixTy(M, C, EltType);
+      }
+      finalizeLowering(MatMul, Result, Builder);
+      return;
+    }
+
+    if (!MatMul->hasOneUse() || MatrixLayout != MatrixLayoutTy::ColumnMajor)
+      return;
+
+    // Lower {ld, ld} -> matmul -> st chains.  No need to call finalizeLowering
+    // since the single store user will be lowered as part of this.
+    auto *LoadOp0 = dyn_cast<LoadInst>(A);
+    auto *LoadOp1 = dyn_cast<LoadInst>(B);
     auto *Store = dyn_cast<StoreInst>(*MatMul->user_begin());
     if (LoadOp0 && LoadOp1 && Store) {
       // The store address must dominate the MatMul instruction, otherwise
       // we create invalid IR.
-      // FIXME: See if we can hoist the store address computation.
-      auto *AddrI = dyn_cast<Instruction>(Store->getOperand(1));
-      if (AddrI && (!DT->dominates(AddrI, MatMul)))
-        return;
+      SetVector<Value *> WorkList;
+      WorkList.insert(Store->getOperand(1));
+      SmallVector<Instruction *> ToHoist;
+      for (unsigned I = 0; I != WorkList.size(); ++I) {
+        Value *Current = WorkList[I];
+        auto *CurrI = dyn_cast<Instruction>(Current);
+        if (!CurrI)
+          continue;
+        if (isa<PHINode>(CurrI))
+          return;
+        if (DT->dominates(CurrI, MatMul))
+          continue;
+        if (CurrI->mayHaveSideEffects() || CurrI->mayReadFromMemory())
+          return;
+        ToHoist.push_back(CurrI);
+        WorkList.insert(CurrI->op_begin(), CurrI->op_end());
+      }
+
+      sort(ToHoist, [this](Instruction *A, Instruction *B) {
+        return DT->dominates(A, B);
+      });
+      for (Instruction *I : ToHoist)
+        I->moveBefore(MatMul);
 
       emitSIMDTiling(MatMul, LoadOp0, LoadOp1, Store, FusedInsts);
       return;
@@ -1384,9 +1643,8 @@ public:
     assert(Lhs.getElementType() == Result.getElementType() &&
            "Matrix multiply result element type does not match arguments.");
 
-    bool AllowContract = AllowContractEnabled || (isa<FPMathOperator>(MatMul) &&
-                                                  MatMul->hasAllowContract());
-    emitMatrixMultiply(Result, Lhs, Rhs, AllowContract, Builder, false);
+    emitMatrixMultiply(Result, Lhs, Rhs, Builder, false, false,
+                       getFastMathFlags(MatMul));
     finalizeLowering(MatMul, Result, Builder);
   }
 
@@ -1423,7 +1681,8 @@ public:
     // account for later simplifications/combines.
     finalizeLowering(
         Inst,
-        Result.addNumComputeOps(2 * ArgShape.NumRows * ArgShape.NumColumns),
+        Result.addNumComputeOps(2 * ArgShape.NumRows * ArgShape.NumColumns)
+            .addNumExposedTransposes(1),
         Builder);
   }
 
@@ -1470,6 +1729,8 @@ public:
            Result.isColumnMajor() == A.isColumnMajor() &&
            "operands must agree on matrix layout");
 
+    Builder.setFastMathFlags(getFastMathFlags(Inst));
+
     // Helper to perform binary op on vectors.
     auto BuildVectorOp = [&Builder, Inst](Value *LHS, Value *RHS) {
       switch (Inst->getOpcode()) {
@@ -1514,6 +1775,8 @@ public:
     MatrixTy Result;
     MatrixTy M = getMatrix(Op, Shape, Builder);
 
+    Builder.setFastMathFlags(getFastMathFlags(Inst));
+
     // Helper to perform unary op on vectors.
     auto BuildVectorOp = [&Builder, Inst](Value *Op) {
       switch (Inst->getOpcode()) {
@@ -1631,7 +1894,7 @@ public:
           return;
         }
         IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
-        write(StringRef(Intrinsic::getName(II->getIntrinsicID(), {}))
+        write(Intrinsic::getBaseName(II->getIntrinsicID())
                   .drop_front(StringRef("llvm.matrix.").size()));
         write(".");
         std::string Tmp;
@@ -1938,7 +2201,9 @@ public:
           Rem << ore::NV("NumStores", Counts.NumStores) << " stores, "
               << ore::NV("NumLoads", Counts.NumLoads) << " loads, "
               << ore::NV("NumComputeOps", Counts.NumComputeOps)
-              << " compute ops";
+              << " compute ops, "
+              << ore::NV("NumExposedTransposes", Counts.NumExposedTransposes)
+              << " exposed transposes";
 
           if (SharedCounts.NumStores > 0 || SharedCounts.NumLoads > 0 ||
               SharedCounts.NumComputeOps > 0) {
diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index a4e695497f30..2e36c50b75fc 100644
--- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -68,7 +68,7 @@ using namespace llvm;
 #define DEBUG_TYPE "memcpyopt"
 
 static cl::opt<bool>
-    EnableMemorySSA("enable-memcpyopt-memoryssa", cl::init(false), cl::Hidden,
+    EnableMemorySSA("enable-memcpyopt-memoryssa", cl::init(true), cl::Hidden,
                     cl::desc("Use MemorySSA-backed MemCpyOpt."));
 
 STATISTIC(NumMemCpyInstr, "Number of memcpy instructions deleted");
@@ -308,6 +308,7 @@ INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
 INITIALIZE_PASS_END(MemCpyOptLegacyPass, "memcpyopt", "MemCpy Optimization",
                     false, false)
 
@@ -398,6 +399,13 @@ Instruction *MemCpyOptPass::tryMergingIntoMemset(Instruction *StartInst,
       }
     }
 
+    // Calls that only access inaccessible memory do not block merging
+    // accessible stores.
+    if (auto *CB = dyn_cast<CallBase>(BI)) {
+      if (CB->onlyAccessesInaccessibleMemory())
+        continue;
+    }
+
     if (!isa<StoreInst>(BI) && !isa<MemSetInst>(BI)) {
       // If the instruction is readnone, ignore it, otherwise bail out.  We
       // don't even allow readonly here because we don't want something like:
@@ -683,7 +691,7 @@ bool MemCpyOptPass::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
 
         // We found an instruction that may write to the loaded memory.
         // We can try to promote at this position instead of the store
-        // position if nothing alias the store memory after this and the store
+        // position if nothing aliases the store memory after this and the store
         // destination is not in the range.
         if (P && P != SI) {
           if (!moveUp(SI, P, LI))
@@ -1049,10 +1057,12 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
 
   // Second, the length of the memcpy's must be the same, or the preceding one
   // must be larger than the following one.
-  ConstantInt *MDepLen = dyn_cast<ConstantInt>(MDep->getLength());
-  ConstantInt *MLen = dyn_cast<ConstantInt>(M->getLength());
-  if (!MDepLen || !MLen || MDepLen->getZExtValue() < MLen->getZExtValue())
-    return false;
+  if (MDep->getLength() != M->getLength()) {
+    ConstantInt *MDepLen = dyn_cast<ConstantInt>(MDep->getLength());
+    ConstantInt *MLen = dyn_cast<ConstantInt>(M->getLength());
+    if (!MDepLen || !MLen || MDepLen->getZExtValue() < MLen->getZExtValue())
+      return false;
+  }
 
   // Verify that the copied-from memory doesn't change in between the two
   // transfers.  For example, in:
@@ -1099,7 +1109,14 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
     NewM = Builder.CreateMemMove(M->getRawDest(), M->getDestAlign(),
                                  MDep->getRawSource(), MDep->getSourceAlign(),
                                  M->getLength(), M->isVolatile());
-  else
+  else if (isa<MemCpyInlineInst>(M)) {
+    // llvm.memcpy may be promoted to llvm.memcpy.inline, but the converse is
+    // never allowed since that would allow the latter to be lowered as a call
+    // to an external function.
+    NewM = Builder.CreateMemCpyInline(
+        M->getRawDest(), M->getDestAlign(), MDep->getRawSource(),
+        MDep->getSourceAlign(), M->getLength(), M->isVolatile());
+  } else
     NewM = Builder.CreateMemCpy(M->getRawDest(), M->getDestAlign(),
                                 MDep->getRawSource(), MDep->getSourceAlign(),
                                 M->getLength(), M->isVolatile());
@@ -1134,7 +1151,7 @@ bool MemCpyOptPass::processMemCpyMemCpyDependence(MemCpyInst *M,
 bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
                                                   MemSetInst *MemSet) {
   // We can only transform memset/memcpy with the same destination.
-  if (MemSet->getDest() != MemCpy->getDest())
+  if (!AA->isMustAlias(MemSet->getDest(), MemCpy->getDest()))
     return false;
 
   // Check that src and dst of the memcpy aren't the same. While memcpy
@@ -1172,6 +1189,13 @@ bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
   if (mayBeVisibleThroughUnwinding(Dest, MemSet, MemCpy))
     return false;
 
+  // If the sizes are the same, simply drop the memset instead of generating
+  // a replacement with zero size.
+  if (DestSize == SrcSize) {
+    eraseInstruction(MemSet);
+    return true;
+  }
+
   // By default, create an unaligned memset.
   unsigned Align = 1;
   // If Dest is aligned, and SrcSize is constant, use the minimum alignment
@@ -1197,8 +1221,11 @@ bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
   Value *SizeDiff = Builder.CreateSub(DestSize, SrcSize);
   Value *MemsetLen = Builder.CreateSelect(
       Ule, ConstantInt::getNullValue(DestSize->getType()), SizeDiff);
+  unsigned DestAS = Dest->getType()->getPointerAddressSpace();
   Instruction *NewMemSet = Builder.CreateMemSet(
-      Builder.CreateGEP(Dest->getType()->getPointerElementType(), Dest,
+      Builder.CreateGEP(Builder.getInt8Ty(),
+                        Builder.CreatePointerCast(Dest,
+                                                  Builder.getInt8PtrTy(DestAS)),
                         SrcSize),
       MemSet->getOperand(1), MemsetLen, MaybeAlign(Align));
 
@@ -1221,21 +1248,23 @@ bool MemCpyOptPass::processMemSetMemCpyDependence(MemCpyInst *MemCpy,
 
 /// Determine whether the instruction has undefined content for the given Size,
 /// either because it was freshly alloca'd or started its lifetime.
-static bool hasUndefContents(Instruction *I, ConstantInt *Size) {
+static bool hasUndefContents(Instruction *I, Value *Size) {
   if (isa<AllocaInst>(I))
     return true;
 
-  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
-    if (II->getIntrinsicID() == Intrinsic::lifetime_start)
-      if (ConstantInt *LTSize = dyn_cast<ConstantInt>(II->getArgOperand(0)))
-        if (LTSize->getZExtValue() >= Size->getZExtValue())
-          return true;
+  if (ConstantInt *CSize = dyn_cast<ConstantInt>(Size)) {
+    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
+      if (II->getIntrinsicID() == Intrinsic::lifetime_start)
+        if (ConstantInt *LTSize = dyn_cast<ConstantInt>(II->getArgOperand(0)))
+          if (LTSize->getZExtValue() >= CSize->getZExtValue())
+            return true;
+  }
 
   return false;
 }
 
 static bool hasUndefContentsMSSA(MemorySSA *MSSA, AliasAnalysis *AA, Value *V,
-                                 MemoryDef *Def, ConstantInt *Size) {
+                                 MemoryDef *Def, Value *Size) {
   if (MSSA->isLiveOnEntryDef(Def))
     return isa<AllocaInst>(getUnderlyingObject(V));
 
@@ -1243,9 +1272,24 @@ static bool hasUndefContentsMSSA(MemorySSA *MSSA, AliasAnalysis *AA, Value *V,
           dyn_cast_or_null<IntrinsicInst>(Def->getMemoryInst())) {
     if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
       ConstantInt *LTSize = cast<ConstantInt>(II->getArgOperand(0));
-      if (AA->isMustAlias(V, II->getArgOperand(1)) &&
-          LTSize->getZExtValue() >= Size->getZExtValue())
-        return true;
+
+      if (ConstantInt *CSize = dyn_cast<ConstantInt>(Size)) {
+        if (AA->isMustAlias(V, II->getArgOperand(1)) &&
+            LTSize->getZExtValue() >= CSize->getZExtValue())
+          return true;
+      }
+
+      // If the lifetime.start covers a whole alloca (as it almost always
+      // does) and we're querying a pointer based on that alloca, then we know
+      // the memory is definitely undef, regardless of how exactly we alias.
+      // The size also doesn't matter, as an out-of-bounds access would be UB.
+      AllocaInst *Alloca = dyn_cast<AllocaInst>(getUnderlyingObject(V));
+      if (getUnderlyingObject(II->getArgOperand(1)) == Alloca) {
+        const DataLayout &DL = Alloca->getModule()->getDataLayout();
+        if (Optional<TypeSize> AllocaSize = Alloca->getAllocationSizeInBits(DL))
+          if (*AllocaSize == LTSize->getValue() * 8)
+            return true;
+      }
     }
   }
 
@@ -1264,8 +1308,6 @@ static bool hasUndefContentsMSSA(MemorySSA *MSSA, AliasAnalysis *AA, Value *V,
 ///   memset(dst2, c, dst2_size);
 /// \endcode
 /// When dst2_size <= dst1_size.
-///
-/// The \p MemCpy must have a Constant length.
 bool MemCpyOptPass::performMemCpyToMemSetOptzn(MemCpyInst *MemCpy,
                                                MemSetInst *MemSet) {
   // Make sure that memcpy(..., memset(...), ...), that is we are memsetting and
@@ -1273,38 +1315,47 @@ bool MemCpyOptPass::performMemCpyToMemSetOptzn(MemCpyInst *MemCpy,
   if (!AA->isMustAlias(MemSet->getRawDest(), MemCpy->getRawSource()))
     return false;
 
-  // A known memset size is required.
-  ConstantInt *MemSetSize = dyn_cast<ConstantInt>(MemSet->getLength());
-  if (!MemSetSize)
-    return false;
+  Value *MemSetSize = MemSet->getLength();
+  Value *CopySize = MemCpy->getLength();
 
-  // Make sure the memcpy doesn't read any more than what the memset wrote.
-  // Don't worry about sizes larger than i64.
-  ConstantInt *CopySize = cast<ConstantInt>(MemCpy->getLength());
-  if (CopySize->getZExtValue() > MemSetSize->getZExtValue()) {
-    // If the memcpy is larger than the memset, but the memory was undef prior
-    // to the memset, we can just ignore the tail. Technically we're only
-    // interested in the bytes from MemSetSize..CopySize here, but as we can't
-    // easily represent this location, we use the full 0..CopySize range.
-    MemoryLocation MemCpyLoc = MemoryLocation::getForSource(MemCpy);
-    bool CanReduceSize = false;
-    if (EnableMemorySSA) {
-      MemoryUseOrDef *MemSetAccess = MSSA->getMemoryAccess(MemSet);
-      MemoryAccess *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(
-          MemSetAccess->getDefiningAccess(), MemCpyLoc);
-      if (auto *MD = dyn_cast<MemoryDef>(Clobber))
-        if (hasUndefContentsMSSA(MSSA, AA, MemCpy->getSource(), MD, CopySize))
-          CanReduceSize = true;
-    } else {
-      MemDepResult DepInfo = MD->getPointerDependencyFrom(
-          MemCpyLoc, true, MemSet->getIterator(), MemSet->getParent());
-      if (DepInfo.isDef() && hasUndefContents(DepInfo.getInst(), CopySize))
-        CanReduceSize = true;
-    }
+  if (MemSetSize != CopySize) {
+    // Make sure the memcpy doesn't read any more than what the memset wrote.
+    // Don't worry about sizes larger than i64.
+
+    // A known memset size is required.
+    ConstantInt *CMemSetSize = dyn_cast<ConstantInt>(MemSetSize);
+    if (!CMemSetSize)
+      return false;
 
-    if (!CanReduceSize)
+    // A known memcpy size is also required.
+    ConstantInt *CCopySize = dyn_cast<ConstantInt>(CopySize);
+    if (!CCopySize)
       return false;
-    CopySize = MemSetSize;
+    if (CCopySize->getZExtValue() > CMemSetSize->getZExtValue()) {
+      // If the memcpy is larger than the memset, but the memory was undef prior
+      // to the memset, we can just ignore the tail. Technically we're only
+      // interested in the bytes from MemSetSize..CopySize here, but as we can't
+      // easily represent this location, we use the full 0..CopySize range.
+      MemoryLocation MemCpyLoc = MemoryLocation::getForSource(MemCpy);
+      bool CanReduceSize = false;
+      if (EnableMemorySSA) {
+        MemoryUseOrDef *MemSetAccess = MSSA->getMemoryAccess(MemSet);
+        MemoryAccess *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(
+            MemSetAccess->getDefiningAccess(), MemCpyLoc);
+        if (auto *MD = dyn_cast<MemoryDef>(Clobber))
+          if (hasUndefContentsMSSA(MSSA, AA, MemCpy->getSource(), MD, CopySize))
+            CanReduceSize = true;
+      } else {
+        MemDepResult DepInfo = MD->getPointerDependencyFrom(
+            MemCpyLoc, true, MemSet->getIterator(), MemSet->getParent());
+        if (DepInfo.isDef() && hasUndefContents(DepInfo.getInst(), CopySize))
+          CanReduceSize = true;
+      }
+
+      if (!CanReduceSize)
+        return false;
+      CopySize = MemSetSize;
+    }
   }
 
   IRBuilder<> Builder(MemCpy);
@@ -1376,10 +1427,6 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
           if (processMemSetMemCpyDependence(M, MDep))
             return true;
 
-    // The optimizations after this point require the memcpy size.
-    ConstantInt *CopySize = dyn_cast<ConstantInt>(M->getLength());
-    if (!CopySize) return false;
-
     MemoryAccess *SrcClobber = MSSA->getWalker()->getClobberingMemoryAccess(
         AnyClobber, MemoryLocation::getForSource(M));
 
@@ -1392,26 +1439,29 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
     //   d) memcpy from a just-memset'd source can be turned into memset.
     if (auto *MD = dyn_cast<MemoryDef>(SrcClobber)) {
       if (Instruction *MI = MD->getMemoryInst()) {
-        if (auto *C = dyn_cast<CallInst>(MI)) {
-          // The memcpy must post-dom the call. Limit to the same block for now.
-          // Additionally, we need to ensure that there are no accesses to dest
-          // between the call and the memcpy. Accesses to src will be checked
-          // by performCallSlotOptzn().
-          // TODO: Support non-local call-slot optimization?
-          if (C->getParent() == M->getParent() &&
-              !accessedBetween(*AA, DestLoc, MD, MA)) {
-            // FIXME: Can we pass in either of dest/src alignment here instead
-            // of conservatively taking the minimum?
-            Align Alignment = std::min(M->getDestAlign().valueOrOne(),
-                                       M->getSourceAlign().valueOrOne());
-            if (performCallSlotOptzn(M, M, M->getDest(), M->getSource(),
-                                     CopySize->getZExtValue(), Alignment, C)) {
-              LLVM_DEBUG(dbgs() << "Performed call slot optimization:\n"
-                                << "    call: " << *C << "\n"
-                                << "    memcpy: " << *M << "\n");
-              eraseInstruction(M);
-              ++NumMemCpyInstr;
-              return true;
+        if (ConstantInt *CopySize = dyn_cast<ConstantInt>(M->getLength())) {
+          if (auto *C = dyn_cast<CallInst>(MI)) {
+            // The memcpy must post-dom the call. Limit to the same block for
+            // now. Additionally, we need to ensure that there are no accesses
+            // to dest between the call and the memcpy. Accesses to src will be
+            // checked by performCallSlotOptzn().
+            // TODO: Support non-local call-slot optimization?
+            if (C->getParent() == M->getParent() &&
+                !accessedBetween(*AA, DestLoc, MD, MA)) {
+              // FIXME: Can we pass in either of dest/src alignment here instead
+              // of conservatively taking the minimum?
+              Align Alignment = std::min(M->getDestAlign().valueOrOne(),
+                                         M->getSourceAlign().valueOrOne());
+              if (performCallSlotOptzn(M, M, M->getDest(), M->getSource(),
+                                       CopySize->getZExtValue(), Alignment,
+                                       C)) {
+                LLVM_DEBUG(dbgs() << "Performed call slot optimization:\n"
+                                  << "    call: " << *C << "\n"
+                                  << "    memcpy: " << *M << "\n");
+                eraseInstruction(M);
+                ++NumMemCpyInstr;
+                return true;
+              }
             }
           }
         }
@@ -1427,7 +1477,7 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
         }
       }
 
-      if (hasUndefContentsMSSA(MSSA, AA, M->getSource(), MD, CopySize)) {
+      if (hasUndefContentsMSSA(MSSA, AA, M->getSource(), MD, M->getLength())) {
         LLVM_DEBUG(dbgs() << "Removed memcpy from undef\n");
         eraseInstruction(M);
         ++NumMemCpyInstr;
@@ -1444,10 +1494,6 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
         if (processMemSetMemCpyDependence(M, MDep))
           return true;
 
-    // The optimizations after this point require the memcpy size.
-    ConstantInt *CopySize = dyn_cast<ConstantInt>(M->getLength());
-    if (!CopySize) return false;
-
     // There are four possible optimizations we can do for memcpy:
     //   a) memcpy-memcpy xform which exposes redundance for DSE.
     //   b) call-memcpy xform for return slot optimization.
@@ -1455,17 +1501,19 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
     //      its lifetime copies undefined data, and we can therefore eliminate
     //      the memcpy in favor of the data that was already at the destination.
     //   d) memcpy from a just-memset'd source can be turned into memset.
-    if (DepInfo.isClobber()) {
-      if (CallInst *C = dyn_cast<CallInst>(DepInfo.getInst())) {
-        // FIXME: Can we pass in either of dest/src alignment here instead
-        // of conservatively taking the minimum?
-        Align Alignment = std::min(M->getDestAlign().valueOrOne(),
-                                   M->getSourceAlign().valueOrOne());
-        if (performCallSlotOptzn(M, M, M->getDest(), M->getSource(),
-                                 CopySize->getZExtValue(), Alignment, C)) {
-          eraseInstruction(M);
-          ++NumMemCpyInstr;
-          return true;
+    if (ConstantInt *CopySize = dyn_cast<ConstantInt>(M->getLength())) {
+      if (DepInfo.isClobber()) {
+        if (CallInst *C = dyn_cast<CallInst>(DepInfo.getInst())) {
+          // FIXME: Can we pass in either of dest/src alignment here instead
+          // of conservatively taking the minimum?
+          Align Alignment = std::min(M->getDestAlign().valueOrOne(),
+                                     M->getSourceAlign().valueOrOne());
+          if (performCallSlotOptzn(M, M, M->getDest(), M->getSource(),
+                                   CopySize->getZExtValue(), Alignment, C)) {
+            eraseInstruction(M);
+            ++NumMemCpyInstr;
+            return true;
+          }
         }
       }
     }
@@ -1478,7 +1526,7 @@ bool MemCpyOptPass::processMemCpy(MemCpyInst *M, BasicBlock::iterator &BBI) {
       if (MemCpyInst *MDep = dyn_cast<MemCpyInst>(SrcDepInfo.getInst()))
         return processMemCpyMemCpyDependence(M, MDep);
     } else if (SrcDepInfo.isDef()) {
-      if (hasUndefContents(SrcDepInfo.getInst(), CopySize)) {
+      if (hasUndefContents(SrcDepInfo.getInst(), M->getLength())) {
         eraseInstruction(M);
         ++NumMemCpyInstr;
         return true;
@@ -1535,12 +1583,14 @@ bool MemCpyOptPass::processByValArgument(CallBase &CB, unsigned ArgNo) {
   const DataLayout &DL = CB.getCaller()->getParent()->getDataLayout();
   // Find out what feeds this byval argument.
   Value *ByValArg = CB.getArgOperand(ArgNo);
-  Type *ByValTy = cast<PointerType>(ByValArg->getType())->getElementType();
+  Type *ByValTy = CB.getParamByValType(ArgNo);
   uint64_t ByValSize = DL.getTypeAllocSize(ByValTy);
   MemoryLocation Loc(ByValArg, LocationSize::precise(ByValSize));
   MemCpyInst *MDep = nullptr;
   if (EnableMemorySSA) {
     MemoryUseOrDef *CallAccess = MSSA->getMemoryAccess(&CB);
+    if (!CallAccess)
+      return false;
     MemoryAccess *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(
         CallAccess->getDefiningAccess(), Loc);
     if (auto *MD = dyn_cast<MemoryDef>(Clobber))
@@ -1684,7 +1734,6 @@ PreservedAnalyses MemCpyOptPass::run(Function &F, FunctionAnalysisManager &AM) {
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
-  PA.preserve<GlobalsAA>();
   if (MD)
     PA.preserve<MemoryDependenceAnalysis>();
   if (MSSA)
diff --git a/llvm/lib/Transforms/Scalar/MergeICmps.cpp b/llvm/lib/Transforms/Scalar/MergeICmps.cpp
index 7f8b75ac8806..f13f24ad2027 100644
--- a/llvm/lib/Transforms/Scalar/MergeICmps.cpp
+++ b/llvm/lib/Transforms/Scalar/MergeICmps.cpp
@@ -176,40 +176,39 @@ BCEAtom visitICmpLoadOperand(Value *const Val, BaseIdentifier &BaseId) {
                  Offset);
 }
 
-// A basic block with a comparison between two BCE atoms, e.g. `a == o.a` in the
-// example at the top.
-// The block might do extra work besides the atom comparison, in which case
-// doesOtherWork() returns true. Under some conditions, the block can be
-// split into the atom comparison part and the "other work" part
-// (see canSplit()).
+// A comparison between two BCE atoms, e.g. `a == o.a` in the example at the
+// top.
 // Note: the terminology is misleading: the comparison is symmetric, so there
 // is no real {l/r}hs. What we want though is to have the same base on the
 // left (resp. right), so that we can detect consecutive loads. To ensure this
 // we put the smallest atom on the left.
-class BCECmpBlock {
- public:
-  BCECmpBlock() {}
+struct BCECmp {
+  BCEAtom Lhs;
+  BCEAtom Rhs;
+  int SizeBits;
+  const ICmpInst *CmpI;
 
-  BCECmpBlock(BCEAtom L, BCEAtom R, int SizeBits)
-      : Lhs_(std::move(L)), Rhs_(std::move(R)), SizeBits_(SizeBits) {
-    if (Rhs_ < Lhs_) std::swap(Rhs_, Lhs_);
+  BCECmp(BCEAtom L, BCEAtom R, int SizeBits, const ICmpInst *CmpI)
+      : Lhs(std::move(L)), Rhs(std::move(R)), SizeBits(SizeBits), CmpI(CmpI) {
+    if (Rhs < Lhs) std::swap(Rhs, Lhs);
   }
+};
 
-  bool IsValid() const { return Lhs_.BaseId != 0 && Rhs_.BaseId != 0; }
+// A basic block with a comparison between two BCE atoms.
+// The block might do extra work besides the atom comparison, in which case
+// doesOtherWork() returns true. Under some conditions, the block can be
+// split into the atom comparison part and the "other work" part
+// (see canSplit()).
+class BCECmpBlock {
+ public:
+  typedef SmallDenseSet<const Instruction *, 8> InstructionSet;
 
-  // Assert the block is consistent: If valid, it should also have
-  // non-null members besides Lhs_ and Rhs_.
-  void AssertConsistent() const {
-    if (IsValid()) {
-      assert(BB);
-      assert(CmpI);
-      assert(BranchI);
-    }
-  }
+  BCECmpBlock(BCECmp Cmp, BasicBlock *BB, InstructionSet BlockInsts)
+      : BB(BB), BlockInsts(std::move(BlockInsts)), Cmp(std::move(Cmp)) {}
 
-  const BCEAtom &Lhs() const { return Lhs_; }
-  const BCEAtom &Rhs() const { return Rhs_; }
-  int SizeBits() const { return SizeBits_; }
+  const BCEAtom &Lhs() const { return Cmp.Lhs; }
+  const BCEAtom &Rhs() const { return Cmp.Rhs; }
+  int SizeBits() const { return Cmp.SizeBits; }
 
   // Returns true if the block does other works besides comparison.
   bool doesOtherWork() const;
@@ -222,8 +221,7 @@ class BCECmpBlock {
   // be sunk below this instruction. By doing this, we know we can separate the
   // BCE-cmp-block instructions from the non-BCE-cmp-block instructions in the
   // block.
-  bool canSinkBCECmpInst(const Instruction *, DenseSet<Instruction *> &,
-                         AliasAnalysis &AA) const;
+  bool canSinkBCECmpInst(const Instruction *, AliasAnalysis &AA) const;
 
   // We can separate the BCE-cmp-block instructions and the non-BCE-cmp-block
   // instructions. Split the old block and move all non-BCE-cmp-insts into the
@@ -231,54 +229,45 @@ class BCECmpBlock {
   void split(BasicBlock *NewParent, AliasAnalysis &AA) const;
 
   // The basic block where this comparison happens.
-  BasicBlock *BB = nullptr;
-  // The ICMP for this comparison.
-  ICmpInst *CmpI = nullptr;
-  // The terminating branch.
-  BranchInst *BranchI = nullptr;
+  BasicBlock *BB;
+  // Instructions relating to the BCECmp and branch.
+  InstructionSet BlockInsts;
   // The block requires splitting.
   bool RequireSplit = false;
 
 private:
-  BCEAtom Lhs_;
-  BCEAtom Rhs_;
-  int SizeBits_ = 0;
+  BCECmp Cmp;
 };
 
 bool BCECmpBlock::canSinkBCECmpInst(const Instruction *Inst,
-                                    DenseSet<Instruction *> &BlockInsts,
                                     AliasAnalysis &AA) const {
-  // If this instruction has side effects and its in middle of the BCE cmp block
-  // instructions, then bail for now.
-  if (Inst->mayHaveSideEffects()) {
+  // If this instruction may clobber the loads and is in middle of the BCE cmp
+  // block instructions, then bail for now.
+  if (Inst->mayWriteToMemory()) {
     // Bail if this is not a simple load or store
     if (!isSimpleLoadOrStore(Inst))
       return false;
     // Disallow stores that might alias the BCE operands
-    MemoryLocation LLoc = MemoryLocation::get(Lhs_.LoadI);
-    MemoryLocation RLoc = MemoryLocation::get(Rhs_.LoadI);
+    MemoryLocation LLoc = MemoryLocation::get(Cmp.Lhs.LoadI);
+    MemoryLocation RLoc = MemoryLocation::get(Cmp.Rhs.LoadI);
     if (isModSet(AA.getModRefInfo(Inst, LLoc)) ||
         isModSet(AA.getModRefInfo(Inst, RLoc)))
       return false;
   }
   // Make sure this instruction does not use any of the BCE cmp block
   // instructions as operand.
-  for (auto BI : BlockInsts) {
-    if (is_contained(Inst->operands(), BI))
-      return false;
-  }
-  return true;
+  return llvm::none_of(Inst->operands(), [&](const Value *Op) {
+    const Instruction *OpI = dyn_cast<Instruction>(Op);
+    return OpI && BlockInsts.contains(OpI);
+  });
 }
 
 void BCECmpBlock::split(BasicBlock *NewParent, AliasAnalysis &AA) const {
-  DenseSet<Instruction *> BlockInsts(
-      {Lhs_.GEP, Rhs_.GEP, Lhs_.LoadI, Rhs_.LoadI, CmpI, BranchI});
   llvm::SmallVector<Instruction *, 4> OtherInsts;
   for (Instruction &Inst : *BB) {
     if (BlockInsts.count(&Inst))
       continue;
-      assert(canSinkBCECmpInst(&Inst, BlockInsts, AA) &&
-             "Split unsplittable block");
+    assert(canSinkBCECmpInst(&Inst, AA) && "Split unsplittable block");
     // This is a non-BCE-cmp-block instruction. And it can be separated
     // from the BCE-cmp-block instruction.
     OtherInsts.push_back(&Inst);
@@ -291,11 +280,9 @@ void BCECmpBlock::split(BasicBlock *NewParent, AliasAnalysis &AA) const {
 }
 
 bool BCECmpBlock::canSplit(AliasAnalysis &AA) const {
-  DenseSet<Instruction *> BlockInsts(
-      {Lhs_.GEP, Rhs_.GEP, Lhs_.LoadI, Rhs_.LoadI, CmpI, BranchI});
   for (Instruction &Inst : *BB) {
     if (!BlockInsts.count(&Inst)) {
-      if (!canSinkBCECmpInst(&Inst, BlockInsts, AA))
+      if (!canSinkBCECmpInst(&Inst, AA))
         return false;
     }
   }
@@ -303,10 +290,6 @@ bool BCECmpBlock::canSplit(AliasAnalysis &AA) const {
 }
 
 bool BCECmpBlock::doesOtherWork() const {
-  AssertConsistent();
-  // All the instructions we care about in the BCE cmp block.
-  DenseSet<Instruction *> BlockInsts(
-      {Lhs_.GEP, Rhs_.GEP, Lhs_.LoadI, Rhs_.LoadI, CmpI, BranchI});
   // TODO(courbet): Can we allow some other things ? This is very conservative.
   // We might be able to get away with anything does not have any side
   // effects outside of the basic block.
@@ -320,9 +303,9 @@ bool BCECmpBlock::doesOtherWork() const {
 
 // Visit the given comparison. If this is a comparison between two valid
 // BCE atoms, returns the comparison.
-BCECmpBlock visitICmp(const ICmpInst *const CmpI,
-                      const ICmpInst::Predicate ExpectedPredicate,
-                      BaseIdentifier &BaseId) {
+Optional<BCECmp> visitICmp(const ICmpInst *const CmpI,
+                           const ICmpInst::Predicate ExpectedPredicate,
+                           BaseIdentifier &BaseId) {
   // The comparison can only be used once:
   //  - For intermediate blocks, as a branch condition.
   //  - For the final block, as an incoming value for the Phi.
@@ -330,65 +313,68 @@ BCECmpBlock visitICmp(const ICmpInst *const CmpI,
   // other comparisons as we would create an orphan use of the value.
   if (!CmpI->hasOneUse()) {
     LLVM_DEBUG(dbgs() << "cmp has several uses\n");
-    return {};
+    return None;
   }
   if (CmpI->getPredicate() != ExpectedPredicate)
-    return {};
+    return None;
   LLVM_DEBUG(dbgs() << "cmp "
                     << (ExpectedPredicate == ICmpInst::ICMP_EQ ? "eq" : "ne")
                     << "\n");
   auto Lhs = visitICmpLoadOperand(CmpI->getOperand(0), BaseId);
   if (!Lhs.BaseId)
-    return {};
+    return None;
   auto Rhs = visitICmpLoadOperand(CmpI->getOperand(1), BaseId);
   if (!Rhs.BaseId)
-    return {};
+    return None;
   const auto &DL = CmpI->getModule()->getDataLayout();
-  return BCECmpBlock(std::move(Lhs), std::move(Rhs),
-                     DL.getTypeSizeInBits(CmpI->getOperand(0)->getType()));
+  return BCECmp(std::move(Lhs), std::move(Rhs),
+                DL.getTypeSizeInBits(CmpI->getOperand(0)->getType()), CmpI);
 }
 
 // Visit the given comparison block. If this is a comparison between two valid
 // BCE atoms, returns the comparison.
-BCECmpBlock visitCmpBlock(Value *const Val, BasicBlock *const Block,
-                          const BasicBlock *const PhiBlock,
-                          BaseIdentifier &BaseId) {
-  if (Block->empty()) return {};
+Optional<BCECmpBlock> visitCmpBlock(Value *const Val, BasicBlock *const Block,
+                                    const BasicBlock *const PhiBlock,
+                                    BaseIdentifier &BaseId) {
+  if (Block->empty()) return None;
   auto *const BranchI = dyn_cast<BranchInst>(Block->getTerminator());
-  if (!BranchI) return {};
+  if (!BranchI) return None;
   LLVM_DEBUG(dbgs() << "branch\n");
+  Value *Cond;
+  ICmpInst::Predicate ExpectedPredicate;
   if (BranchI->isUnconditional()) {
     // In this case, we expect an incoming value which is the result of the
     // comparison. This is the last link in the chain of comparisons (note
     // that this does not mean that this is the last incoming value, blocks
     // can be reordered).
-    auto *const CmpI = dyn_cast<ICmpInst>(Val);
-    if (!CmpI) return {};
-    LLVM_DEBUG(dbgs() << "icmp\n");
-    auto Result = visitICmp(CmpI, ICmpInst::ICMP_EQ, BaseId);
-    Result.CmpI = CmpI;
-    Result.BranchI = BranchI;
-    return Result;
+    Cond = Val;
+    ExpectedPredicate = ICmpInst::ICMP_EQ;
   } else {
     // In this case, we expect a constant incoming value (the comparison is
     // chained).
     const auto *const Const = cast<ConstantInt>(Val);
     LLVM_DEBUG(dbgs() << "const\n");
-    if (!Const->isZero()) return {};
+    if (!Const->isZero()) return None;
     LLVM_DEBUG(dbgs() << "false\n");
-    auto *const CmpI = dyn_cast<ICmpInst>(BranchI->getCondition());
-    if (!CmpI) return {};
-    LLVM_DEBUG(dbgs() << "icmp\n");
     assert(BranchI->getNumSuccessors() == 2 && "expecting a cond branch");
     BasicBlock *const FalseBlock = BranchI->getSuccessor(1);
-    auto Result = visitICmp(
-        CmpI, FalseBlock == PhiBlock ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE,
-        BaseId);
-    Result.CmpI = CmpI;
-    Result.BranchI = BranchI;
-    return Result;
+    Cond = BranchI->getCondition();
+    ExpectedPredicate =
+        FalseBlock == PhiBlock ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE;
   }
-  return {};
+
+  auto *CmpI = dyn_cast<ICmpInst>(Cond);
+  if (!CmpI) return None;
+  LLVM_DEBUG(dbgs() << "icmp\n");
+
+  Optional<BCECmp> Result = visitICmp(CmpI, ExpectedPredicate, BaseId);
+  if (!Result)
+    return None;
+
+  BCECmpBlock::InstructionSet BlockInsts(
+      {Result->Lhs.GEP, Result->Rhs.GEP, Result->Lhs.LoadI, Result->Rhs.LoadI,
+       Result->CmpI, BranchI});
+  return BCECmpBlock(std::move(*Result), Block, BlockInsts);
 }
 
 static inline void enqueueBlock(std::vector<BCECmpBlock> &Comparisons,
@@ -440,18 +426,16 @@ BCECmpChain::BCECmpChain(const std::vector<BasicBlock *> &Blocks, PHINode &Phi,
   // Now look inside blocks to check for BCE comparisons.
   std::vector<BCECmpBlock> Comparisons;
   BaseIdentifier BaseId;
-  for (size_t BlockIdx = 0; BlockIdx < Blocks.size(); ++BlockIdx) {
-    BasicBlock *const Block = Blocks[BlockIdx];
+  for (BasicBlock *const Block : Blocks) {
     assert(Block && "invalid block");
-    BCECmpBlock Comparison = visitCmpBlock(Phi.getIncomingValueForBlock(Block),
-                                           Block, Phi.getParent(), BaseId);
-    Comparison.BB = Block;
-    if (!Comparison.IsValid()) {
+    Optional<BCECmpBlock> Comparison = visitCmpBlock(
+        Phi.getIncomingValueForBlock(Block), Block, Phi.getParent(), BaseId);
+    if (!Comparison) {
       LLVM_DEBUG(dbgs() << "chain with invalid BCECmpBlock, no merge.\n");
       return;
     }
-    if (Comparison.doesOtherWork()) {
-      LLVM_DEBUG(dbgs() << "block '" << Comparison.BB->getName()
+    if (Comparison->doesOtherWork()) {
+      LLVM_DEBUG(dbgs() << "block '" << Comparison->BB->getName()
                         << "' does extra work besides compare\n");
       if (Comparisons.empty()) {
         // This is the initial block in the chain, in case this block does other
@@ -467,15 +451,15 @@ BCECmpChain::BCECmpChain(const std::vector<BasicBlock *> &Blocks, PHINode &Phi,
         // and start anew.
         //
         // NOTE: we only handle blocks a with single predecessor for now.
-        if (Comparison.canSplit(AA)) {
+        if (Comparison->canSplit(AA)) {
           LLVM_DEBUG(dbgs()
-                     << "Split initial block '" << Comparison.BB->getName()
+                     << "Split initial block '" << Comparison->BB->getName()
                      << "' that does extra work besides compare\n");
-          Comparison.RequireSplit = true;
-          enqueueBlock(Comparisons, std::move(Comparison));
+          Comparison->RequireSplit = true;
+          enqueueBlock(Comparisons, std::move(*Comparison));
         } else {
           LLVM_DEBUG(dbgs()
-                     << "ignoring initial block '" << Comparison.BB->getName()
+                     << "ignoring initial block '" << Comparison->BB->getName()
                      << "' that does extra work besides compare\n");
         }
         continue;
@@ -505,7 +489,7 @@ BCECmpChain::BCECmpChain(const std::vector<BasicBlock *> &Blocks, PHINode &Phi,
       // We could still merge bb1 and bb2 though.
       return;
     }
-    enqueueBlock(Comparisons, std::move(Comparison));
+    enqueueBlock(Comparisons, std::move(*Comparison));
   }
 
   // It is possible we have no suitable comparison to merge.
@@ -597,7 +581,7 @@ private:
         append(BB->getName());
       }
     }
-    return StringRef(Scratch);
+    return Scratch.str();
   }
 };
 } // namespace
@@ -733,8 +717,7 @@ bool BCECmpChain::simplify(const TargetLibraryInfo &TLI, AliasAnalysis &AA,
 
   // If the old cmp chain was the function entry, we need to update the function
   // entry.
-  const bool ChainEntryIsFnEntry =
-      (EntryBlock_ == &EntryBlock_->getParent()->getEntryBlock());
+  const bool ChainEntryIsFnEntry = EntryBlock_->isEntryBlock();
   if (ChainEntryIsFnEntry && DTU.hasDomTree()) {
     LLVM_DEBUG(dbgs() << "Changing function entry from "
                       << EntryBlock_->getName() << " to "
@@ -940,7 +923,6 @@ PreservedAnalyses MergeICmpsPass::run(Function &F,
   if (!MadeChanges)
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
-  PA.preserve<GlobalsAA>();
   PA.preserve<DominatorTreeAnalysis>();
   return PA;
 }
diff --git a/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp b/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
index 69aa0cebe170..033fc168a67f 100644
--- a/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
+++ b/llvm/lib/Transforms/Scalar/MergedLoadStoreMotion.cpp
@@ -418,6 +418,5 @@ MergedLoadStoreMotionPass::run(Function &F, FunctionAnalysisManager &AM) {
   PreservedAnalyses PA;
   if (!Options.SplitFooterBB)
     PA.preserveSet<CFGAnalyses>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
diff --git a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
index 32bb62129e8f..ded5caf53b5a 100644
--- a/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
+++ b/llvm/lib/Transforms/Scalar/NaryReassociate.cpp
@@ -80,6 +80,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/AssumptionCache.h"
 #include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -106,6 +107,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/ScalarEvolutionExpander.h"
 #include <cassert>
 #include <cstdint>
 
@@ -222,15 +224,14 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
   SmallVector<WeakTrackingVH, 16> DeadInsts;
   for (const auto Node : depth_first(DT)) {
     BasicBlock *BB = Node->getBlock();
-    for (auto I = BB->begin(); I != BB->end(); ++I) {
-      Instruction *OrigI = &*I;
+    for (Instruction &OrigI : *BB) {
       const SCEV *OrigSCEV = nullptr;
-      if (Instruction *NewI = tryReassociate(OrigI, OrigSCEV)) {
+      if (Instruction *NewI = tryReassociate(&OrigI, OrigSCEV)) {
         Changed = true;
-        OrigI->replaceAllUsesWith(NewI);
+        OrigI.replaceAllUsesWith(NewI);
 
         // Add 'OrigI' to the list of dead instructions.
-        DeadInsts.push_back(WeakTrackingVH(OrigI));
+        DeadInsts.push_back(WeakTrackingVH(&OrigI));
         // Add the rewritten instruction to SeenExprs; the original
         // instruction is deleted.
         const SCEV *NewSCEV = SE->getSCEV(NewI);
@@ -258,7 +259,7 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
         if (NewSCEV != OrigSCEV)
           SeenExprs[OrigSCEV].push_back(WeakTrackingVH(NewI));
       } else if (OrigSCEV)
-        SeenExprs[OrigSCEV].push_back(WeakTrackingVH(OrigI));
+        SeenExprs[OrigSCEV].push_back(WeakTrackingVH(&OrigI));
     }
   }
   // Delete all dead instructions from 'DeadInsts'.
@@ -269,6 +270,24 @@ bool NaryReassociatePass::doOneIteration(Function &F) {
   return Changed;
 }
 
+template <typename PredT>
+Instruction *
+NaryReassociatePass::matchAndReassociateMinOrMax(Instruction *I,
+                                                 const SCEV *&OrigSCEV) {
+  Value *LHS = nullptr;
+  Value *RHS = nullptr;
+
+  auto MinMaxMatcher =
+      MaxMin_match<ICmpInst, bind_ty<Value>, bind_ty<Value>, PredT>(
+          m_Value(LHS), m_Value(RHS));
+  if (match(I, MinMaxMatcher)) {
+    OrigSCEV = SE->getSCEV(I);
+    return dyn_cast_or_null<Instruction>(
+        tryReassociateMinOrMax(I, MinMaxMatcher, LHS, RHS));
+  }
+  return nullptr;
+}
+
 Instruction *NaryReassociatePass::tryReassociate(Instruction * I,
                                                  const SCEV *&OrigSCEV) {
 
@@ -284,10 +303,21 @@ Instruction *NaryReassociatePass::tryReassociate(Instruction * I,
     OrigSCEV = SE->getSCEV(I);
     return tryReassociateGEP(cast<GetElementPtrInst>(I));
   default:
-    return nullptr;
+    break;
   }
 
-  llvm_unreachable("should not be reached");
+  // Try to match signed/unsigned Min/Max.
+  Instruction *ResI = nullptr;
+  // TODO: Currently min/max reassociation is restricted to integer types only
+  // due to use of SCEVExpander which my introduce incompatible forms of min/max
+  // for pointer types.
+  if (I->getType()->isIntegerTy())
+    if ((ResI = matchAndReassociateMinOrMax<umin_pred_ty>(I, OrigSCEV)) ||
+        (ResI = matchAndReassociateMinOrMax<smin_pred_ty>(I, OrigSCEV)) ||
+        (ResI = matchAndReassociateMinOrMax<umax_pred_ty>(I, OrigSCEV)) ||
+        (ResI = matchAndReassociateMinOrMax<smax_pred_ty>(I, OrigSCEV)))
+      return ResI;
+
   return nullptr;
 }
 
@@ -364,8 +394,8 @@ NaryReassociatePass::tryReassociateGEPAtIndex(GetElementPtrInst *GEP,
   // Look for GEP's closest dominator that has the same SCEV as GEP except that
   // the I-th index is replaced with LHS.
   SmallVector<const SCEV *, 4> IndexExprs;
-  for (auto Index = GEP->idx_begin(); Index != GEP->idx_end(); ++Index)
-    IndexExprs.push_back(SE->getSCEV(*Index));
+  for (Use &Index : GEP->indices())
+    IndexExprs.push_back(SE->getSCEV(Index));
   // Replace the I-th index with LHS.
   IndexExprs[I] = SE->getSCEV(LHS);
   if (isKnownNonNegative(LHS, *DL, 0, AC, GEP, DT) &&
@@ -540,3 +570,84 @@ NaryReassociatePass::findClosestMatchingDominator(const SCEV *CandidateExpr,
   }
   return nullptr;
 }
+
+template <typename MaxMinT> static SCEVTypes convertToSCEVype(MaxMinT &MM) {
+  if (std::is_same<smax_pred_ty, typename MaxMinT::PredType>::value)
+    return scSMaxExpr;
+  else if (std::is_same<umax_pred_ty, typename MaxMinT::PredType>::value)
+    return scUMaxExpr;
+  else if (std::is_same<smin_pred_ty, typename MaxMinT::PredType>::value)
+    return scSMinExpr;
+  else if (std::is_same<umin_pred_ty, typename MaxMinT::PredType>::value)
+    return scUMinExpr;
+
+  llvm_unreachable("Can't convert MinMax pattern to SCEV type");
+  return scUnknown;
+}
+
+// Parameters:
+//  I - instruction matched by MaxMinMatch matcher
+//  MaxMinMatch - min/max idiom matcher
+//  LHS - first operand of I
+//  RHS - second operand of I
+template <typename MaxMinT>
+Value *NaryReassociatePass::tryReassociateMinOrMax(Instruction *I,
+                                                   MaxMinT MaxMinMatch,
+                                                   Value *LHS, Value *RHS) {
+  Value *A = nullptr, *B = nullptr;
+  MaxMinT m_MaxMin(m_Value(A), m_Value(B));
+  for (unsigned int i = 0; i < 2; ++i) {
+    if (!LHS->hasNUsesOrMore(3) && match(LHS, m_MaxMin)) {
+      const SCEV *AExpr = SE->getSCEV(A), *BExpr = SE->getSCEV(B);
+      const SCEV *RHSExpr = SE->getSCEV(RHS);
+      for (unsigned int j = 0; j < 2; ++j) {
+        if (j == 0) {
+          if (BExpr == RHSExpr)
+            continue;
+          // Transform 'I = (A op B) op RHS' to 'I = (A op RHS) op B' on the
+          // first iteration.
+          std::swap(BExpr, RHSExpr);
+        } else {
+          if (AExpr == RHSExpr)
+            continue;
+          // Transform 'I = (A op RHS) op B' 'I = (B op RHS) op A' on the second
+          // iteration.
+          std::swap(AExpr, RHSExpr);
+        }
+
+        // The optimization is profitable only if LHS can be removed in the end.
+        // In other words LHS should be used (directly or indirectly) by I only.
+        if (llvm::any_of(LHS->users(), [&](auto *U) {
+              return U != I && !(U->hasOneUser() && *U->users().begin() == I);
+            }))
+          continue;
+
+        SCEVExpander Expander(*SE, *DL, "nary-reassociate");
+        SmallVector<const SCEV *, 2> Ops1{ BExpr, AExpr };
+        const SCEVTypes SCEVType = convertToSCEVype(m_MaxMin);
+        const SCEV *R1Expr = SE->getMinMaxExpr(SCEVType, Ops1);
+
+        Instruction *R1MinMax = findClosestMatchingDominator(R1Expr, I);
+
+        if (!R1MinMax)
+          continue;
+
+        LLVM_DEBUG(dbgs() << "NARY: Found common sub-expr: " << *R1MinMax
+                          << "\n");
+
+        R1Expr = SE->getUnknown(R1MinMax);
+        SmallVector<const SCEV *, 2> Ops2{ RHSExpr, R1Expr };
+        const SCEV *R2Expr = SE->getMinMaxExpr(SCEVType, Ops2);
+
+        Value *NewMinMax = Expander.expandCodeFor(R2Expr, I->getType(), I);
+        NewMinMax->setName(Twine(I->getName()).concat(".nary"));
+
+        LLVM_DEBUG(dbgs() << "NARY: Deleting:  " << *I << "\n"
+                          << "NARY: Inserting: " << *NewMinMax << "\n");
+        return NewMinMax;
+      }
+    }
+    std::swap(LHS, RHS);
+  }
+  return nullptr;
+}
diff --git a/llvm/lib/Transforms/Scalar/NewGVN.cpp b/llvm/lib/Transforms/Scalar/NewGVN.cpp
index 281d47c8625f..a137d13c6ea0 100644
--- a/llvm/lib/Transforms/Scalar/NewGVN.cpp
+++ b/llvm/lib/Transforms/Scalar/NewGVN.cpp
@@ -62,6 +62,7 @@
 #include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/SetOperations.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/SparseBitVector.h"
@@ -389,8 +390,7 @@ public:
     if (Members.size() != Other->Members.size())
       return false;
 
-    return all_of(Members,
-                  [&](const Value *V) { return Other->Members.count(V); });
+    return llvm::set_is_subset(Members, Other->Members);
   }
 
 private:
@@ -668,8 +668,45 @@ public:
   bool runGVN();
 
 private:
+  /// Helper struct return a Expression with an optional extra dependency.
+  struct ExprResult {
+    const Expression *Expr;
+    Value *ExtraDep;
+    const PredicateBase *PredDep;
+
+    ExprResult(const Expression *Expr, Value *ExtraDep = nullptr,
+               const PredicateBase *PredDep = nullptr)
+        : Expr(Expr), ExtraDep(ExtraDep), PredDep(PredDep) {}
+    ExprResult(const ExprResult &) = delete;
+    ExprResult(ExprResult &&Other)
+        : Expr(Other.Expr), ExtraDep(Other.ExtraDep), PredDep(Other.PredDep) {
+      Other.Expr = nullptr;
+      Other.ExtraDep = nullptr;
+      Other.PredDep = nullptr;
+    }
+    ExprResult &operator=(const ExprResult &Other) = delete;
+    ExprResult &operator=(ExprResult &&Other) = delete;
+
+    ~ExprResult() { assert(!ExtraDep && "unhandled ExtraDep"); }
+
+    operator bool() const { return Expr; }
+
+    static ExprResult none() { return {nullptr, nullptr, nullptr}; }
+    static ExprResult some(const Expression *Expr, Value *ExtraDep = nullptr) {
+      return {Expr, ExtraDep, nullptr};
+    }
+    static ExprResult some(const Expression *Expr,
+                           const PredicateBase *PredDep) {
+      return {Expr, nullptr, PredDep};
+    }
+    static ExprResult some(const Expression *Expr, Value *ExtraDep,
+                           const PredicateBase *PredDep) {
+      return {Expr, ExtraDep, PredDep};
+    }
+  };
+
   // Expression handling.
-  const Expression *createExpression(Instruction *) const;
+  ExprResult createExpression(Instruction *) const;
   const Expression *createBinaryExpression(unsigned, Type *, Value *, Value *,
                                            Instruction *) const;
 
@@ -742,23 +779,22 @@ private:
   void valueNumberInstruction(Instruction *);
 
   // Symbolic evaluation.
-  const Expression *checkSimplificationResults(Expression *, Instruction *,
-                                               Value *) const;
-  const Expression *performSymbolicEvaluation(Value *,
-                                              SmallPtrSetImpl<Value *> &) const;
+  ExprResult checkExprResults(Expression *, Instruction *, Value *) const;
+  ExprResult performSymbolicEvaluation(Value *,
+                                       SmallPtrSetImpl<Value *> &) const;
   const Expression *performSymbolicLoadCoercion(Type *, Value *, LoadInst *,
                                                 Instruction *,
                                                 MemoryAccess *) const;
   const Expression *performSymbolicLoadEvaluation(Instruction *) const;
   const Expression *performSymbolicStoreEvaluation(Instruction *) const;
-  const Expression *performSymbolicCallEvaluation(Instruction *) const;
+  ExprResult performSymbolicCallEvaluation(Instruction *) const;
   void sortPHIOps(MutableArrayRef<ValPair> Ops) const;
   const Expression *performSymbolicPHIEvaluation(ArrayRef<ValPair>,
                                                  Instruction *I,
                                                  BasicBlock *PHIBlock) const;
   const Expression *performSymbolicAggrValueEvaluation(Instruction *) const;
-  const Expression *performSymbolicCmpEvaluation(Instruction *) const;
-  const Expression *performSymbolicPredicateInfoEvaluation(Instruction *) const;
+  ExprResult performSymbolicCmpEvaluation(Instruction *) const;
+  ExprResult performSymbolicPredicateInfoEvaluation(Instruction *) const;
 
   // Congruence finding.
   bool someEquivalentDominates(const Instruction *, const Instruction *) const;
@@ -811,9 +847,9 @@ private:
   void markValueLeaderChangeTouched(CongruenceClass *CC);
   void markMemoryLeaderChangeTouched(CongruenceClass *CC);
   void markPhiOfOpsChanged(const Expression *E);
-  void addPredicateUsers(const PredicateBase *, Instruction *) const;
   void addMemoryUsers(const MemoryAccess *To, MemoryAccess *U) const;
   void addAdditionalUsers(Value *To, Value *User) const;
+  void addAdditionalUsers(ExprResult &Res, Instruction *User) const;
 
   // Main loop of value numbering
   void iterateTouchedInstructions();
@@ -1052,19 +1088,21 @@ const Expression *NewGVN::createBinaryExpression(unsigned Opcode, Type *T,
   E->op_push_back(lookupOperandLeader(Arg2));
 
   Value *V = SimplifyBinOp(Opcode, E->getOperand(0), E->getOperand(1), SQ);
-  if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
-    return SimplifiedE;
+  if (auto Simplified = checkExprResults(E, I, V)) {
+    addAdditionalUsers(Simplified, I);
+    return Simplified.Expr;
+  }
   return E;
 }
 
 // Take a Value returned by simplification of Expression E/Instruction
 // I, and see if it resulted in a simpler expression. If so, return
 // that expression.
-const Expression *NewGVN::checkSimplificationResults(Expression *E,
-                                                     Instruction *I,
-                                                     Value *V) const {
+NewGVN::ExprResult NewGVN::checkExprResults(Expression *E, Instruction *I,
+                                            Value *V) const {
   if (!V)
-    return nullptr;
+    return ExprResult::none();
+
   if (auto *C = dyn_cast<Constant>(V)) {
     if (I)
       LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
@@ -1073,52 +1111,37 @@ const Expression *NewGVN::checkSimplificationResults(Expression *E,
     assert(isa<BasicExpression>(E) &&
            "We should always have had a basic expression here");
     deleteExpression(E);
-    return createConstantExpression(C);
+    return ExprResult::some(createConstantExpression(C));
   } else if (isa<Argument>(V) || isa<GlobalVariable>(V)) {
     if (I)
       LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
                         << " variable " << *V << "\n");
     deleteExpression(E);
-    return createVariableExpression(V);
+    return ExprResult::some(createVariableExpression(V));
   }
 
   CongruenceClass *CC = ValueToClass.lookup(V);
   if (CC) {
     if (CC->getLeader() && CC->getLeader() != I) {
-      // If we simplified to something else, we need to communicate
-      // that we're users of the value we simplified to.
-      if (I != V) {
-        // Don't add temporary instructions to the user lists.
-        if (!AllTempInstructions.count(I))
-          addAdditionalUsers(V, I);
-      }
-      return createVariableOrConstant(CC->getLeader());
+      return ExprResult::some(createVariableOrConstant(CC->getLeader()), V);
     }
     if (CC->getDefiningExpr()) {
-      // If we simplified to something else, we need to communicate
-      // that we're users of the value we simplified to.
-      if (I != V) {
-        // Don't add temporary instructions to the user lists.
-        if (!AllTempInstructions.count(I))
-          addAdditionalUsers(V, I);
-      }
-
       if (I)
         LLVM_DEBUG(dbgs() << "Simplified " << *I << " to "
                           << " expression " << *CC->getDefiningExpr() << "\n");
       NumGVNOpsSimplified++;
       deleteExpression(E);
-      return CC->getDefiningExpr();
+      return ExprResult::some(CC->getDefiningExpr(), V);
     }
   }
 
-  return nullptr;
+  return ExprResult::none();
 }
 
 // Create a value expression from the instruction I, replacing operands with
 // their leaders.
 
-const Expression *NewGVN::createExpression(Instruction *I) const {
+NewGVN::ExprResult NewGVN::createExpression(Instruction *I) const {
   auto *E = new (ExpressionAllocator) BasicExpression(I->getNumOperands());
 
   bool AllConstant = setBasicExpressionInfo(I, E);
@@ -1149,8 +1172,8 @@ const Expression *NewGVN::createExpression(Instruction *I) const {
             E->getOperand(1)->getType() == I->getOperand(1)->getType()));
     Value *V =
         SimplifyCmpInst(Predicate, E->getOperand(0), E->getOperand(1), SQ);
-    if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
-      return SimplifiedE;
+    if (auto Simplified = checkExprResults(E, I, V))
+      return Simplified;
   } else if (isa<SelectInst>(I)) {
     if (isa<Constant>(E->getOperand(0)) ||
         E->getOperand(1) == E->getOperand(2)) {
@@ -1158,24 +1181,24 @@ const Expression *NewGVN::createExpression(Instruction *I) const {
              E->getOperand(2)->getType() == I->getOperand(2)->getType());
       Value *V = SimplifySelectInst(E->getOperand(0), E->getOperand(1),
                                     E->getOperand(2), SQ);
-      if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
-        return SimplifiedE;
+      if (auto Simplified = checkExprResults(E, I, V))
+        return Simplified;
     }
   } else if (I->isBinaryOp()) {
     Value *V =
         SimplifyBinOp(E->getOpcode(), E->getOperand(0), E->getOperand(1), SQ);
-    if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
-      return SimplifiedE;
+    if (auto Simplified = checkExprResults(E, I, V))
+      return Simplified;
   } else if (auto *CI = dyn_cast<CastInst>(I)) {
     Value *V =
         SimplifyCastInst(CI->getOpcode(), E->getOperand(0), CI->getType(), SQ);
-    if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
-      return SimplifiedE;
+    if (auto Simplified = checkExprResults(E, I, V))
+      return Simplified;
   } else if (isa<GetElementPtrInst>(I)) {
     Value *V = SimplifyGEPInst(
         E->getType(), ArrayRef<Value *>(E->op_begin(), E->op_end()), SQ);
-    if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
-      return SimplifiedE;
+    if (auto Simplified = checkExprResults(E, I, V))
+      return Simplified;
   } else if (AllConstant) {
     // We don't bother trying to simplify unless all of the operands
     // were constant.
@@ -1189,10 +1212,10 @@ const Expression *NewGVN::createExpression(Instruction *I) const {
       C.emplace_back(cast<Constant>(Arg));
 
     if (Value *V = ConstantFoldInstOperands(I, C, DL, TLI))
-      if (const Expression *SimplifiedE = checkSimplificationResults(E, I, V))
-        return SimplifiedE;
+      if (auto Simplified = checkExprResults(E, I, V))
+        return Simplified;
   }
-  return E;
+  return ExprResult::some(E);
 }
 
 const AggregateValueExpression *
@@ -1527,17 +1550,17 @@ const Expression *NewGVN::performSymbolicLoadEvaluation(Instruction *I) const {
   return LE;
 }
 
-const Expression *
+NewGVN::ExprResult
 NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) const {
   auto *PI = PredInfo->getPredicateInfoFor(I);
   if (!PI)
-    return nullptr;
+    return ExprResult::none();
 
   LLVM_DEBUG(dbgs() << "Found predicate info from instruction !\n");
 
   const Optional<PredicateConstraint> &Constraint = PI->getConstraint();
   if (!Constraint)
-    return nullptr;
+    return ExprResult::none();
 
   CmpInst::Predicate Predicate = Constraint->Predicate;
   Value *CmpOp0 = I->getOperand(0);
@@ -1554,45 +1577,43 @@ NewGVN::performSymbolicPredicateInfoEvaluation(Instruction *I) const {
     AdditionallyUsedValue = CmpOp1;
   }
 
-  if (Predicate == CmpInst::ICMP_EQ) {
-    addPredicateUsers(PI, I);
-    addAdditionalUsers(AdditionallyUsedValue, I);
-    return createVariableOrConstant(FirstOp);
-  }
+  if (Predicate == CmpInst::ICMP_EQ)
+    return ExprResult::some(createVariableOrConstant(FirstOp),
+                            AdditionallyUsedValue, PI);
 
   // Handle the special case of floating point.
   if (Predicate == CmpInst::FCMP_OEQ && isa<ConstantFP>(FirstOp) &&
-      !cast<ConstantFP>(FirstOp)->isZero()) {
-    addPredicateUsers(PI, I);
-    addAdditionalUsers(AdditionallyUsedValue, I);
-    return createConstantExpression(cast<Constant>(FirstOp));
-  }
+      !cast<ConstantFP>(FirstOp)->isZero())
+    return ExprResult::some(createConstantExpression(cast<Constant>(FirstOp)),
+                            AdditionallyUsedValue, PI);
 
-  return nullptr;
+  return ExprResult::none();
 }
 
 // Evaluate read only and pure calls, and create an expression result.
-const Expression *NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
+NewGVN::ExprResult NewGVN::performSymbolicCallEvaluation(Instruction *I) const {
   auto *CI = cast<CallInst>(I);
   if (auto *II = dyn_cast<IntrinsicInst>(I)) {
     // Intrinsics with the returned attribute are copies of arguments.
     if (auto *ReturnedValue = II->getReturnedArgOperand()) {
       if (II->getIntrinsicID() == Intrinsic::ssa_copy)
-        if (const auto *Result = performSymbolicPredicateInfoEvaluation(I))
-          return Result;
-      return createVariableOrConstant(ReturnedValue);
+        if (auto Res = performSymbolicPredicateInfoEvaluation(I))
+          return Res;
+      return ExprResult::some(createVariableOrConstant(ReturnedValue));
     }
   }
   if (AA->doesNotAccessMemory(CI)) {
-    return createCallExpression(CI, TOPClass->getMemoryLeader());
+    return ExprResult::some(
+        createCallExpression(CI, TOPClass->getMemoryLeader()));
   } else if (AA->onlyReadsMemory(CI)) {
     if (auto *MA = MSSA->getMemoryAccess(CI)) {
       auto *DefiningAccess = MSSAWalker->getClobberingMemoryAccess(MA);
-      return createCallExpression(CI, DefiningAccess);
+      return ExprResult::some(createCallExpression(CI, DefiningAccess));
     } else // MSSA determined that CI does not access memory.
-      return createCallExpression(CI, TOPClass->getMemoryLeader());
+      return ExprResult::some(
+          createCallExpression(CI, TOPClass->getMemoryLeader()));
   }
-  return nullptr;
+  return ExprResult::none();
 }
 
 // Retrieve the memory class for a given MemoryAccess.
@@ -1778,7 +1799,7 @@ NewGVN::performSymbolicAggrValueEvaluation(Instruction *I) const {
   return createAggregateValueExpression(I);
 }
 
-const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
+NewGVN::ExprResult NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
   assert(isa<CmpInst>(I) && "Expected a cmp instruction.");
 
   auto *CI = cast<CmpInst>(I);
@@ -1798,14 +1819,17 @@ const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
   // of an assume.
   auto *CmpPI = PredInfo->getPredicateInfoFor(I);
   if (dyn_cast_or_null<PredicateAssume>(CmpPI))
-    return createConstantExpression(ConstantInt::getTrue(CI->getType()));
+    return ExprResult::some(
+        createConstantExpression(ConstantInt::getTrue(CI->getType())));
 
   if (Op0 == Op1) {
     // This condition does not depend on predicates, no need to add users
     if (CI->isTrueWhenEqual())
-      return createConstantExpression(ConstantInt::getTrue(CI->getType()));
+      return ExprResult::some(
+          createConstantExpression(ConstantInt::getTrue(CI->getType())));
     else if (CI->isFalseWhenEqual())
-      return createConstantExpression(ConstantInt::getFalse(CI->getType()));
+      return ExprResult::some(
+          createConstantExpression(ConstantInt::getFalse(CI->getType())));
   }
 
   // NOTE: Because we are comparing both operands here and below, and using
@@ -1864,31 +1888,31 @@ const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
           // edge then we may be implied true or false.
           if (CmpInst::isImpliedTrueByMatchingCmp(BranchPredicate,
                                                   OurPredicate)) {
-            addPredicateUsers(PI, I);
-            return createConstantExpression(
-                ConstantInt::getTrue(CI->getType()));
+            return ExprResult::some(
+                createConstantExpression(ConstantInt::getTrue(CI->getType())),
+                PI);
           }
 
           if (CmpInst::isImpliedFalseByMatchingCmp(BranchPredicate,
                                                    OurPredicate)) {
-            addPredicateUsers(PI, I);
-            return createConstantExpression(
-                ConstantInt::getFalse(CI->getType()));
+            return ExprResult::some(
+                createConstantExpression(ConstantInt::getFalse(CI->getType())),
+                PI);
           }
         } else {
           // Just handle the ne and eq cases, where if we have the same
           // operands, we may know something.
           if (BranchPredicate == OurPredicate) {
-            addPredicateUsers(PI, I);
             // Same predicate, same ops,we know it was false, so this is false.
-            return createConstantExpression(
-                ConstantInt::getFalse(CI->getType()));
+            return ExprResult::some(
+                createConstantExpression(ConstantInt::getFalse(CI->getType())),
+                PI);
           } else if (BranchPredicate ==
                      CmpInst::getInversePredicate(OurPredicate)) {
-            addPredicateUsers(PI, I);
             // Inverse predicate, we know the other was false, so this is true.
-            return createConstantExpression(
-                ConstantInt::getTrue(CI->getType()));
+            return ExprResult::some(
+                createConstantExpression(ConstantInt::getTrue(CI->getType())),
+                PI);
           }
         }
       }
@@ -1899,9 +1923,10 @@ const Expression *NewGVN::performSymbolicCmpEvaluation(Instruction *I) const {
 }
 
 // Substitute and symbolize the value before value numbering.
-const Expression *
+NewGVN::ExprResult
 NewGVN::performSymbolicEvaluation(Value *V,
                                   SmallPtrSetImpl<Value *> &Visited) const {
+
   const Expression *E = nullptr;
   if (auto *C = dyn_cast<Constant>(V))
     E = createConstantExpression(C);
@@ -1927,7 +1952,7 @@ NewGVN::performSymbolicEvaluation(Value *V,
       E = performSymbolicPHIEvaluation(Ops, I, getBlockForValue(I));
     } break;
     case Instruction::Call:
-      E = performSymbolicCallEvaluation(I);
+      return performSymbolicCallEvaluation(I);
       break;
     case Instruction::Store:
       E = performSymbolicStoreEvaluation(I);
@@ -1937,11 +1962,11 @@ NewGVN::performSymbolicEvaluation(Value *V,
       break;
     case Instruction::BitCast:
     case Instruction::AddrSpaceCast:
-      E = createExpression(I);
+      return createExpression(I);
       break;
     case Instruction::ICmp:
     case Instruction::FCmp:
-      E = performSymbolicCmpEvaluation(I);
+      return performSymbolicCmpEvaluation(I);
       break;
     case Instruction::FNeg:
     case Instruction::Add:
@@ -1977,16 +2002,16 @@ NewGVN::performSymbolicEvaluation(Value *V,
     case Instruction::ExtractElement:
     case Instruction::InsertElement:
     case Instruction::GetElementPtr:
-      E = createExpression(I);
+      return createExpression(I);
       break;
     case Instruction::ShuffleVector:
       // FIXME: Add support for shufflevector to createExpression.
-      return nullptr;
+      return ExprResult::none();
     default:
-      return nullptr;
+      return ExprResult::none();
     }
   }
-  return E;
+  return ExprResult::some(E);
 }
 
 // Look up a container of values/instructions in a map, and touch all the
@@ -2007,6 +2032,20 @@ void NewGVN::addAdditionalUsers(Value *To, Value *User) const {
     AdditionalUsers[To].insert(User);
 }
 
+void NewGVN::addAdditionalUsers(ExprResult &Res, Instruction *User) const {
+  if (Res.ExtraDep && Res.ExtraDep != User)
+    addAdditionalUsers(Res.ExtraDep, User);
+  Res.ExtraDep = nullptr;
+
+  if (Res.PredDep) {
+    if (const auto *PBranch = dyn_cast<PredicateBranch>(Res.PredDep))
+      PredicateToUsers[PBranch->Condition].insert(User);
+    else if (const auto *PAssume = dyn_cast<PredicateAssume>(Res.PredDep))
+      PredicateToUsers[PAssume->Condition].insert(User);
+  }
+  Res.PredDep = nullptr;
+}
+
 void NewGVN::markUsersTouched(Value *V) {
   // Now mark the users as touched.
   for (auto *User : V->users()) {
@@ -2033,18 +2072,6 @@ void NewGVN::markMemoryUsersTouched(const MemoryAccess *MA) {
   touchAndErase(MemoryToUsers, MA);
 }
 
-// Add I to the set of users of a given predicate.
-void NewGVN::addPredicateUsers(const PredicateBase *PB, Instruction *I) const {
-  // Don't add temporary instructions to the user lists.
-  if (AllTempInstructions.count(I))
-    return;
-
-  if (auto *PBranch = dyn_cast<PredicateBranch>(PB))
-    PredicateToUsers[PBranch->Condition].insert(I);
-  else if (auto *PAssume = dyn_cast<PredicateAssume>(PB))
-    PredicateToUsers[PAssume->Condition].insert(I);
-}
-
 // Touch all the predicates that depend on this instruction.
 void NewGVN::markPredicateUsersTouched(Instruction *I) {
   touchAndErase(PredicateToUsers, I);
@@ -2414,9 +2441,15 @@ void NewGVN::processOutgoingEdges(Instruction *TI, BasicBlock *B) {
     Value *CondEvaluated = findConditionEquivalence(Cond);
     if (!CondEvaluated) {
       if (auto *I = dyn_cast<Instruction>(Cond)) {
-        const Expression *E = createExpression(I);
-        if (const auto *CE = dyn_cast<ConstantExpression>(E)) {
+        SmallPtrSet<Value *, 4> Visited;
+        auto Res = performSymbolicEvaluation(I, Visited);
+        if (const auto *CE = dyn_cast_or_null<ConstantExpression>(Res.Expr)) {
           CondEvaluated = CE->getConstantValue();
+          addAdditionalUsers(Res, I);
+        } else {
+          // Did not use simplification result, no need to add the extra
+          // dependency.
+          Res.ExtraDep = nullptr;
         }
       } else if (isa<ConstantInt>(Cond)) {
         CondEvaluated = Cond;
@@ -2600,7 +2633,9 @@ Value *NewGVN::findLeaderForInst(Instruction *TransInst,
   TempToBlock.insert({TransInst, PredBB});
   InstrDFS.insert({TransInst, IDFSNum});
 
-  const Expression *E = performSymbolicEvaluation(TransInst, Visited);
+  auto Res = performSymbolicEvaluation(TransInst, Visited);
+  const Expression *E = Res.Expr;
+  addAdditionalUsers(Res, OrigInst);
   InstrDFS.erase(TransInst);
   AllTempInstructions.erase(TransInst);
   TempToBlock.erase(TransInst);
@@ -2758,6 +2793,14 @@ NewGVN::makePossiblePHIOfOps(Instruction *I,
         dbgs()
         << "Not creating real PHI of ops because it simplified to existing "
            "value or constant\n");
+    // We have leaders for all operands, but do not create a real PHI node with
+    // those leaders as operands, so the link between the operands and the
+    // PHI-of-ops is not materialized in the IR. If any of those leaders
+    // changes, the PHI-of-op may change also, so we need to add the operands as
+    // additional users.
+    for (auto &O : PHIOps)
+      addAdditionalUsers(O.first, I);
+
     return E;
   }
   auto *ValuePHI = RealToTemp.lookup(I);
@@ -3019,7 +3062,10 @@ void NewGVN::valueNumberInstruction(Instruction *I) {
     const Expression *Symbolized = nullptr;
     SmallPtrSet<Value *, 2> Visited;
     if (DebugCounter::shouldExecute(VNCounter)) {
-      Symbolized = performSymbolicEvaluation(I, Visited);
+      auto Res = performSymbolicEvaluation(I, Visited);
+      Symbolized = Res.Expr;
+      addAdditionalUsers(Res, I);
+
       // Make a phi of ops if necessary
       if (Symbolized && !isa<ConstantExpression>(Symbolized) &&
           !isa<VariableExpression>(Symbolized) && PHINodeUses.count(I)) {
@@ -4176,6 +4222,5 @@ PreservedAnalyses NewGVNPass::run(Function &F, AnalysisManager<Function> &AM) {
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
   PA.preserve<DominatorTreeAnalysis>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
diff --git a/llvm/lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp b/llvm/lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp
index 58763ec72ece..7872c553b412 100644
--- a/llvm/lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp
+++ b/llvm/lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp
@@ -13,8 +13,10 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Transforms/Scalar/PartiallyInlineLibCalls.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/TargetLibraryInfo.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/Support/DebugCounter.h"
@@ -30,7 +32,7 @@ DEBUG_COUNTER(PILCounter, "partially-inline-libcalls-transform",
 
 static bool optimizeSQRT(CallInst *Call, Function *CalledFunc,
                          BasicBlock &CurrBB, Function::iterator &BB,
-                         const TargetTransformInfo *TTI) {
+                         const TargetTransformInfo *TTI, DomTreeUpdater *DTU) {
   // There is no need to change the IR, since backend will emit sqrt
   // instruction if the call has already been marked read-only.
   if (Call->onlyReadsMemory())
@@ -51,33 +53,44 @@ static bool optimizeSQRT(CallInst *Call, Function *CalledFunc,
   // dst = phi(v0, v1)
   //
 
-  // Move all instructions following Call to newly created block JoinBB.
-  // Create phi and replace all uses.
-  BasicBlock *JoinBB = llvm::SplitBlock(&CurrBB, Call->getNextNode());
-  IRBuilder<> Builder(JoinBB, JoinBB->begin());
   Type *Ty = Call->getType();
+  IRBuilder<> Builder(Call->getNextNode());
+
+  // Split CurrBB right after the call, create a 'then' block (that branches
+  // back to split-off tail of CurrBB) into which we'll insert a libcall.
+  Instruction *LibCallTerm = SplitBlockAndInsertIfThen(
+      Builder.getTrue(), Call->getNextNode(), /*Unreachable=*/false,
+      /*BranchWeights*/ nullptr, DTU);
+
+  auto *CurrBBTerm = cast<BranchInst>(CurrBB.getTerminator());
+  // We want an 'else' block though, not a 'then' block.
+  cast<BranchInst>(CurrBBTerm)->swapSuccessors();
+
+  // Create phi that will merge results of either sqrt and replace all uses.
+  BasicBlock *JoinBB = LibCallTerm->getSuccessor(0);
+  JoinBB->setName(CurrBB.getName() + ".split");
+  Builder.SetInsertPoint(JoinBB, JoinBB->begin());
   PHINode *Phi = Builder.CreatePHI(Ty, 2);
   Call->replaceAllUsesWith(Phi);
 
-  // Create basic block LibCallBB and insert a call to library function sqrt.
-  BasicBlock *LibCallBB = BasicBlock::Create(CurrBB.getContext(), "call.sqrt",
-                                             CurrBB.getParent(), JoinBB);
-  Builder.SetInsertPoint(LibCallBB);
+  // Finally, insert the libcall into 'else' block.
+  BasicBlock *LibCallBB = LibCallTerm->getParent();
+  LibCallBB->setName("call.sqrt");
+  Builder.SetInsertPoint(LibCallTerm);
   Instruction *LibCall = Call->clone();
   Builder.Insert(LibCall);
-  Builder.CreateBr(JoinBB);
 
   // Add attribute "readnone" so that backend can use a native sqrt instruction
-  // for this call. Insert a FP compare instruction and a conditional branch
-  // at the end of CurrBB.
+  // for this call.
   Call->addAttribute(AttributeList::FunctionIndex, Attribute::ReadNone);
-  CurrBB.getTerminator()->eraseFromParent();
-  Builder.SetInsertPoint(&CurrBB);
+
+  // Insert a FP compare instruction and use it as the CurrBB branch condition.
+  Builder.SetInsertPoint(CurrBBTerm);
   Value *FCmp = TTI->isFCmpOrdCheaperThanFCmpZero(Ty)
                     ? Builder.CreateFCmpORD(Call, Call)
                     : Builder.CreateFCmpOGE(Call->getOperand(0),
                                             ConstantFP::get(Ty, 0.0));
-  Builder.CreateCondBr(FCmp, JoinBB, LibCallBB);
+  CurrBBTerm->setCondition(FCmp);
 
   // Add phi operands.
   Phi->addIncoming(Call, &CurrBB);
@@ -88,7 +101,12 @@ static bool optimizeSQRT(CallInst *Call, Function *CalledFunc,
 }
 
 static bool runPartiallyInlineLibCalls(Function &F, TargetLibraryInfo *TLI,
-                                       const TargetTransformInfo *TTI) {
+                                       const TargetTransformInfo *TTI,
+                                       DominatorTree *DT) {
+  Optional<DomTreeUpdater> DTU;
+  if (DT)
+    DTU.emplace(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+
   bool Changed = false;
 
   Function::iterator CurrBB;
@@ -103,7 +121,7 @@ static bool runPartiallyInlineLibCalls(Function &F, TargetLibraryInfo *TLI,
       if (!Call || !(CalledFunc = Call->getCalledFunction()))
         continue;
 
-      if (Call->isNoBuiltin())
+      if (Call->isNoBuiltin() || Call->isStrictFP())
         continue;
 
       // Skip if function either has local linkage or is not a known library
@@ -117,7 +135,8 @@ static bool runPartiallyInlineLibCalls(Function &F, TargetLibraryInfo *TLI,
       case LibFunc_sqrtf:
       case LibFunc_sqrt:
         if (TTI->haveFastSqrt(Call->getType()) &&
-            optimizeSQRT(Call, CalledFunc, *CurrBB, BB, TTI))
+            optimizeSQRT(Call, CalledFunc, *CurrBB, BB, TTI,
+                         DTU.hasValue() ? DTU.getPointer() : nullptr))
           break;
         continue;
       default:
@@ -136,9 +155,12 @@ PreservedAnalyses
 PartiallyInlineLibCallsPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
   auto &TTI = AM.getResult<TargetIRAnalysis>(F);
-  if (!runPartiallyInlineLibCalls(F, &TLI, &TTI))
+  auto *DT = AM.getCachedResult<DominatorTreeAnalysis>(F);
+  if (!runPartiallyInlineLibCalls(F, &TLI, &TTI, DT))
     return PreservedAnalyses::all();
-  return PreservedAnalyses::none();
+  PreservedAnalyses PA;
+  PA.preserve<DominatorTreeAnalysis>();
+  return PA;
 }
 
 namespace {
@@ -154,6 +176,7 @@ public:
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<TargetLibraryInfoWrapperPass>();
     AU.addRequired<TargetTransformInfoWrapperPass>();
+    AU.addPreserved<DominatorTreeWrapperPass>();
     FunctionPass::getAnalysisUsage(AU);
   }
 
@@ -165,7 +188,10 @@ public:
         &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
     const TargetTransformInfo *TTI =
         &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
-    return runPartiallyInlineLibCalls(F, TLI, TTI);
+    DominatorTree *DT = nullptr;
+    if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
+      DT = &DTWP->getDomTree();
+    return runPartiallyInlineLibCalls(F, TLI, TTI, DT);
   }
 };
 }
@@ -176,6 +202,7 @@ INITIALIZE_PASS_BEGIN(PartiallyInlineLibCallsLegacyPass,
                       "Partially inline calls to library functions", false,
                       false)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_END(PartiallyInlineLibCallsLegacyPass,
                     "partially-inline-libcalls",
diff --git a/llvm/lib/Transforms/Scalar/Reassociate.cpp b/llvm/lib/Transforms/Scalar/Reassociate.cpp
index dffeb7cc227b..888edc4d69a8 100644
--- a/llvm/lib/Transforms/Scalar/Reassociate.cpp
+++ b/llvm/lib/Transforms/Scalar/Reassociate.cpp
@@ -975,12 +975,13 @@ static bool isLoadCombineCandidate(Instruction *Or) {
 }
 
 /// Return true if it may be profitable to convert this (X|Y) into (X+Y).
-static bool ShouldConvertOrWithNoCommonBitsToAdd(Instruction *Or) {
+static bool shouldConvertOrWithNoCommonBitsToAdd(Instruction *Or) {
   // Don't bother to convert this up unless either the LHS is an associable add
   // or subtract or mul or if this is only used by one of the above.
   // This is only a compile-time improvement, it is not needed for correctness!
   auto isInteresting = [](Value *V) {
-    for (auto Op : {Instruction::Add, Instruction::Sub, Instruction::Mul})
+    for (auto Op : {Instruction::Add, Instruction::Sub, Instruction::Mul,
+                    Instruction::Shl})
       if (isReassociableOp(V, Op))
         return true;
     return false;
@@ -998,7 +999,7 @@ static bool ShouldConvertOrWithNoCommonBitsToAdd(Instruction *Or) {
 
 /// If we have (X|Y), and iff X and Y have no common bits set,
 /// transform this into (X+Y) to allow arithmetics reassociation.
-static BinaryOperator *ConvertOrWithNoCommonBitsToAdd(Instruction *Or) {
+static BinaryOperator *convertOrWithNoCommonBitsToAdd(Instruction *Or) {
   // Convert an or into an add.
   BinaryOperator *New =
       CreateAdd(Or->getOperand(0), Or->getOperand(1), "", Or, Or);
@@ -2212,11 +2213,11 @@ void ReassociatePass::OptimizeInst(Instruction *I) {
   // If this is a bitwise or instruction of operands
   // with no common bits set, convert it to X+Y.
   if (I->getOpcode() == Instruction::Or &&
-      ShouldConvertOrWithNoCommonBitsToAdd(I) && !isLoadCombineCandidate(I) &&
+      shouldConvertOrWithNoCommonBitsToAdd(I) && !isLoadCombineCandidate(I) &&
       haveNoCommonBitsSet(I->getOperand(0), I->getOperand(1),
                           I->getModule()->getDataLayout(), /*AC=*/nullptr, I,
                           /*DT=*/nullptr)) {
-    Instruction *NI = ConvertOrWithNoCommonBitsToAdd(I);
+    Instruction *NI = convertOrWithNoCommonBitsToAdd(I);
     RedoInsts.insert(I);
     MadeChange = true;
     I = NI;
@@ -2567,9 +2568,6 @@ PreservedAnalyses ReassociatePass::run(Function &F, FunctionAnalysisManager &) {
   if (MadeChange) {
     PreservedAnalyses PA;
     PA.preserveSet<CFGAnalyses>();
-    PA.preserve<AAManager>();
-    PA.preserve<BasicAA>();
-    PA.preserve<GlobalsAA>();
     return PA;
   }
 
diff --git a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
index b7830555bf73..bc0fecc972fc 100644
--- a/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
+++ b/llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
@@ -522,6 +522,14 @@ static BaseDefiningValueResult findBaseDefiningValue(Value *I) {
         ConstantPointerNull::get(cast<PointerType>(I->getType())), true);
   }
 
+  // inttoptrs in an integral address space are currently ill-defined.  We
+  // treat them as defining base pointers here for consistency with the
+  // constant rule above and because we don't really have a better semantic
+  // to give them.  Note that the optimizer is always free to insert undefined
+  // behavior on dynamically dead paths as well.
+  if (isa<IntToPtrInst>(I))
+    return BaseDefiningValueResult(I, true);
+
   if (CastInst *CI = dyn_cast<CastInst>(I)) {
     Value *Def = CI->stripPointerCasts();
     // If stripping pointer casts changes the address space there is an
@@ -562,6 +570,8 @@ static BaseDefiningValueResult findBaseDefiningValue(Value *I) {
       // implications much.
       llvm_unreachable(
           "interaction with the gcroot mechanism is not supported");
+    case Intrinsic::experimental_gc_get_pointer_base:
+      return findBaseDefiningValue(II->getOperand(0));
     }
   }
   // We assume that functions in the source language only return base
@@ -594,6 +604,11 @@ static BaseDefiningValueResult findBaseDefiningValue(Value *I) {
   assert(!isa<InsertValueInst>(I) &&
          "Base pointer for a struct is meaningless");
 
+  // This value might have been generated by findBasePointer() called when
+  // substituting gc.get.pointer.base() intrinsic.
+  bool IsKnownBase =
+      isa<Instruction>(I) && cast<Instruction>(I)->getMetadata("is_base_value");
+
   // An extractelement produces a base result exactly when it's input does.
   // We may need to insert a parallel instruction to extract the appropriate
   // element out of the base vector corresponding to the input. Given this,
@@ -602,7 +617,7 @@ static BaseDefiningValueResult findBaseDefiningValue(Value *I) {
     // Note: There a lot of obvious peephole cases here.  This are deliberately
     // handled after the main base pointer inference algorithm to make writing
     // test cases to exercise that code easier.
-    return BaseDefiningValueResult(I, false);
+    return BaseDefiningValueResult(I, IsKnownBase);
 
   // The last two cases here don't return a base pointer.  Instead, they
   // return a value which dynamically selects from among several base
@@ -610,7 +625,7 @@ static BaseDefiningValueResult findBaseDefiningValue(Value *I) {
   // the caller to resolve these.
   assert((isa<SelectInst>(I) || isa<PHINode>(I)) &&
          "missing instruction case in findBaseDefiningValing");
-  return BaseDefiningValueResult(I, false);
+  return BaseDefiningValueResult(I, IsKnownBase);
 }
 
 /// Returns the base defining value for this value.
@@ -676,26 +691,72 @@ namespace {
 /// the base of this BDV.
 class BDVState {
 public:
-  enum Status { Unknown, Base, Conflict };
+  enum StatusTy {
+     // Starting state of lattice
+     Unknown,
+     // Some specific base value -- does *not* mean that instruction
+     // propagates the base of the object
+     // ex: gep %arg, 16 -> %arg is the base value
+     Base,
+     // Need to insert a node to represent a merge.
+     Conflict
+  };
 
-  BDVState() : BaseValue(nullptr) {}
+  BDVState() {
+    llvm_unreachable("missing state in map");
+  }
 
-  explicit BDVState(Status Status, Value *BaseValue = nullptr)
-      : Status(Status), BaseValue(BaseValue) {
+  explicit BDVState(Value *OriginalValue)
+    : OriginalValue(OriginalValue) {}
+  explicit BDVState(Value *OriginalValue, StatusTy Status, Value *BaseValue = nullptr)
+    : OriginalValue(OriginalValue), Status(Status), BaseValue(BaseValue) {
     assert(Status != Base || BaseValue);
   }
 
-  explicit BDVState(Value *BaseValue) : Status(Base), BaseValue(BaseValue) {}
-
-  Status getStatus() const { return Status; }
+  StatusTy getStatus() const { return Status; }
+  Value *getOriginalValue() const { return OriginalValue; }
   Value *getBaseValue() const { return BaseValue; }
 
   bool isBase() const { return getStatus() == Base; }
   bool isUnknown() const { return getStatus() == Unknown; }
   bool isConflict() const { return getStatus() == Conflict; }
 
+  // Values of type BDVState form a lattice, and this function implements the
+  // meet
+  // operation.
+  void meet(const BDVState &Other) {
+    auto markConflict = [&]() {
+      Status = BDVState::Conflict;
+      BaseValue = nullptr;
+    };
+    // Conflict is a final state.
+    if (isConflict())
+      return;
+    // if we are not known - just take other state.
+    if (isUnknown()) {
+      Status = Other.getStatus();
+      BaseValue = Other.getBaseValue();
+      return;
+    }
+    // We are base.
+    assert(isBase() && "Unknown state");
+    // If other is unknown - just keep our state.
+    if (Other.isUnknown())
+      return;
+    // If other is conflict - it is a final state.
+    if (Other.isConflict())
+      return markConflict();
+    // Other is base as well.
+    assert(Other.isBase() && "Unknown state");
+    // If bases are different - Conflict.
+    if (getBaseValue() != Other.getBaseValue())
+      return markConflict();
+    // We are identical, do nothing.
+  }
+
   bool operator==(const BDVState &Other) const {
-    return BaseValue == Other.BaseValue && Status == Other.Status;
+    return OriginalValue == OriginalValue && BaseValue == Other.BaseValue &&
+      Status == Other.Status;
   }
 
   bool operator!=(const BDVState &other) const { return !(*this == other); }
@@ -718,13 +779,15 @@ public:
       OS << "C";
       break;
     }
-    OS << " (" << getBaseValue() << " - "
-       << (getBaseValue() ? getBaseValue()->getName() : "nullptr") << "): ";
+    OS << " (base " << getBaseValue() << " - "
+       << (getBaseValue() ? getBaseValue()->getName() : "nullptr") << ")"
+       << " for  "  << OriginalValue->getName() << ":";
   }
 
 private:
-  Status Status = Unknown;
-  AssertingVH<Value> BaseValue; // Non-null only if Status == Base.
+  AssertingVH<Value> OriginalValue; // instruction this state corresponds to
+  StatusTy Status = Unknown;
+  AssertingVH<Value> BaseValue = nullptr; // Non-null only if Status == Base.
 };
 
 } // end anonymous namespace
@@ -736,41 +799,6 @@ static raw_ostream &operator<<(raw_ostream &OS, const BDVState &State) {
 }
 #endif
 
-static BDVState meetBDVStateImpl(const BDVState &LHS, const BDVState &RHS) {
-  switch (LHS.getStatus()) {
-  case BDVState::Unknown:
-    return RHS;
-
-  case BDVState::Base:
-    assert(LHS.getBaseValue() && "can't be null");
-    if (RHS.isUnknown())
-      return LHS;
-
-    if (RHS.isBase()) {
-      if (LHS.getBaseValue() == RHS.getBaseValue()) {
-        assert(LHS == RHS && "equality broken!");
-        return LHS;
-      }
-      return BDVState(BDVState::Conflict);
-    }
-    assert(RHS.isConflict() && "only three states!");
-    return BDVState(BDVState::Conflict);
-
-  case BDVState::Conflict:
-    return LHS;
-  }
-  llvm_unreachable("only three states!");
-}
-
-// Values of type BDVState form a lattice, and this function implements the meet
-// operation.
-static BDVState meetBDVState(const BDVState &LHS, const BDVState &RHS) {
-  BDVState Result = meetBDVStateImpl(LHS, RHS);
-  assert(Result == meetBDVStateImpl(RHS, LHS) &&
-         "Math is wrong: meet does not commute!");
-  return Result;
-}
-
 /// For a given value or instruction, figure out what base ptr its derived from.
 /// For gc objects, this is simply itself.  On success, returns a value which is
 /// the base pointer.  (This is reliable and can be used for relocation.)  On
@@ -818,12 +846,44 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
   // below.  This is important for deterministic compilation.
   MapVector<Value *, BDVState> States;
 
+#ifndef NDEBUG
+  auto VerifyStates = [&]() {
+    for (auto &Entry : States) {
+      assert(Entry.first == Entry.second.getOriginalValue());
+    }
+  };
+#endif
+
+  auto visitBDVOperands = [](Value *BDV, std::function<void (Value*)> F) {
+    if (PHINode *PN = dyn_cast<PHINode>(BDV)) {
+      for (Value *InVal : PN->incoming_values())
+        F(InVal);
+    } else if (SelectInst *SI = dyn_cast<SelectInst>(BDV)) {
+      F(SI->getTrueValue());
+      F(SI->getFalseValue());
+    } else if (auto *EE = dyn_cast<ExtractElementInst>(BDV)) {
+      F(EE->getVectorOperand());
+    } else if (auto *IE = dyn_cast<InsertElementInst>(BDV)) {
+      F(IE->getOperand(0));
+      F(IE->getOperand(1));
+    } else if (auto *SV = dyn_cast<ShuffleVectorInst>(BDV)) {
+      // For a canonical broadcast, ignore the undef argument
+      // (without this, we insert a parallel base shuffle for every broadcast)
+      F(SV->getOperand(0));
+      if (!SV->isZeroEltSplat())
+        F(SV->getOperand(1));
+    } else {
+      llvm_unreachable("unexpected BDV type");
+    }
+  };
+
+
   // Recursively fill in all base defining values reachable from the initial
   // one for which we don't already know a definite base value for
   /* scope */ {
     SmallVector<Value*, 16> Worklist;
     Worklist.push_back(Def);
-    States.insert({Def, BDVState()});
+    States.insert({Def, BDVState(Def)});
     while (!Worklist.empty()) {
       Value *Current = Worklist.pop_back_val();
       assert(!isOriginalBaseResult(Current) && "why did it get added?");
@@ -839,45 +899,67 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
           return;
         assert(isExpectedBDVType(Base) && "the only non-base values "
                "we see should be base defining values");
-        if (States.insert(std::make_pair(Base, BDVState())).second)
+        if (States.insert(std::make_pair(Base, BDVState(Base))).second)
           Worklist.push_back(Base);
       };
-      if (PHINode *PN = dyn_cast<PHINode>(Current)) {
-        for (Value *InVal : PN->incoming_values())
-          visitIncomingValue(InVal);
-      } else if (SelectInst *SI = dyn_cast<SelectInst>(Current)) {
-        visitIncomingValue(SI->getTrueValue());
-        visitIncomingValue(SI->getFalseValue());
-      } else if (auto *EE = dyn_cast<ExtractElementInst>(Current)) {
-        visitIncomingValue(EE->getVectorOperand());
-      } else if (auto *IE = dyn_cast<InsertElementInst>(Current)) {
-        visitIncomingValue(IE->getOperand(0)); // vector operand
-        visitIncomingValue(IE->getOperand(1)); // scalar operand
-      } else if (auto *SV = dyn_cast<ShuffleVectorInst>(Current)) {
-        visitIncomingValue(SV->getOperand(0));
-        visitIncomingValue(SV->getOperand(1));
-      }
-      else {
-        llvm_unreachable("Unimplemented instruction case");
-      }
+
+      visitBDVOperands(Current, visitIncomingValue);
     }
   }
 
 #ifndef NDEBUG
+  VerifyStates();
   LLVM_DEBUG(dbgs() << "States after initialization:\n");
   for (auto Pair : States) {
     LLVM_DEBUG(dbgs() << " " << Pair.second << " for " << *Pair.first << "\n");
   }
 #endif
 
+  // Iterate forward through the value graph pruning any node from the state
+  // list where all of the inputs are base pointers.  The purpose of this is to
+  // reuse existing values when the derived pointer we were asked to materialize
+  // a base pointer for happens to be a base pointer itself.  (Or a sub-graph
+  // feeding it does.)
+  SmallVector<Value *> ToRemove;
+  do {
+    ToRemove.clear();
+    for (auto Pair : States) {
+      Value *BDV = Pair.first;
+      auto canPruneInput = [&](Value *V) {
+        Value *BDV = findBaseOrBDV(V, Cache);
+        if (V->stripPointerCasts() != BDV)
+          return false;
+        // The assumption is that anything not in the state list is
+        // propagates a base pointer.
+        return States.count(BDV) == 0;
+      };
+
+      bool CanPrune = true;
+      visitBDVOperands(BDV, [&](Value *Op) {
+        CanPrune = CanPrune && canPruneInput(Op);
+      });
+      if (CanPrune)
+        ToRemove.push_back(BDV);
+    }
+    for (Value *V : ToRemove) {
+      States.erase(V);
+      // Cache the fact V is it's own base for later usage.
+      Cache[V] = V;
+    }
+  } while (!ToRemove.empty());
+
+  // Did we manage to prove that Def itself must be a base pointer?
+  if (!States.count(Def))
+    return Def;
+
   // Return a phi state for a base defining value.  We'll generate a new
   // base state for known bases and expect to find a cached state otherwise.
   auto GetStateForBDV = [&](Value *BaseValue, Value *Input) {
-    if (isKnownBaseResult(BaseValue) && areBothVectorOrScalar(BaseValue, Input))
-      return BDVState(BaseValue);
     auto I = States.find(BaseValue);
-    assert(I != States.end() && "lookup failed!");
-    return I->second;
+    if (I != States.end())
+      return I->second;
+    assert(areBothVectorOrScalar(BaseValue, Input));
+    return BDVState(BaseValue, BDVState::Base, BaseValue);
   };
 
   bool Progress = true;
@@ -899,38 +981,12 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
              !areBothVectorOrScalar(BDV, Pair.second.getBaseValue())) &&
                  "why did it get added?");
 
-      // Given an input value for the current instruction, return a BDVState
-      // instance which represents the BDV of that value.
-      auto getStateForInput = [&](Value *V) mutable {
-        Value *BDV = findBaseOrBDV(V, Cache);
-        return GetStateForBDV(BDV, V);
-      };
-
-      BDVState NewState;
-      if (SelectInst *SI = dyn_cast<SelectInst>(BDV)) {
-        NewState = meetBDVState(NewState, getStateForInput(SI->getTrueValue()));
-        NewState =
-            meetBDVState(NewState, getStateForInput(SI->getFalseValue()));
-      } else if (PHINode *PN = dyn_cast<PHINode>(BDV)) {
-        for (Value *Val : PN->incoming_values())
-          NewState = meetBDVState(NewState, getStateForInput(Val));
-      } else if (auto *EE = dyn_cast<ExtractElementInst>(BDV)) {
-        // The 'meet' for an extractelement is slightly trivial, but it's still
-        // useful in that it drives us to conflict if our input is.
-        NewState =
-            meetBDVState(NewState, getStateForInput(EE->getVectorOperand()));
-      } else if (auto *IE = dyn_cast<InsertElementInst>(BDV)){
-        // Given there's a inherent type mismatch between the operands, will
-        // *always* produce Conflict.
-        NewState = meetBDVState(NewState, getStateForInput(IE->getOperand(0)));
-        NewState = meetBDVState(NewState, getStateForInput(IE->getOperand(1)));
-      } else {
-        // The only instance this does not return a Conflict is when both the
-        // vector operands are the same vector.
-        auto *SV = cast<ShuffleVectorInst>(BDV);
-        NewState = meetBDVState(NewState, getStateForInput(SV->getOperand(0)));
-        NewState = meetBDVState(NewState, getStateForInput(SV->getOperand(1)));
-      }
+      BDVState NewState(BDV);
+      visitBDVOperands(BDV, [&](Value *Op) {
+        Value *BDV = findBaseOrBDV(Op, Cache);
+        auto OpState = GetStateForBDV(BDV, Op);
+        NewState.meet(OpState);
+      });
 
       BDVState OldState = States[BDV];
       if (OldState != NewState) {
@@ -944,6 +1000,7 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
   }
 
 #ifndef NDEBUG
+  VerifyStates();
   LLVM_DEBUG(dbgs() << "States after meet iteration:\n");
   for (auto Pair : States) {
     LLVM_DEBUG(dbgs() << " " << Pair.second << " for " << *Pair.first << "\n");
@@ -977,17 +1034,21 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
       auto *BaseInst = ExtractElementInst::Create(
           State.getBaseValue(), EE->getIndexOperand(), "base_ee", EE);
       BaseInst->setMetadata("is_base_value", MDNode::get(I->getContext(), {}));
-      States[I] = BDVState(BDVState::Base, BaseInst);
+      States[I] = BDVState(I, BDVState::Base, BaseInst);
     } else if (!isa<VectorType>(I->getType())) {
       // We need to handle cases that have a vector base but the instruction is
       // a scalar type (these could be phis or selects or any instruction that
       // are of scalar type, but the base can be a vector type).  We
       // conservatively set this as conflict.  Setting the base value for these
       // conflicts is handled in the next loop which traverses States.
-      States[I] = BDVState(BDVState::Conflict);
+      States[I] = BDVState(I, BDVState::Conflict);
     }
   }
 
+#ifndef NDEBUG
+  VerifyStates();
+#endif
+
   // Insert Phis for all conflicts
   // TODO: adjust naming patterns to avoid this order of iteration dependency
   for (auto Pair : States) {
@@ -1008,45 +1069,32 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
     if (!State.isConflict())
       continue;
 
-    /// Create and insert a new instruction which will represent the base of
-    /// the given instruction 'I'.
-    auto MakeBaseInstPlaceholder = [](Instruction *I) -> Instruction* {
+    auto getMangledName = [](Instruction *I) -> std::string {
       if (isa<PHINode>(I)) {
-        BasicBlock *BB = I->getParent();
-        int NumPreds = pred_size(BB);
-        assert(NumPreds > 0 && "how did we reach here");
-        std::string Name = suffixed_name_or(I, ".base", "base_phi");
-        return PHINode::Create(I->getType(), NumPreds, Name, I);
-      } else if (SelectInst *SI = dyn_cast<SelectInst>(I)) {
-        // The undef will be replaced later
-        UndefValue *Undef = UndefValue::get(SI->getType());
-        std::string Name = suffixed_name_or(I, ".base", "base_select");
-        return SelectInst::Create(SI->getCondition(), Undef, Undef, Name, SI);
-      } else if (auto *EE = dyn_cast<ExtractElementInst>(I)) {
-        UndefValue *Undef = UndefValue::get(EE->getVectorOperand()->getType());
-        std::string Name = suffixed_name_or(I, ".base", "base_ee");
-        return ExtractElementInst::Create(Undef, EE->getIndexOperand(), Name,
-                                          EE);
-      } else if (auto *IE = dyn_cast<InsertElementInst>(I)) {
-        UndefValue *VecUndef = UndefValue::get(IE->getOperand(0)->getType());
-        UndefValue *ScalarUndef = UndefValue::get(IE->getOperand(1)->getType());
-        std::string Name = suffixed_name_or(I, ".base", "base_ie");
-        return InsertElementInst::Create(VecUndef, ScalarUndef,
-                                         IE->getOperand(2), Name, IE);
+        return suffixed_name_or(I, ".base", "base_phi");
+      } else if (isa<SelectInst>(I)) {
+        return suffixed_name_or(I, ".base", "base_select");
+      } else if (isa<ExtractElementInst>(I)) {
+        return suffixed_name_or(I, ".base", "base_ee");
+      } else if (isa<InsertElementInst>(I)) {
+        return suffixed_name_or(I, ".base", "base_ie");
       } else {
-        auto *SV = cast<ShuffleVectorInst>(I);
-        UndefValue *VecUndef = UndefValue::get(SV->getOperand(0)->getType());
-        std::string Name = suffixed_name_or(I, ".base", "base_sv");
-        return new ShuffleVectorInst(VecUndef, VecUndef, SV->getShuffleMask(),
-                                     Name, SV);
+        return suffixed_name_or(I, ".base", "base_sv");
       }
     };
-    Instruction *BaseInst = MakeBaseInstPlaceholder(I);
+
+    Instruction *BaseInst = I->clone();
+    BaseInst->insertBefore(I);
+    BaseInst->setName(getMangledName(I));
     // Add metadata marking this as a base value
     BaseInst->setMetadata("is_base_value", MDNode::get(I->getContext(), {}));
-    States[I] = BDVState(BDVState::Conflict, BaseInst);
+    States[I] = BDVState(I, BDVState::Conflict, BaseInst);
   }
 
+#ifndef NDEBUG
+  VerifyStates();
+#endif
+
   // Returns a instruction which produces the base pointer for a given
   // instruction.  The instruction is assumed to be an input to one of the BDVs
   // seen in the inference algorithm above.  As such, we must either already
@@ -1058,7 +1106,8 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
   auto getBaseForInput = [&](Value *Input, Instruction *InsertPt) {
     Value *BDV = findBaseOrBDV(Input, Cache);
     Value *Base = nullptr;
-    if (isKnownBaseResult(BDV) && areBothVectorOrScalar(BDV, Input)) {
+    if (!States.count(BDV)) {
+      assert(areBothVectorOrScalar(BDV, Input));
       Base = BDV;
     } else {
       // Either conflict or base.
@@ -1091,26 +1140,21 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
 
     if (PHINode *BasePHI = dyn_cast<PHINode>(State.getBaseValue())) {
       PHINode *PN = cast<PHINode>(BDV);
-      unsigned NumPHIValues = PN->getNumIncomingValues();
+      const unsigned NumPHIValues = PN->getNumIncomingValues();
+
+      // The IR verifier requires phi nodes with multiple entries from the
+      // same basic block to have the same incoming value for each of those
+      // entries.  Since we're inserting bitcasts in the loop, make sure we
+      // do so at least once per incoming block.
+      DenseMap<BasicBlock *, Value*> BlockToValue;
       for (unsigned i = 0; i < NumPHIValues; i++) {
         Value *InVal = PN->getIncomingValue(i);
         BasicBlock *InBB = PN->getIncomingBlock(i);
-
-        // If we've already seen InBB, add the same incoming value
-        // we added for it earlier.  The IR verifier requires phi
-        // nodes with multiple entries from the same basic block
-        // to have the same incoming value for each of those
-        // entries.  If we don't do this check here and basephi
-        // has a different type than base, we'll end up adding two
-        // bitcasts (and hence two distinct values) as incoming
-        // values for the same basic block.
-
-        int BlockIndex = BasePHI->getBasicBlockIndex(InBB);
-        if (BlockIndex != -1) {
-          Value *OldBase = BasePHI->getIncomingValue(BlockIndex);
-          BasePHI->addIncoming(OldBase, InBB);
-
+        if (!BlockToValue.count(InBB))
+          BlockToValue[InBB] = getBaseForInput(InVal, InBB->getTerminator());
+        else {
 #ifndef NDEBUG
+          Value *OldBase = BlockToValue[InBB];
           Value *Base = getBaseForInput(InVal, nullptr);
           // In essence this assert states: the only way two values
           // incoming from the same basic block may be different is by
@@ -1121,16 +1165,10 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
           assert(Base->stripPointerCasts() == OldBase->stripPointerCasts() &&
                  "Sanity -- findBaseOrBDV should be pure!");
 #endif
-          continue;
         }
-
-        // Find the instruction which produces the base for each input.  We may
-        // need to insert a bitcast in the incoming block.
-        // TODO: Need to split critical edges if insertion is needed
-        Value *Base = getBaseForInput(InVal, InBB->getTerminator());
-        BasePHI->addIncoming(Base, InBB);
+        Value *Base = BlockToValue[InBB];
+        BasePHI->setIncomingValue(i, Base);
       }
-      assert(BasePHI->getNumIncomingValues() == NumPHIValues);
     } else if (SelectInst *BaseSI =
                    dyn_cast<SelectInst>(State.getBaseValue())) {
       SelectInst *SI = cast<SelectInst>(BDV);
@@ -1163,10 +1201,20 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
         BaseSV->setOperand(OperandIdx, Base);
       };
       UpdateOperand(0); // vector operand
-      UpdateOperand(1); // vector operand
+      if (!BdvSV->isZeroEltSplat())
+        UpdateOperand(1); // vector operand
+      else {
+        // Never read, so just use undef
+        Value *InVal = BdvSV->getOperand(1);
+        BaseSV->setOperand(1, UndefValue::get(InVal->getType()));
+      }
     }
   }
 
+#ifndef NDEBUG
+  VerifyStates();
+#endif
+
   // Cache all of our results so we can cheaply reuse them
   // NOTE: This is actually two caches: one of the base defining value
   // relation and one of the base pointer relation!  FIXME
@@ -1186,14 +1234,6 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
                << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none")
                << " to: " << Base->getName() << "\n");
 
-    if (Cache.count(BDV)) {
-      assert(isKnownBaseResult(Base) &&
-             "must be something we 'know' is a base pointer");
-      // Once we transition from the BDV relation being store in the Cache to
-      // the base relation being stored, it must be stable
-      assert((!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) &&
-             "base relation should be stable");
-    }
     Cache[BDV] = Base;
   }
   assert(Cache.count(Def));
@@ -1236,7 +1276,20 @@ static void findBasePointers(DominatorTree &DT, DefiningValueMapTy &DVCache,
                              CallBase *Call,
                              PartiallyConstructedSafepointRecord &result) {
   MapVector<Value *, Value *> PointerToBase;
-  findBasePointers(result.LiveSet, PointerToBase, &DT, DVCache);
+  StatepointLiveSetTy PotentiallyDerivedPointers = result.LiveSet;
+  // We assume that all pointers passed to deopt are base pointers; as an
+  // optimization, we can use this to avoid seperately materializing the base
+  // pointer graph.  This is only relevant since we're very conservative about
+  // generating new conflict nodes during base pointer insertion.  If we were
+  // smarter there, this would be irrelevant.
+  if (auto Opt = Call->getOperandBundle(LLVMContext::OB_deopt))
+    for (Value *V : Opt->Inputs) {
+      if (!PotentiallyDerivedPointers.count(V))
+        continue;
+      PotentiallyDerivedPointers.remove(V);
+      PointerToBase[V] = V;
+    }
+  findBasePointers(PotentiallyDerivedPointers, PointerToBase, &DT, DVCache);
 
   if (PrintBasePointers) {
     errs() << "Base Pairs (w/o Relocation):\n";
@@ -1295,6 +1348,27 @@ normalizeForInvokeSafepoint(BasicBlock *BB, BasicBlock *InvokeParent,
   return Ret;
 }
 
+// List of all function attributes which must be stripped when lowering from
+// abstract machine model to physical machine model.  Essentially, these are
+// all the effects a safepoint might have which we ignored in the abstract
+// machine model for purposes of optimization.  We have to strip these on
+// both function declarations and call sites.
+static constexpr Attribute::AttrKind FnAttrsToStrip[] =
+  {Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly,
+   Attribute::ArgMemOnly, Attribute::InaccessibleMemOnly,
+   Attribute::InaccessibleMemOrArgMemOnly,
+   Attribute::NoSync, Attribute::NoFree};
+
+// List of all parameter and return attributes which must be stripped when
+// lowering from the abstract machine model.  Note that we list attributes
+// here which aren't valid as return attributes, that is okay.  There are
+// also some additional attributes with arguments which are handled
+// explicitly and are not in this list.
+static constexpr Attribute::AttrKind ParamAttrsToStrip[] =
+  {Attribute::ReadNone, Attribute::ReadOnly, Attribute::WriteOnly,
+   Attribute::NoAlias, Attribute::NoFree};
+
+
 // Create new attribute set containing only attributes which can be transferred
 // from original call to the safepoint.
 static AttributeList legalizeCallAttributes(LLVMContext &Ctx,
@@ -1304,8 +1378,9 @@ static AttributeList legalizeCallAttributes(LLVMContext &Ctx,
 
   // Remove the readonly, readnone, and statepoint function attributes.
   AttrBuilder FnAttrs = AL.getFnAttributes();
-  FnAttrs.removeAttribute(Attribute::ReadNone);
-  FnAttrs.removeAttribute(Attribute::ReadOnly);
+  for (auto Attr : FnAttrsToStrip)
+    FnAttrs.removeAttribute(Attr);
+
   for (Attribute A : AL.getFnAttributes()) {
     if (isStatepointDirectiveAttr(A))
       FnAttrs.remove(A);
@@ -2324,9 +2399,60 @@ static void rematerializeLiveValues(CallBase *Call,
   }
 }
 
+static bool inlineGetBaseAndOffset(Function &F,
+                                   SmallVectorImpl<CallInst *> &Intrinsics,
+                                   DefiningValueMapTy &DVCache) {
+  auto &Context = F.getContext();
+  auto &DL = F.getParent()->getDataLayout();
+  bool Changed = false;
+
+  for (auto *Callsite : Intrinsics)
+    switch (Callsite->getIntrinsicID()) {
+    case Intrinsic::experimental_gc_get_pointer_base: {
+      Changed = true;
+      Value *Base = findBasePointer(Callsite->getOperand(0), DVCache);
+      assert(!DVCache.count(Callsite));
+      auto *BaseBC = IRBuilder<>(Callsite).CreateBitCast(
+          Base, Callsite->getType(), suffixed_name_or(Base, ".cast", ""));
+      if (BaseBC != Base)
+        DVCache[BaseBC] = Base;
+      Callsite->replaceAllUsesWith(BaseBC);
+      if (!BaseBC->hasName())
+        BaseBC->takeName(Callsite);
+      Callsite->eraseFromParent();
+      break;
+    }
+    case Intrinsic::experimental_gc_get_pointer_offset: {
+      Changed = true;
+      Value *Derived = Callsite->getOperand(0);
+      Value *Base = findBasePointer(Derived, DVCache);
+      assert(!DVCache.count(Callsite));
+      unsigned AddressSpace = Derived->getType()->getPointerAddressSpace();
+      unsigned IntPtrSize = DL.getPointerSizeInBits(AddressSpace);
+      IRBuilder<> Builder(Callsite);
+      Value *BaseInt =
+          Builder.CreatePtrToInt(Base, Type::getIntNTy(Context, IntPtrSize),
+                                 suffixed_name_or(Base, ".int", ""));
+      Value *DerivedInt =
+          Builder.CreatePtrToInt(Derived, Type::getIntNTy(Context, IntPtrSize),
+                                 suffixed_name_or(Derived, ".int", ""));
+      Value *Offset = Builder.CreateSub(DerivedInt, BaseInt);
+      Callsite->replaceAllUsesWith(Offset);
+      Offset->takeName(Callsite);
+      Callsite->eraseFromParent();
+      break;
+    }
+    default:
+      llvm_unreachable("Unknown intrinsic");
+    }
+
+  return Changed;
+}
+
 static bool insertParsePoints(Function &F, DominatorTree &DT,
                               TargetTransformInfo &TTI,
-                              SmallVectorImpl<CallBase *> &ToUpdate) {
+                              SmallVectorImpl<CallBase *> &ToUpdate,
+                              DefiningValueMapTy &DVCache) {
 #ifndef NDEBUG
   // sanity check the input
   std::set<CallBase *> Uniqued;
@@ -2377,17 +2503,10 @@ static bool insertParsePoints(Function &F, DominatorTree &DT,
   findLiveReferences(F, DT, ToUpdate, Records);
 
   // B) Find the base pointers for each live pointer
-  /* scope for caching */ {
-    // Cache the 'defining value' relation used in the computation and
-    // insertion of base phis and selects.  This ensures that we don't insert
-    // large numbers of duplicate base_phis.
-    DefiningValueMapTy DVCache;
-
-    for (size_t i = 0; i < Records.size(); i++) {
-      PartiallyConstructedSafepointRecord &info = Records[i];
-      findBasePointers(DT, DVCache, ToUpdate[i], info);
-    }
-  } // end of cache scope
+  for (size_t i = 0; i < Records.size(); i++) {
+    PartiallyConstructedSafepointRecord &info = Records[i];
+    findBasePointers(DT, DVCache, ToUpdate[i], info);
+  }
 
   // The base phi insertion logic (for any safepoint) may have inserted new
   // instructions which are now live at some safepoint.  The simplest such
@@ -2543,8 +2662,9 @@ static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH,
   if (AH.getDereferenceableOrNullBytes(Index))
     R.addAttribute(Attribute::get(Ctx, Attribute::DereferenceableOrNull,
                                   AH.getDereferenceableOrNullBytes(Index)));
-  if (AH.getAttributes().hasAttribute(Index, Attribute::NoAlias))
-    R.addAttribute(Attribute::NoAlias);
+  for (auto Attr : ParamAttrsToStrip)
+    if (AH.getAttributes().hasAttribute(Index, Attr))
+      R.addAttribute(Attr);
 
   if (!R.empty())
     AH.setAttributes(AH.getAttributes().removeAttributes(Ctx, Index, R));
@@ -2553,6 +2673,16 @@ static void RemoveNonValidAttrAtIndex(LLVMContext &Ctx, AttrHolder &AH,
 static void stripNonValidAttributesFromPrototype(Function &F) {
   LLVMContext &Ctx = F.getContext();
 
+  // Intrinsics are very delicate.  Lowering sometimes depends the presence
+  // of certain attributes for correctness, but we may have also inferred
+  // additional ones in the abstract machine model which need stripped.  This
+  // assumes that the attributes defined in Intrinsic.td are conservatively
+  // correct for both physical and abstract model.
+  if (Intrinsic::ID id = F.getIntrinsicID()) {
+    F.setAttributes(Intrinsic::getAttributes(Ctx, id));
+    return;
+  }
+
   for (Argument &A : F.args())
     if (isa<PointerType>(A.getType()))
       RemoveNonValidAttrAtIndex(Ctx, F,
@@ -2560,6 +2690,9 @@ static void stripNonValidAttributesFromPrototype(Function &F) {
 
   if (isa<PointerType>(F.getReturnType()))
     RemoveNonValidAttrAtIndex(Ctx, F, AttributeList::ReturnIndex);
+
+  for (auto Attr : FnAttrsToStrip)
+    F.removeFnAttr(Attr);
 }
 
 /// Certain metadata on instructions are invalid after running RS4GC.
@@ -2711,6 +2844,7 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
   // consider those in reachable code since we need to ask dominance queries
   // when rewriting.  We'll delete the unreachable ones in a moment.
   SmallVector<CallBase *, 64> ParsePointNeeded;
+  SmallVector<CallInst *, 64> Intrinsics;
   for (Instruction &I : instructions(F)) {
     // TODO: only the ones with the flag set!
     if (NeedsRewrite(I)) {
@@ -2722,10 +2856,14 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
             "no unreachable blocks expected");
       ParsePointNeeded.push_back(cast<CallBase>(&I));
     }
+    if (auto *CI = dyn_cast<CallInst>(&I))
+      if (CI->getIntrinsicID() == Intrinsic::experimental_gc_get_pointer_base ||
+          CI->getIntrinsicID() == Intrinsic::experimental_gc_get_pointer_offset)
+        Intrinsics.emplace_back(CI);
   }
 
   // Return early if no work to do.
-  if (ParsePointNeeded.empty())
+  if (ParsePointNeeded.empty() && Intrinsics.empty())
     return MadeChange;
 
   // As a prepass, go ahead and aggressively destroy single entry phi nodes.
@@ -2794,7 +2932,20 @@ bool RewriteStatepointsForGC::runOnFunction(Function &F, DominatorTree &DT,
     }
   }
 
-  MadeChange |= insertParsePoints(F, DT, TTI, ParsePointNeeded);
+  // Cache the 'defining value' relation used in the computation and
+  // insertion of base phis and selects.  This ensures that we don't insert
+  // large numbers of duplicate base_phis. Use one cache for both
+  // inlineGetBaseAndOffset() and insertParsePoints().
+  DefiningValueMapTy DVCache;
+
+  if (!Intrinsics.empty())
+    // Inline @gc.get.pointer.base() and @gc.get.pointer.offset() before finding
+    // live references.
+    MadeChange |= inlineGetBaseAndOffset(F, Intrinsics, DVCache);
+
+  if (!ParsePointNeeded.empty())
+    MadeChange |= insertParsePoints(F, DT, TTI, ParsePointNeeded, DVCache);
+
   return MadeChange;
 }
 
@@ -2986,11 +3137,7 @@ static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
   // We may have base pointers which are now live that weren't before.  We need
   // to update the PointerToBase structure to reflect this.
   for (auto V : Updated)
-    if (Info.PointerToBase.insert({V, V}).second) {
-      assert(isKnownBaseResult(V) &&
-             "Can't find base for unexpected live value!");
-      continue;
-    }
+    Info.PointerToBase.insert({V, V});
 
 #ifndef NDEBUG
   for (auto V : Updated)
diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp
index de6be52adf21..b09f896d0157 100644
--- a/llvm/lib/Transforms/Scalar/SCCP.cpp
+++ b/llvm/lib/Transforms/Scalar/SCCP.cpp
@@ -80,22 +80,11 @@ STATISTIC(
     IPNumInstReplaced,
     "Number of instructions replaced with (simpler) instruction by IPSCCP");
 
-// The maximum number of range extensions allowed for operations requiring
-// widening.
-static const unsigned MaxNumRangeExtensions = 10;
-
-/// Returns MergeOptions with MaxWidenSteps set to MaxNumRangeExtensions.
-static ValueLatticeElement::MergeOptions getMaxWidenStepsOpts() {
-  return ValueLatticeElement::MergeOptions().setMaxWidenSteps(
-      MaxNumRangeExtensions);
-}
-namespace {
-
 // Helper to check if \p LV is either a constant or a constant
 // range with a single element. This should cover exactly the same cases as the
 // old ValueLatticeElement::isConstant() and is intended to be used in the
 // transition to ValueLatticeElement.
-bool isConstant(const ValueLatticeElement &LV) {
+static bool isConstant(const ValueLatticeElement &LV) {
   return LV.isConstant() ||
          (LV.isConstantRange() && LV.getConstantRange().isSingleElement());
 }
@@ -104,1525 +93,10 @@ bool isConstant(const ValueLatticeElement &LV) {
 // than a single element. This should cover exactly the same cases as the old
 // ValueLatticeElement::isOverdefined() and is intended to be used in the
 // transition to ValueLatticeElement.
-bool isOverdefined(const ValueLatticeElement &LV) {
+static bool isOverdefined(const ValueLatticeElement &LV) {
   return !LV.isUnknownOrUndef() && !isConstant(LV);
 }
 
-//===----------------------------------------------------------------------===//
-//
-/// SCCPSolver - This class is a general purpose solver for Sparse Conditional
-/// Constant Propagation.
-///
-class SCCPSolver : public InstVisitor<SCCPSolver> {
-  const DataLayout &DL;
-  std::function<const TargetLibraryInfo &(Function &)> GetTLI;
-  SmallPtrSet<BasicBlock *, 8> BBExecutable; // The BBs that are executable.
-  DenseMap<Value *, ValueLatticeElement>
-      ValueState; // The state each value is in.
-
-  /// StructValueState - This maintains ValueState for values that have
-  /// StructType, for example for formal arguments, calls, insertelement, etc.
-  DenseMap<std::pair<Value *, unsigned>, ValueLatticeElement> StructValueState;
-
-  /// GlobalValue - If we are tracking any values for the contents of a global
-  /// variable, we keep a mapping from the constant accessor to the element of
-  /// the global, to the currently known value.  If the value becomes
-  /// overdefined, it's entry is simply removed from this map.
-  DenseMap<GlobalVariable *, ValueLatticeElement> TrackedGlobals;
-
-  /// TrackedRetVals - If we are tracking arguments into and the return
-  /// value out of a function, it will have an entry in this map, indicating
-  /// what the known return value for the function is.
-  MapVector<Function *, ValueLatticeElement> TrackedRetVals;
-
-  /// TrackedMultipleRetVals - Same as TrackedRetVals, but used for functions
-  /// that return multiple values.
-  MapVector<std::pair<Function *, unsigned>, ValueLatticeElement>
-      TrackedMultipleRetVals;
-
-  /// MRVFunctionsTracked - Each function in TrackedMultipleRetVals is
-  /// represented here for efficient lookup.
-  SmallPtrSet<Function *, 16> MRVFunctionsTracked;
-
-  /// MustTailFunctions - Each function here is a callee of non-removable
-  /// musttail call site.
-  SmallPtrSet<Function *, 16> MustTailCallees;
-
-  /// TrackingIncomingArguments - This is the set of functions for whose
-  /// arguments we make optimistic assumptions about and try to prove as
-  /// constants.
-  SmallPtrSet<Function *, 16> TrackingIncomingArguments;
-
-  /// The reason for two worklists is that overdefined is the lowest state
-  /// on the lattice, and moving things to overdefined as fast as possible
-  /// makes SCCP converge much faster.
-  ///
-  /// By having a separate worklist, we accomplish this because everything
-  /// possibly overdefined will become overdefined at the soonest possible
-  /// point.
-  SmallVector<Value *, 64> OverdefinedInstWorkList;
-  SmallVector<Value *, 64> InstWorkList;
-
-  // The BasicBlock work list
-  SmallVector<BasicBlock *, 64>  BBWorkList;
-
-  /// KnownFeasibleEdges - Entries in this set are edges which have already had
-  /// PHI nodes retriggered.
-  using Edge = std::pair<BasicBlock *, BasicBlock *>;
-  DenseSet<Edge> KnownFeasibleEdges;
-
-  DenseMap<Function *, AnalysisResultsForFn> AnalysisResults;
-  DenseMap<Value *, SmallPtrSet<User *, 2>> AdditionalUsers;
-
-  LLVMContext &Ctx;
-
-public:
-  void addAnalysis(Function &F, AnalysisResultsForFn A) {
-    AnalysisResults.insert({&F, std::move(A)});
-  }
-
-  const PredicateBase *getPredicateInfoFor(Instruction *I) {
-    auto A = AnalysisResults.find(I->getParent()->getParent());
-    if (A == AnalysisResults.end())
-      return nullptr;
-    return A->second.PredInfo->getPredicateInfoFor(I);
-  }
-
-  DomTreeUpdater getDTU(Function &F) {
-    auto A = AnalysisResults.find(&F);
-    assert(A != AnalysisResults.end() && "Need analysis results for function.");
-    return {A->second.DT, A->second.PDT, DomTreeUpdater::UpdateStrategy::Lazy};
-  }
-
-  SCCPSolver(const DataLayout &DL,
-             std::function<const TargetLibraryInfo &(Function &)> GetTLI,
-             LLVMContext &Ctx)
-      : DL(DL), GetTLI(std::move(GetTLI)), Ctx(Ctx) {}
-
-  /// MarkBlockExecutable - This method can be used by clients to mark all of
-  /// the blocks that are known to be intrinsically live in the processed unit.
-  ///
-  /// This returns true if the block was not considered live before.
-  bool MarkBlockExecutable(BasicBlock *BB) {
-    if (!BBExecutable.insert(BB).second)
-      return false;
-    LLVM_DEBUG(dbgs() << "Marking Block Executable: " << BB->getName() << '\n');
-    BBWorkList.push_back(BB);  // Add the block to the work list!
-    return true;
-  }
-
-  /// TrackValueOfGlobalVariable - Clients can use this method to
-  /// inform the SCCPSolver that it should track loads and stores to the
-  /// specified global variable if it can.  This is only legal to call if
-  /// performing Interprocedural SCCP.
-  void TrackValueOfGlobalVariable(GlobalVariable *GV) {
-    // We only track the contents of scalar globals.
-    if (GV->getValueType()->isSingleValueType()) {
-      ValueLatticeElement &IV = TrackedGlobals[GV];
-      if (!isa<UndefValue>(GV->getInitializer()))
-        IV.markConstant(GV->getInitializer());
-    }
-  }
-
-  /// AddTrackedFunction - If the SCCP solver is supposed to track calls into
-  /// and out of the specified function (which cannot have its address taken),
-  /// this method must be called.
-  void AddTrackedFunction(Function *F) {
-    // Add an entry, F -> undef.
-    if (auto *STy = dyn_cast<StructType>(F->getReturnType())) {
-      MRVFunctionsTracked.insert(F);
-      for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
-        TrackedMultipleRetVals.insert(
-            std::make_pair(std::make_pair(F, i), ValueLatticeElement()));
-    } else if (!F->getReturnType()->isVoidTy())
-      TrackedRetVals.insert(std::make_pair(F, ValueLatticeElement()));
-  }
-
-  /// AddMustTailCallee - If the SCCP solver finds that this function is called
-  /// from non-removable musttail call site.
-  void AddMustTailCallee(Function *F) {
-    MustTailCallees.insert(F);
-  }
-
-  /// Returns true if the given function is called from non-removable musttail
-  /// call site.
-  bool isMustTailCallee(Function *F) {
-    return MustTailCallees.count(F);
-  }
-
-  void AddArgumentTrackedFunction(Function *F) {
-    TrackingIncomingArguments.insert(F);
-  }
-
-  /// Returns true if the given function is in the solver's set of
-  /// argument-tracked functions.
-  bool isArgumentTrackedFunction(Function *F) {
-    return TrackingIncomingArguments.count(F);
-  }
-
-  /// Solve - Solve for constants and executable blocks.
-  void Solve();
-
-  /// ResolvedUndefsIn - While solving the dataflow for a function, we assume
-  /// that branches on undef values cannot reach any of their successors.
-  /// However, this is not a safe assumption.  After we solve dataflow, this
-  /// method should be use to handle this.  If this returns true, the solver
-  /// should be rerun.
-  bool ResolvedUndefsIn(Function &F);
-
-  bool isBlockExecutable(BasicBlock *BB) const {
-    return BBExecutable.count(BB);
-  }
-
-  // isEdgeFeasible - Return true if the control flow edge from the 'From' basic
-  // block to the 'To' basic block is currently feasible.
-  bool isEdgeFeasible(BasicBlock *From, BasicBlock *To) const;
-
-  std::vector<ValueLatticeElement> getStructLatticeValueFor(Value *V) const {
-    std::vector<ValueLatticeElement> StructValues;
-    auto *STy = dyn_cast<StructType>(V->getType());
-    assert(STy && "getStructLatticeValueFor() can be called only on structs");
-    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-      auto I = StructValueState.find(std::make_pair(V, i));
-      assert(I != StructValueState.end() && "Value not in valuemap!");
-      StructValues.push_back(I->second);
-    }
-    return StructValues;
-  }
-
-  void removeLatticeValueFor(Value *V) { ValueState.erase(V); }
-
-  const ValueLatticeElement &getLatticeValueFor(Value *V) const {
-    assert(!V->getType()->isStructTy() &&
-           "Should use getStructLatticeValueFor");
-    DenseMap<Value *, ValueLatticeElement>::const_iterator I =
-        ValueState.find(V);
-    assert(I != ValueState.end() &&
-           "V not found in ValueState nor Paramstate map!");
-    return I->second;
-  }
-
-  /// getTrackedRetVals - Get the inferred return value map.
-  const MapVector<Function *, ValueLatticeElement> &getTrackedRetVals() {
-    return TrackedRetVals;
-  }
-
-  /// getTrackedGlobals - Get and return the set of inferred initializers for
-  /// global variables.
-  const DenseMap<GlobalVariable *, ValueLatticeElement> &getTrackedGlobals() {
-    return TrackedGlobals;
-  }
-
-  /// getMRVFunctionsTracked - Get the set of functions which return multiple
-  /// values tracked by the pass.
-  const SmallPtrSet<Function *, 16> getMRVFunctionsTracked() {
-    return MRVFunctionsTracked;
-  }
-
-  /// getMustTailCallees - Get the set of functions which are called
-  /// from non-removable musttail call sites.
-  const SmallPtrSet<Function *, 16> getMustTailCallees() {
-    return MustTailCallees;
-  }
-
-  /// markOverdefined - Mark the specified value overdefined.  This
-  /// works with both scalars and structs.
-  void markOverdefined(Value *V) {
-    if (auto *STy = dyn_cast<StructType>(V->getType()))
-      for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
-        markOverdefined(getStructValueState(V, i), V);
-    else
-      markOverdefined(ValueState[V], V);
-  }
-
-  // isStructLatticeConstant - Return true if all the lattice values
-  // corresponding to elements of the structure are constants,
-  // false otherwise.
-  bool isStructLatticeConstant(Function *F, StructType *STy) {
-    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-      const auto &It = TrackedMultipleRetVals.find(std::make_pair(F, i));
-      assert(It != TrackedMultipleRetVals.end());
-      ValueLatticeElement LV = It->second;
-      if (!isConstant(LV))
-        return false;
-    }
-    return true;
-  }
-
-  /// Helper to return a Constant if \p LV is either a constant or a constant
-  /// range with a single element.
-  Constant *getConstant(const ValueLatticeElement &LV) const {
-    if (LV.isConstant())
-      return LV.getConstant();
-
-    if (LV.isConstantRange()) {
-      auto &CR = LV.getConstantRange();
-      if (CR.getSingleElement())
-        return ConstantInt::get(Ctx, *CR.getSingleElement());
-    }
-    return nullptr;
-  }
-
-private:
-  ConstantInt *getConstantInt(const ValueLatticeElement &IV) const {
-    return dyn_cast_or_null<ConstantInt>(getConstant(IV));
-  }
-
-  // pushToWorkList - Helper for markConstant/markOverdefined
-  void pushToWorkList(ValueLatticeElement &IV, Value *V) {
-    if (IV.isOverdefined())
-      return OverdefinedInstWorkList.push_back(V);
-    InstWorkList.push_back(V);
-  }
-
-  // Helper to push \p V to the worklist, after updating it to \p IV. Also
-  // prints a debug message with the updated value.
-  void pushToWorkListMsg(ValueLatticeElement &IV, Value *V) {
-    LLVM_DEBUG(dbgs() << "updated " << IV << ": " << *V << '\n');
-    pushToWorkList(IV, V);
-  }
-
-  // markConstant - Make a value be marked as "constant".  If the value
-  // is not already a constant, add it to the instruction work list so that
-  // the users of the instruction are updated later.
-  bool markConstant(ValueLatticeElement &IV, Value *V, Constant *C,
-                    bool MayIncludeUndef = false) {
-    if (!IV.markConstant(C, MayIncludeUndef))
-      return false;
-    LLVM_DEBUG(dbgs() << "markConstant: " << *C << ": " << *V << '\n');
-    pushToWorkList(IV, V);
-    return true;
-  }
-
-  bool markConstant(Value *V, Constant *C) {
-    assert(!V->getType()->isStructTy() && "structs should use mergeInValue");
-    return markConstant(ValueState[V], V, C);
-  }
-
-  // markOverdefined - Make a value be marked as "overdefined". If the
-  // value is not already overdefined, add it to the overdefined instruction
-  // work list so that the users of the instruction are updated later.
-  bool markOverdefined(ValueLatticeElement &IV, Value *V) {
-    if (!IV.markOverdefined()) return false;
-
-    LLVM_DEBUG(dbgs() << "markOverdefined: ";
-               if (auto *F = dyn_cast<Function>(V)) dbgs()
-               << "Function '" << F->getName() << "'\n";
-               else dbgs() << *V << '\n');
-    // Only instructions go on the work list
-    pushToWorkList(IV, V);
-    return true;
-  }
-
-  /// Merge \p MergeWithV into \p IV and push \p V to the worklist, if \p IV
-  /// changes.
-  bool mergeInValue(ValueLatticeElement &IV, Value *V,
-                    ValueLatticeElement MergeWithV,
-                    ValueLatticeElement::MergeOptions Opts = {
-                        /*MayIncludeUndef=*/false, /*CheckWiden=*/false}) {
-    if (IV.mergeIn(MergeWithV, Opts)) {
-      pushToWorkList(IV, V);
-      LLVM_DEBUG(dbgs() << "Merged " << MergeWithV << " into " << *V << " : "
-                        << IV << "\n");
-      return true;
-    }
-    return false;
-  }
-
-  bool mergeInValue(Value *V, ValueLatticeElement MergeWithV,
-                    ValueLatticeElement::MergeOptions Opts = {
-                        /*MayIncludeUndef=*/false, /*CheckWiden=*/false}) {
-    assert(!V->getType()->isStructTy() &&
-           "non-structs should use markConstant");
-    return mergeInValue(ValueState[V], V, MergeWithV, Opts);
-  }
-
-  /// getValueState - Return the ValueLatticeElement object that corresponds to
-  /// the value.  This function handles the case when the value hasn't been seen
-  /// yet by properly seeding constants etc.
-  ValueLatticeElement &getValueState(Value *V) {
-    assert(!V->getType()->isStructTy() && "Should use getStructValueState");
-
-    auto I = ValueState.insert(std::make_pair(V, ValueLatticeElement()));
-    ValueLatticeElement &LV = I.first->second;
-
-    if (!I.second)
-      return LV;  // Common case, already in the map.
-
-    if (auto *C = dyn_cast<Constant>(V))
-      LV.markConstant(C);          // Constants are constant
-
-    // All others are unknown by default.
-    return LV;
-  }
-
-  /// getStructValueState - Return the ValueLatticeElement object that
-  /// corresponds to the value/field pair.  This function handles the case when
-  /// the value hasn't been seen yet by properly seeding constants etc.
-  ValueLatticeElement &getStructValueState(Value *V, unsigned i) {
-    assert(V->getType()->isStructTy() && "Should use getValueState");
-    assert(i < cast<StructType>(V->getType())->getNumElements() &&
-           "Invalid element #");
-
-    auto I = StructValueState.insert(
-        std::make_pair(std::make_pair(V, i), ValueLatticeElement()));
-    ValueLatticeElement &LV = I.first->second;
-
-    if (!I.second)
-      return LV;  // Common case, already in the map.
-
-    if (auto *C = dyn_cast<Constant>(V)) {
-      Constant *Elt = C->getAggregateElement(i);
-
-      if (!Elt)
-        LV.markOverdefined();      // Unknown sort of constant.
-      else if (isa<UndefValue>(Elt))
-        ; // Undef values remain unknown.
-      else
-        LV.markConstant(Elt);      // Constants are constant.
-    }
-
-    // All others are underdefined by default.
-    return LV;
-  }
-
-  /// markEdgeExecutable - Mark a basic block as executable, adding it to the BB
-  /// work list if it is not already executable.
-  bool markEdgeExecutable(BasicBlock *Source, BasicBlock *Dest) {
-    if (!KnownFeasibleEdges.insert(Edge(Source, Dest)).second)
-      return false;  // This edge is already known to be executable!
-
-    if (!MarkBlockExecutable(Dest)) {
-      // If the destination is already executable, we just made an *edge*
-      // feasible that wasn't before.  Revisit the PHI nodes in the block
-      // because they have potentially new operands.
-      LLVM_DEBUG(dbgs() << "Marking Edge Executable: " << Source->getName()
-                        << " -> " << Dest->getName() << '\n');
-
-      for (PHINode &PN : Dest->phis())
-        visitPHINode(PN);
-    }
-    return true;
-  }
-
-  // getFeasibleSuccessors - Return a vector of booleans to indicate which
-  // successors are reachable from a given terminator instruction.
-  void getFeasibleSuccessors(Instruction &TI, SmallVectorImpl<bool> &Succs);
-
-  // OperandChangedState - This method is invoked on all of the users of an
-  // instruction that was just changed state somehow.  Based on this
-  // information, we need to update the specified user of this instruction.
-  void OperandChangedState(Instruction *I) {
-    if (BBExecutable.count(I->getParent()))   // Inst is executable?
-      visit(*I);
-  }
-
-  // Add U as additional user of V.
-  void addAdditionalUser(Value *V, User *U) {
-    auto Iter = AdditionalUsers.insert({V, {}});
-    Iter.first->second.insert(U);
-  }
-
-  // Mark I's users as changed, including AdditionalUsers.
-  void markUsersAsChanged(Value *I) {
-    // Functions include their arguments in the use-list. Changed function
-    // values mean that the result of the function changed. We only need to
-    // update the call sites with the new function result and do not have to
-    // propagate the call arguments.
-    if (isa<Function>(I)) {
-      for (User *U : I->users()) {
-        if (auto *CB = dyn_cast<CallBase>(U))
-          handleCallResult(*CB);
-      }
-    } else {
-      for (User *U : I->users())
-        if (auto *UI = dyn_cast<Instruction>(U))
-          OperandChangedState(UI);
-    }
-
-    auto Iter = AdditionalUsers.find(I);
-    if (Iter != AdditionalUsers.end()) {
-      for (User *U : Iter->second)
-        if (auto *UI = dyn_cast<Instruction>(U))
-          OperandChangedState(UI);
-    }
-  }
-  void handleCallOverdefined(CallBase &CB);
-  void handleCallResult(CallBase &CB);
-  void handleCallArguments(CallBase &CB);
-
-private:
-  friend class InstVisitor<SCCPSolver>;
-
-  // visit implementations - Something changed in this instruction.  Either an
-  // operand made a transition, or the instruction is newly executable.  Change
-  // the value type of I to reflect these changes if appropriate.
-  void visitPHINode(PHINode &I);
-
-  // Terminators
-
-  void visitReturnInst(ReturnInst &I);
-  void visitTerminator(Instruction &TI);
-
-  void visitCastInst(CastInst &I);
-  void visitSelectInst(SelectInst &I);
-  void visitUnaryOperator(Instruction &I);
-  void visitBinaryOperator(Instruction &I);
-  void visitCmpInst(CmpInst &I);
-  void visitExtractValueInst(ExtractValueInst &EVI);
-  void visitInsertValueInst(InsertValueInst &IVI);
-
-  void visitCatchSwitchInst(CatchSwitchInst &CPI) {
-    markOverdefined(&CPI);
-    visitTerminator(CPI);
-  }
-
-  // Instructions that cannot be folded away.
-
-  void visitStoreInst     (StoreInst &I);
-  void visitLoadInst      (LoadInst &I);
-  void visitGetElementPtrInst(GetElementPtrInst &I);
-
-  void visitCallInst      (CallInst &I) {
-    visitCallBase(I);
-  }
-
-  void visitInvokeInst    (InvokeInst &II) {
-    visitCallBase(II);
-    visitTerminator(II);
-  }
-
-  void visitCallBrInst    (CallBrInst &CBI) {
-    visitCallBase(CBI);
-    visitTerminator(CBI);
-  }
-
-  void visitCallBase      (CallBase &CB);
-  void visitResumeInst    (ResumeInst &I) { /*returns void*/ }
-  void visitUnreachableInst(UnreachableInst &I) { /*returns void*/ }
-  void visitFenceInst     (FenceInst &I) { /*returns void*/ }
-
-  void visitInstruction(Instruction &I) {
-    // All the instructions we don't do any special handling for just
-    // go to overdefined.
-    LLVM_DEBUG(dbgs() << "SCCP: Don't know how to handle: " << I << '\n');
-    markOverdefined(&I);
-  }
-};
-
-} // end anonymous namespace
-
-// getFeasibleSuccessors - Return a vector of booleans to indicate which
-// successors are reachable from a given terminator instruction.
-void SCCPSolver::getFeasibleSuccessors(Instruction &TI,
-                                       SmallVectorImpl<bool> &Succs) {
-  Succs.resize(TI.getNumSuccessors());
-  if (auto *BI = dyn_cast<BranchInst>(&TI)) {
-    if (BI->isUnconditional()) {
-      Succs[0] = true;
-      return;
-    }
-
-    ValueLatticeElement BCValue = getValueState(BI->getCondition());
-    ConstantInt *CI = getConstantInt(BCValue);
-    if (!CI) {
-      // Overdefined condition variables, and branches on unfoldable constant
-      // conditions, mean the branch could go either way.
-      if (!BCValue.isUnknownOrUndef())
-        Succs[0] = Succs[1] = true;
-      return;
-    }
-
-    // Constant condition variables mean the branch can only go a single way.
-    Succs[CI->isZero()] = true;
-    return;
-  }
-
-  // Unwinding instructions successors are always executable.
-  if (TI.isExceptionalTerminator()) {
-    Succs.assign(TI.getNumSuccessors(), true);
-    return;
-  }
-
-  if (auto *SI = dyn_cast<SwitchInst>(&TI)) {
-    if (!SI->getNumCases()) {
-      Succs[0] = true;
-      return;
-    }
-    const ValueLatticeElement &SCValue = getValueState(SI->getCondition());
-    if (ConstantInt *CI = getConstantInt(SCValue)) {
-      Succs[SI->findCaseValue(CI)->getSuccessorIndex()] = true;
-      return;
-    }
-
-    // TODO: Switch on undef is UB. Stop passing false once the rest of LLVM
-    // is ready.
-    if (SCValue.isConstantRange(/*UndefAllowed=*/false)) {
-      const ConstantRange &Range = SCValue.getConstantRange();
-      for (const auto &Case : SI->cases()) {
-        const APInt &CaseValue = Case.getCaseValue()->getValue();
-        if (Range.contains(CaseValue))
-          Succs[Case.getSuccessorIndex()] = true;
-      }
-
-      // TODO: Determine whether default case is reachable.
-      Succs[SI->case_default()->getSuccessorIndex()] = true;
-      return;
-    }
-
-    // Overdefined or unknown condition? All destinations are executable!
-    if (!SCValue.isUnknownOrUndef())
-      Succs.assign(TI.getNumSuccessors(), true);
-    return;
-  }
-
-  // In case of indirect branch and its address is a blockaddress, we mark
-  // the target as executable.
-  if (auto *IBR = dyn_cast<IndirectBrInst>(&TI)) {
-    // Casts are folded by visitCastInst.
-    ValueLatticeElement IBRValue = getValueState(IBR->getAddress());
-    BlockAddress *Addr = dyn_cast_or_null<BlockAddress>(getConstant(IBRValue));
-    if (!Addr) {   // Overdefined or unknown condition?
-      // All destinations are executable!
-      if (!IBRValue.isUnknownOrUndef())
-        Succs.assign(TI.getNumSuccessors(), true);
-      return;
-    }
-
-    BasicBlock* T = Addr->getBasicBlock();
-    assert(Addr->getFunction() == T->getParent() &&
-           "Block address of a different function ?");
-    for (unsigned i = 0; i < IBR->getNumSuccessors(); ++i) {
-      // This is the target.
-      if (IBR->getDestination(i) == T) {
-        Succs[i] = true;
-        return;
-      }
-    }
-
-    // If we didn't find our destination in the IBR successor list, then we
-    // have undefined behavior. Its ok to assume no successor is executable.
-    return;
-  }
-
-  // In case of callbr, we pessimistically assume that all successors are
-  // feasible.
-  if (isa<CallBrInst>(&TI)) {
-    Succs.assign(TI.getNumSuccessors(), true);
-    return;
-  }
-
-  LLVM_DEBUG(dbgs() << "Unknown terminator instruction: " << TI << '\n');
-  llvm_unreachable("SCCP: Don't know how to handle this terminator!");
-}
-
-// isEdgeFeasible - Return true if the control flow edge from the 'From' basic
-// block to the 'To' basic block is currently feasible.
-bool SCCPSolver::isEdgeFeasible(BasicBlock *From, BasicBlock *To) const {
-  // Check if we've called markEdgeExecutable on the edge yet. (We could
-  // be more aggressive and try to consider edges which haven't been marked
-  // yet, but there isn't any need.)
-  return KnownFeasibleEdges.count(Edge(From, To));
-}
-
-// visit Implementations - Something changed in this instruction, either an
-// operand made a transition, or the instruction is newly executable.  Change
-// the value type of I to reflect these changes if appropriate.  This method
-// makes sure to do the following actions:
-//
-// 1. If a phi node merges two constants in, and has conflicting value coming
-//    from different branches, or if the PHI node merges in an overdefined
-//    value, then the PHI node becomes overdefined.
-// 2. If a phi node merges only constants in, and they all agree on value, the
-//    PHI node becomes a constant value equal to that.
-// 3. If V <- x (op) y && isConstant(x) && isConstant(y) V = Constant
-// 4. If V <- x (op) y && (isOverdefined(x) || isOverdefined(y)) V = Overdefined
-// 5. If V <- MEM or V <- CALL or V <- (unknown) then V = Overdefined
-// 6. If a conditional branch has a value that is constant, make the selected
-//    destination executable
-// 7. If a conditional branch has a value that is overdefined, make all
-//    successors executable.
-void SCCPSolver::visitPHINode(PHINode &PN) {
-  // If this PN returns a struct, just mark the result overdefined.
-  // TODO: We could do a lot better than this if code actually uses this.
-  if (PN.getType()->isStructTy())
-    return (void)markOverdefined(&PN);
-
-  if (getValueState(&PN).isOverdefined())
-    return; // Quick exit
-
-  // Super-extra-high-degree PHI nodes are unlikely to ever be marked constant,
-  // and slow us down a lot.  Just mark them overdefined.
-  if (PN.getNumIncomingValues() > 64)
-    return (void)markOverdefined(&PN);
-
-  unsigned NumActiveIncoming = 0;
-
-  // Look at all of the executable operands of the PHI node.  If any of them
-  // are overdefined, the PHI becomes overdefined as well.  If they are all
-  // constant, and they agree with each other, the PHI becomes the identical
-  // constant.  If they are constant and don't agree, the PHI is a constant
-  // range. If there are no executable operands, the PHI remains unknown.
-  ValueLatticeElement PhiState = getValueState(&PN);
-  for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
-    if (!isEdgeFeasible(PN.getIncomingBlock(i), PN.getParent()))
-      continue;
-
-    ValueLatticeElement IV = getValueState(PN.getIncomingValue(i));
-    PhiState.mergeIn(IV);
-    NumActiveIncoming++;
-    if (PhiState.isOverdefined())
-      break;
-  }
-
-  // We allow up to 1 range extension per active incoming value and one
-  // additional extension. Note that we manually adjust the number of range
-  // extensions to match the number of active incoming values. This helps to
-  // limit multiple extensions caused by the same incoming value, if other
-  // incoming values are equal.
-  mergeInValue(&PN, PhiState,
-               ValueLatticeElement::MergeOptions().setMaxWidenSteps(
-                   NumActiveIncoming + 1));
-  ValueLatticeElement &PhiStateRef = getValueState(&PN);
-  PhiStateRef.setNumRangeExtensions(
-      std::max(NumActiveIncoming, PhiStateRef.getNumRangeExtensions()));
-}
-
-void SCCPSolver::visitReturnInst(ReturnInst &I) {
-  if (I.getNumOperands() == 0) return;  // ret void
-
-  Function *F = I.getParent()->getParent();
-  Value *ResultOp = I.getOperand(0);
-
-  // If we are tracking the return value of this function, merge it in.
-  if (!TrackedRetVals.empty() && !ResultOp->getType()->isStructTy()) {
-    auto TFRVI = TrackedRetVals.find(F);
-    if (TFRVI != TrackedRetVals.end()) {
-      mergeInValue(TFRVI->second, F, getValueState(ResultOp));
-      return;
-    }
-  }
-
-  // Handle functions that return multiple values.
-  if (!TrackedMultipleRetVals.empty()) {
-    if (auto *STy = dyn_cast<StructType>(ResultOp->getType()))
-      if (MRVFunctionsTracked.count(F))
-        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
-          mergeInValue(TrackedMultipleRetVals[std::make_pair(F, i)], F,
-                       getStructValueState(ResultOp, i));
-  }
-}
-
-void SCCPSolver::visitTerminator(Instruction &TI) {
-  SmallVector<bool, 16> SuccFeasible;
-  getFeasibleSuccessors(TI, SuccFeasible);
-
-  BasicBlock *BB = TI.getParent();
-
-  // Mark all feasible successors executable.
-  for (unsigned i = 0, e = SuccFeasible.size(); i != e; ++i)
-    if (SuccFeasible[i])
-      markEdgeExecutable(BB, TI.getSuccessor(i));
-}
-
-void SCCPSolver::visitCastInst(CastInst &I) {
-  // ResolvedUndefsIn might mark I as overdefined. Bail out, even if we would
-  // discover a concrete value later.
-  if (ValueState[&I].isOverdefined())
-    return;
-
-  ValueLatticeElement OpSt = getValueState(I.getOperand(0));
-  if (Constant *OpC = getConstant(OpSt)) {
-    // Fold the constant as we build.
-    Constant *C = ConstantFoldCastOperand(I.getOpcode(), OpC, I.getType(), DL);
-    if (isa<UndefValue>(C))
-      return;
-    // Propagate constant value
-    markConstant(&I, C);
-  } else if (OpSt.isConstantRange() && I.getDestTy()->isIntegerTy()) {
-    auto &LV = getValueState(&I);
-    ConstantRange OpRange = OpSt.getConstantRange();
-    Type *DestTy = I.getDestTy();
-    // Vectors where all elements have the same known constant range are treated
-    // as a single constant range in the lattice. When bitcasting such vectors,
-    // there is a mis-match between the width of the lattice value (single
-    // constant range) and the original operands (vector). Go to overdefined in
-    // that case.
-    if (I.getOpcode() == Instruction::BitCast &&
-        I.getOperand(0)->getType()->isVectorTy() &&
-        OpRange.getBitWidth() < DL.getTypeSizeInBits(DestTy))
-      return (void)markOverdefined(&I);
-
-    ConstantRange Res =
-        OpRange.castOp(I.getOpcode(), DL.getTypeSizeInBits(DestTy));
-    mergeInValue(LV, &I, ValueLatticeElement::getRange(Res));
-  } else if (!OpSt.isUnknownOrUndef())
-    markOverdefined(&I);
-}
-
-void SCCPSolver::visitExtractValueInst(ExtractValueInst &EVI) {
-  // If this returns a struct, mark all elements over defined, we don't track
-  // structs in structs.
-  if (EVI.getType()->isStructTy())
-    return (void)markOverdefined(&EVI);
-
-  // ResolvedUndefsIn might mark I as overdefined. Bail out, even if we would
-  // discover a concrete value later.
-  if (ValueState[&EVI].isOverdefined())
-    return (void)markOverdefined(&EVI);
-
-  // If this is extracting from more than one level of struct, we don't know.
-  if (EVI.getNumIndices() != 1)
-    return (void)markOverdefined(&EVI);
-
-  Value *AggVal = EVI.getAggregateOperand();
-  if (AggVal->getType()->isStructTy()) {
-    unsigned i = *EVI.idx_begin();
-    ValueLatticeElement EltVal = getStructValueState(AggVal, i);
-    mergeInValue(getValueState(&EVI), &EVI, EltVal);
-  } else {
-    // Otherwise, must be extracting from an array.
-    return (void)markOverdefined(&EVI);
-  }
-}
-
-void SCCPSolver::visitInsertValueInst(InsertValueInst &IVI) {
-  auto *STy = dyn_cast<StructType>(IVI.getType());
-  if (!STy)
-    return (void)markOverdefined(&IVI);
-
-  // ResolvedUndefsIn might mark I as overdefined. Bail out, even if we would
-  // discover a concrete value later.
-  if (isOverdefined(ValueState[&IVI]))
-    return (void)markOverdefined(&IVI);
-
-  // If this has more than one index, we can't handle it, drive all results to
-  // undef.
-  if (IVI.getNumIndices() != 1)
-    return (void)markOverdefined(&IVI);
-
-  Value *Aggr = IVI.getAggregateOperand();
-  unsigned Idx = *IVI.idx_begin();
-
-  // Compute the result based on what we're inserting.
-  for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-    // This passes through all values that aren't the inserted element.
-    if (i != Idx) {
-      ValueLatticeElement EltVal = getStructValueState(Aggr, i);
-      mergeInValue(getStructValueState(&IVI, i), &IVI, EltVal);
-      continue;
-    }
-
-    Value *Val = IVI.getInsertedValueOperand();
-    if (Val->getType()->isStructTy())
-      // We don't track structs in structs.
-      markOverdefined(getStructValueState(&IVI, i), &IVI);
-    else {
-      ValueLatticeElement InVal = getValueState(Val);
-      mergeInValue(getStructValueState(&IVI, i), &IVI, InVal);
-    }
-  }
-}
-
-void SCCPSolver::visitSelectInst(SelectInst &I) {
-  // If this select returns a struct, just mark the result overdefined.
-  // TODO: We could do a lot better than this if code actually uses this.
-  if (I.getType()->isStructTy())
-    return (void)markOverdefined(&I);
-
-  // ResolvedUndefsIn might mark I as overdefined. Bail out, even if we would
-  // discover a concrete value later.
-  if (ValueState[&I].isOverdefined())
-    return (void)markOverdefined(&I);
-
-  ValueLatticeElement CondValue = getValueState(I.getCondition());
-  if (CondValue.isUnknownOrUndef())
-    return;
-
-  if (ConstantInt *CondCB = getConstantInt(CondValue)) {
-    Value *OpVal = CondCB->isZero() ? I.getFalseValue() : I.getTrueValue();
-    mergeInValue(&I, getValueState(OpVal));
-    return;
-  }
-
-  // Otherwise, the condition is overdefined or a constant we can't evaluate.
-  // See if we can produce something better than overdefined based on the T/F
-  // value.
-  ValueLatticeElement TVal = getValueState(I.getTrueValue());
-  ValueLatticeElement FVal = getValueState(I.getFalseValue());
-
-  bool Changed = ValueState[&I].mergeIn(TVal);
-  Changed |= ValueState[&I].mergeIn(FVal);
-  if (Changed)
-    pushToWorkListMsg(ValueState[&I], &I);
-}
-
-// Handle Unary Operators.
-void SCCPSolver::visitUnaryOperator(Instruction &I) {
-  ValueLatticeElement V0State = getValueState(I.getOperand(0));
-
-  ValueLatticeElement &IV = ValueState[&I];
-  // ResolvedUndefsIn might mark I as overdefined. Bail out, even if we would
-  // discover a concrete value later.
-  if (isOverdefined(IV))
-    return (void)markOverdefined(&I);
-
-  if (isConstant(V0State)) {
-    Constant *C = ConstantExpr::get(I.getOpcode(), getConstant(V0State));
-
-    // op Y -> undef.
-    if (isa<UndefValue>(C))
-      return;
-    return (void)markConstant(IV, &I, C);
-  }
-
-  // If something is undef, wait for it to resolve.
-  if (!isOverdefined(V0State))
-    return;
-
-  markOverdefined(&I);
-}
-
-// Handle Binary Operators.
-void SCCPSolver::visitBinaryOperator(Instruction &I) {
-  ValueLatticeElement V1State = getValueState(I.getOperand(0));
-  ValueLatticeElement V2State = getValueState(I.getOperand(1));
-
-  ValueLatticeElement &IV = ValueState[&I];
-  if (IV.isOverdefined())
-    return;
-
-  // If something is undef, wait for it to resolve.
-  if (V1State.isUnknownOrUndef() || V2State.isUnknownOrUndef())
-    return;
-
-  if (V1State.isOverdefined() && V2State.isOverdefined())
-    return (void)markOverdefined(&I);
-
-  // If either of the operands is a constant, try to fold it to a constant.
-  // TODO: Use information from notconstant better.
-  if ((V1State.isConstant() || V2State.isConstant())) {
-    Value *V1 = isConstant(V1State) ? getConstant(V1State) : I.getOperand(0);
-    Value *V2 = isConstant(V2State) ? getConstant(V2State) : I.getOperand(1);
-    Value *R = SimplifyBinOp(I.getOpcode(), V1, V2, SimplifyQuery(DL));
-    auto *C = dyn_cast_or_null<Constant>(R);
-    if (C) {
-      // X op Y -> undef.
-      if (isa<UndefValue>(C))
-        return;
-      // Conservatively assume that the result may be based on operands that may
-      // be undef. Note that we use mergeInValue to combine the constant with
-      // the existing lattice value for I, as different constants might be found
-      // after one of the operands go to overdefined, e.g. due to one operand
-      // being a special floating value.
-      ValueLatticeElement NewV;
-      NewV.markConstant(C, /*MayIncludeUndef=*/true);
-      return (void)mergeInValue(&I, NewV);
-    }
-  }
-
-  // Only use ranges for binary operators on integers.
-  if (!I.getType()->isIntegerTy())
-    return markOverdefined(&I);
-
-  // Try to simplify to a constant range.
-  ConstantRange A = ConstantRange::getFull(I.getType()->getScalarSizeInBits());
-  ConstantRange B = ConstantRange::getFull(I.getType()->getScalarSizeInBits());
-  if (V1State.isConstantRange())
-    A = V1State.getConstantRange();
-  if (V2State.isConstantRange())
-    B = V2State.getConstantRange();
-
-  ConstantRange R = A.binaryOp(cast<BinaryOperator>(&I)->getOpcode(), B);
-  mergeInValue(&I, ValueLatticeElement::getRange(R));
-
-  // TODO: Currently we do not exploit special values that produce something
-  // better than overdefined with an overdefined operand for vector or floating
-  // point types, like and <4 x i32> overdefined, zeroinitializer.
-}
-
-// Handle ICmpInst instruction.
-void SCCPSolver::visitCmpInst(CmpInst &I) {
-  // Do not cache this lookup, getValueState calls later in the function might
-  // invalidate the reference.
-  if (isOverdefined(ValueState[&I]))
-    return (void)markOverdefined(&I);
-
-  Value *Op1 = I.getOperand(0);
-  Value *Op2 = I.getOperand(1);
-
-  // For parameters, use ParamState which includes constant range info if
-  // available.
-  auto V1State = getValueState(Op1);
-  auto V2State = getValueState(Op2);
-
-  Constant *C = V1State.getCompare(I.getPredicate(), I.getType(), V2State);
-  if (C) {
-    if (isa<UndefValue>(C))
-      return;
-    ValueLatticeElement CV;
-    CV.markConstant(C);
-    mergeInValue(&I, CV);
-    return;
-  }
-
-  // If operands are still unknown, wait for it to resolve.
-  if ((V1State.isUnknownOrUndef() || V2State.isUnknownOrUndef()) &&
-      !isConstant(ValueState[&I]))
-    return;
-
-  markOverdefined(&I);
-}
-
-// Handle getelementptr instructions.  If all operands are constants then we
-// can turn this into a getelementptr ConstantExpr.
-void SCCPSolver::visitGetElementPtrInst(GetElementPtrInst &I) {
-  if (isOverdefined(ValueState[&I]))
-    return (void)markOverdefined(&I);
-
-  SmallVector<Constant*, 8> Operands;
-  Operands.reserve(I.getNumOperands());
-
-  for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i) {
-    ValueLatticeElement State = getValueState(I.getOperand(i));
-    if (State.isUnknownOrUndef())
-      return;  // Operands are not resolved yet.
-
-    if (isOverdefined(State))
-      return (void)markOverdefined(&I);
-
-    if (Constant *C = getConstant(State)) {
-      Operands.push_back(C);
-      continue;
-    }
-
-    return (void)markOverdefined(&I);
-  }
-
-  Constant *Ptr = Operands[0];
-  auto Indices = makeArrayRef(Operands.begin() + 1, Operands.end());
-  Constant *C =
-      ConstantExpr::getGetElementPtr(I.getSourceElementType(), Ptr, Indices);
-  if (isa<UndefValue>(C))
-      return;
-  markConstant(&I, C);
-}
-
-void SCCPSolver::visitStoreInst(StoreInst &SI) {
-  // If this store is of a struct, ignore it.
-  if (SI.getOperand(0)->getType()->isStructTy())
-    return;
-
-  if (TrackedGlobals.empty() || !isa<GlobalVariable>(SI.getOperand(1)))
-    return;
-
-  GlobalVariable *GV = cast<GlobalVariable>(SI.getOperand(1));
-  auto I = TrackedGlobals.find(GV);
-  if (I == TrackedGlobals.end())
-    return;
-
-  // Get the value we are storing into the global, then merge it.
-  mergeInValue(I->second, GV, getValueState(SI.getOperand(0)),
-               ValueLatticeElement::MergeOptions().setCheckWiden(false));
-  if (I->second.isOverdefined())
-    TrackedGlobals.erase(I);      // No need to keep tracking this!
-}
-
-static ValueLatticeElement getValueFromMetadata(const Instruction *I) {
-  if (MDNode *Ranges = I->getMetadata(LLVMContext::MD_range))
-    if (I->getType()->isIntegerTy())
-      return ValueLatticeElement::getRange(
-          getConstantRangeFromMetadata(*Ranges));
-  if (I->hasMetadata(LLVMContext::MD_nonnull))
-    return ValueLatticeElement::getNot(
-        ConstantPointerNull::get(cast<PointerType>(I->getType())));
-  return ValueLatticeElement::getOverdefined();
-}
-
-// Handle load instructions.  If the operand is a constant pointer to a constant
-// global, we can replace the load with the loaded constant value!
-void SCCPSolver::visitLoadInst(LoadInst &I) {
-  // If this load is of a struct or the load is volatile, just mark the result
-  // as overdefined.
-  if (I.getType()->isStructTy() || I.isVolatile())
-    return (void)markOverdefined(&I);
-
-  // ResolvedUndefsIn might mark I as overdefined. Bail out, even if we would
-  // discover a concrete value later.
-  if (ValueState[&I].isOverdefined())
-    return (void)markOverdefined(&I);
-
-  ValueLatticeElement PtrVal = getValueState(I.getOperand(0));
-  if (PtrVal.isUnknownOrUndef())
-    return; // The pointer is not resolved yet!
-
-  ValueLatticeElement &IV = ValueState[&I];
-
-  if (isConstant(PtrVal)) {
-    Constant *Ptr = getConstant(PtrVal);
-
-    // load null is undefined.
-    if (isa<ConstantPointerNull>(Ptr)) {
-      if (NullPointerIsDefined(I.getFunction(), I.getPointerAddressSpace()))
-        return (void)markOverdefined(IV, &I);
-      else
-        return;
-    }
-
-    // Transform load (constant global) into the value loaded.
-    if (auto *GV = dyn_cast<GlobalVariable>(Ptr)) {
-      if (!TrackedGlobals.empty()) {
-        // If we are tracking this global, merge in the known value for it.
-        auto It = TrackedGlobals.find(GV);
-        if (It != TrackedGlobals.end()) {
-          mergeInValue(IV, &I, It->second, getMaxWidenStepsOpts());
-          return;
-        }
-      }
-    }
-
-    // Transform load from a constant into a constant if possible.
-    if (Constant *C = ConstantFoldLoadFromConstPtr(Ptr, I.getType(), DL)) {
-      if (isa<UndefValue>(C))
-        return;
-      return (void)markConstant(IV, &I, C);
-    }
-  }
-
-  // Fall back to metadata.
-  mergeInValue(&I, getValueFromMetadata(&I));
-}
-
-void SCCPSolver::visitCallBase(CallBase &CB) {
-  handleCallResult(CB);
-  handleCallArguments(CB);
-}
-
-void SCCPSolver::handleCallOverdefined(CallBase &CB) {
-  Function *F = CB.getCalledFunction();
-
-  // Void return and not tracking callee, just bail.
-  if (CB.getType()->isVoidTy())
-    return;
-
-  // Always mark struct return as overdefined.
-  if (CB.getType()->isStructTy())
-    return (void)markOverdefined(&CB);
-
-  // Otherwise, if we have a single return value case, and if the function is
-  // a declaration, maybe we can constant fold it.
-  if (F && F->isDeclaration() && canConstantFoldCallTo(&CB, F)) {
-    SmallVector<Constant *, 8> Operands;
-    for (auto AI = CB.arg_begin(), E = CB.arg_end(); AI != E; ++AI) {
-      if (AI->get()->getType()->isStructTy())
-        return markOverdefined(&CB); // Can't handle struct args.
-      ValueLatticeElement State = getValueState(*AI);
-
-      if (State.isUnknownOrUndef())
-        return; // Operands are not resolved yet.
-      if (isOverdefined(State))
-        return (void)markOverdefined(&CB);
-      assert(isConstant(State) && "Unknown state!");
-      Operands.push_back(getConstant(State));
-    }
-
-    if (isOverdefined(getValueState(&CB)))
-      return (void)markOverdefined(&CB);
-
-    // If we can constant fold this, mark the result of the call as a
-    // constant.
-    if (Constant *C = ConstantFoldCall(&CB, F, Operands, &GetTLI(*F))) {
-      // call -> undef.
-      if (isa<UndefValue>(C))
-        return;
-      return (void)markConstant(&CB, C);
-    }
-  }
-
-  // Fall back to metadata.
-  mergeInValue(&CB, getValueFromMetadata(&CB));
-}
-
-void SCCPSolver::handleCallArguments(CallBase &CB) {
-  Function *F = CB.getCalledFunction();
-  // If this is a local function that doesn't have its address taken, mark its
-  // entry block executable and merge in the actual arguments to the call into
-  // the formal arguments of the function.
-  if (!TrackingIncomingArguments.empty() &&
-      TrackingIncomingArguments.count(F)) {
-    MarkBlockExecutable(&F->front());
-
-    // Propagate information from this call site into the callee.
-    auto CAI = CB.arg_begin();
-    for (Function::arg_iterator AI = F->arg_begin(), E = F->arg_end(); AI != E;
-         ++AI, ++CAI) {
-      // If this argument is byval, and if the function is not readonly, there
-      // will be an implicit copy formed of the input aggregate.
-      if (AI->hasByValAttr() && !F->onlyReadsMemory()) {
-        markOverdefined(&*AI);
-        continue;
-      }
-
-      if (auto *STy = dyn_cast<StructType>(AI->getType())) {
-        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-          ValueLatticeElement CallArg = getStructValueState(*CAI, i);
-          mergeInValue(getStructValueState(&*AI, i), &*AI, CallArg,
-                       getMaxWidenStepsOpts());
-        }
-      } else
-        mergeInValue(&*AI, getValueState(*CAI), getMaxWidenStepsOpts());
-    }
-  }
-}
-
-void SCCPSolver::handleCallResult(CallBase &CB) {
-  Function *F = CB.getCalledFunction();
-
-  if (auto *II = dyn_cast<IntrinsicInst>(&CB)) {
-    if (II->getIntrinsicID() == Intrinsic::ssa_copy) {
-      if (ValueState[&CB].isOverdefined())
-        return;
-
-      Value *CopyOf = CB.getOperand(0);
-      ValueLatticeElement CopyOfVal = getValueState(CopyOf);
-      auto *PI = getPredicateInfoFor(&CB);
-      assert(PI && "Missing predicate info for ssa.copy");
-
-      const Optional<PredicateConstraint> &Constraint = PI->getConstraint();
-      if (!Constraint) {
-        mergeInValue(ValueState[&CB], &CB, CopyOfVal);
-        return;
-      }
-
-      CmpInst::Predicate Pred = Constraint->Predicate;
-      Value *OtherOp = Constraint->OtherOp;
-
-      // Wait until OtherOp is resolved.
-      if (getValueState(OtherOp).isUnknown()) {
-        addAdditionalUser(OtherOp, &CB);
-        return;
-      }
-
-      // TODO: Actually filp MayIncludeUndef for the created range to false,
-      // once most places in the optimizer respect the branches on
-      // undef/poison are UB rule. The reason why the new range cannot be
-      // undef is as follows below:
-      // The new range is based on a branch condition. That guarantees that
-      // neither of the compare operands can be undef in the branch targets,
-      // unless we have conditions that are always true/false (e.g. icmp ule
-      // i32, %a, i32_max). For the latter overdefined/empty range will be
-      // inferred, but the branch will get folded accordingly anyways.
-      bool MayIncludeUndef = !isa<PredicateAssume>(PI);
-
-      ValueLatticeElement CondVal = getValueState(OtherOp);
-      ValueLatticeElement &IV = ValueState[&CB];
-      if (CondVal.isConstantRange() || CopyOfVal.isConstantRange()) {
-        auto ImposedCR =
-            ConstantRange::getFull(DL.getTypeSizeInBits(CopyOf->getType()));
-
-        // Get the range imposed by the condition.
-        if (CondVal.isConstantRange())
-          ImposedCR = ConstantRange::makeAllowedICmpRegion(
-              Pred, CondVal.getConstantRange());
-
-        // Combine range info for the original value with the new range from the
-        // condition.
-        auto CopyOfCR = CopyOfVal.isConstantRange()
-                            ? CopyOfVal.getConstantRange()
-                            : ConstantRange::getFull(
-                                  DL.getTypeSizeInBits(CopyOf->getType()));
-        auto NewCR = ImposedCR.intersectWith(CopyOfCR);
-        // If the existing information is != x, do not use the information from
-        // a chained predicate, as the != x information is more likely to be
-        // helpful in practice.
-        if (!CopyOfCR.contains(NewCR) && CopyOfCR.getSingleMissingElement())
-          NewCR = CopyOfCR;
-
-        addAdditionalUser(OtherOp, &CB);
-        mergeInValue(
-            IV, &CB,
-            ValueLatticeElement::getRange(NewCR, MayIncludeUndef));
-        return;
-      } else if (Pred == CmpInst::ICMP_EQ && CondVal.isConstant()) {
-        // For non-integer values or integer constant expressions, only
-        // propagate equal constants.
-        addAdditionalUser(OtherOp, &CB);
-        mergeInValue(IV, &CB, CondVal);
-        return;
-      } else if (Pred == CmpInst::ICMP_NE && CondVal.isConstant() &&
-                 !MayIncludeUndef) {
-        // Propagate inequalities.
-        addAdditionalUser(OtherOp, &CB);
-        mergeInValue(IV, &CB,
-                     ValueLatticeElement::getNot(CondVal.getConstant()));
-        return;
-      }
-
-      return (void)mergeInValue(IV, &CB, CopyOfVal);
-    }
-
-    if (ConstantRange::isIntrinsicSupported(II->getIntrinsicID())) {
-      // Compute result range for intrinsics supported by ConstantRange.
-      // Do this even if we don't know a range for all operands, as we may
-      // still know something about the result range, e.g. of abs(x).
-      SmallVector<ConstantRange, 2> OpRanges;
-      for (Value *Op : II->args()) {
-        const ValueLatticeElement &State = getValueState(Op);
-        if (State.isConstantRange())
-          OpRanges.push_back(State.getConstantRange());
-        else
-          OpRanges.push_back(
-              ConstantRange::getFull(Op->getType()->getScalarSizeInBits()));
-      }
-
-      ConstantRange Result =
-          ConstantRange::intrinsic(II->getIntrinsicID(), OpRanges);
-      return (void)mergeInValue(II, ValueLatticeElement::getRange(Result));
-    }
-  }
-
-  // The common case is that we aren't tracking the callee, either because we
-  // are not doing interprocedural analysis or the callee is indirect, or is
-  // external.  Handle these cases first.
-  if (!F || F->isDeclaration())
-    return handleCallOverdefined(CB);
-
-  // If this is a single/zero retval case, see if we're tracking the function.
-  if (auto *STy = dyn_cast<StructType>(F->getReturnType())) {
-    if (!MRVFunctionsTracked.count(F))
-      return handleCallOverdefined(CB); // Not tracking this callee.
-
-    // If we are tracking this callee, propagate the result of the function
-    // into this call site.
-    for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i)
-      mergeInValue(getStructValueState(&CB, i), &CB,
-                   TrackedMultipleRetVals[std::make_pair(F, i)],
-                   getMaxWidenStepsOpts());
-  } else {
-    auto TFRVI = TrackedRetVals.find(F);
-    if (TFRVI == TrackedRetVals.end())
-      return handleCallOverdefined(CB); // Not tracking this callee.
-
-    // If so, propagate the return value of the callee into this call result.
-    mergeInValue(&CB, TFRVI->second, getMaxWidenStepsOpts());
-  }
-}
-
-void SCCPSolver::Solve() {
-  // Process the work lists until they are empty!
-  while (!BBWorkList.empty() || !InstWorkList.empty() ||
-         !OverdefinedInstWorkList.empty()) {
-    // Process the overdefined instruction's work list first, which drives other
-    // things to overdefined more quickly.
-    while (!OverdefinedInstWorkList.empty()) {
-      Value *I = OverdefinedInstWorkList.pop_back_val();
-
-      LLVM_DEBUG(dbgs() << "\nPopped off OI-WL: " << *I << '\n');
-
-      // "I" got into the work list because it either made the transition from
-      // bottom to constant, or to overdefined.
-      //
-      // Anything on this worklist that is overdefined need not be visited
-      // since all of its users will have already been marked as overdefined
-      // Update all of the users of this instruction's value.
-      //
-      markUsersAsChanged(I);
-    }
-
-    // Process the instruction work list.
-    while (!InstWorkList.empty()) {
-      Value *I = InstWorkList.pop_back_val();
-
-      LLVM_DEBUG(dbgs() << "\nPopped off I-WL: " << *I << '\n');
-
-      // "I" got into the work list because it made the transition from undef to
-      // constant.
-      //
-      // Anything on this worklist that is overdefined need not be visited
-      // since all of its users will have already been marked as overdefined.
-      // Update all of the users of this instruction's value.
-      //
-      if (I->getType()->isStructTy() || !getValueState(I).isOverdefined())
-        markUsersAsChanged(I);
-    }
-
-    // Process the basic block work list.
-    while (!BBWorkList.empty()) {
-      BasicBlock *BB = BBWorkList.pop_back_val();
-
-      LLVM_DEBUG(dbgs() << "\nPopped off BBWL: " << *BB << '\n');
-
-      // Notify all instructions in this basic block that they are newly
-      // executable.
-      visit(BB);
-    }
-  }
-}
-
-/// ResolvedUndefsIn - While solving the dataflow for a function, we assume
-/// that branches on undef values cannot reach any of their successors.
-/// However, this is not a safe assumption.  After we solve dataflow, this
-/// method should be use to handle this.  If this returns true, the solver
-/// should be rerun.
-///
-/// This method handles this by finding an unresolved branch and marking it one
-/// of the edges from the block as being feasible, even though the condition
-/// doesn't say it would otherwise be.  This allows SCCP to find the rest of the
-/// CFG and only slightly pessimizes the analysis results (by marking one,
-/// potentially infeasible, edge feasible).  This cannot usefully modify the
-/// constraints on the condition of the branch, as that would impact other users
-/// of the value.
-///
-/// This scan also checks for values that use undefs. It conservatively marks
-/// them as overdefined.
-bool SCCPSolver::ResolvedUndefsIn(Function &F) {
-  bool MadeChange = false;
-  for (BasicBlock &BB : F) {
-    if (!BBExecutable.count(&BB))
-      continue;
-
-    for (Instruction &I : BB) {
-      // Look for instructions which produce undef values.
-      if (I.getType()->isVoidTy()) continue;
-
-      if (auto *STy = dyn_cast<StructType>(I.getType())) {
-        // Only a few things that can be structs matter for undef.
-
-        // Tracked calls must never be marked overdefined in ResolvedUndefsIn.
-        if (auto *CB = dyn_cast<CallBase>(&I))
-          if (Function *F = CB->getCalledFunction())
-            if (MRVFunctionsTracked.count(F))
-              continue;
-
-        // extractvalue and insertvalue don't need to be marked; they are
-        // tracked as precisely as their operands.
-        if (isa<ExtractValueInst>(I) || isa<InsertValueInst>(I))
-          continue;
-        // Send the results of everything else to overdefined.  We could be
-        // more precise than this but it isn't worth bothering.
-        for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-          ValueLatticeElement &LV = getStructValueState(&I, i);
-          if (LV.isUnknownOrUndef()) {
-            markOverdefined(LV, &I);
-            MadeChange = true;
-          }
-        }
-        continue;
-      }
-
-      ValueLatticeElement &LV = getValueState(&I);
-      if (!LV.isUnknownOrUndef())
-        continue;
-
-      // There are two reasons a call can have an undef result
-      // 1. It could be tracked.
-      // 2. It could be constant-foldable.
-      // Because of the way we solve return values, tracked calls must
-      // never be marked overdefined in ResolvedUndefsIn.
-      if (auto *CB = dyn_cast<CallBase>(&I))
-        if (Function *F = CB->getCalledFunction())
-          if (TrackedRetVals.count(F))
-            continue;
-
-      if (isa<LoadInst>(I)) {
-        // A load here means one of two things: a load of undef from a global,
-        // a load from an unknown pointer.  Either way, having it return undef
-        // is okay.
-        continue;
-      }
-
-      markOverdefined(&I);
-      MadeChange = true;
-    }
-
-    // Check to see if we have a branch or switch on an undefined value.  If so
-    // we force the branch to go one way or the other to make the successor
-    // values live.  It doesn't really matter which way we force it.
-    Instruction *TI = BB.getTerminator();
-    if (auto *BI = dyn_cast<BranchInst>(TI)) {
-      if (!BI->isConditional()) continue;
-      if (!getValueState(BI->getCondition()).isUnknownOrUndef())
-        continue;
-
-      // If the input to SCCP is actually branch on undef, fix the undef to
-      // false.
-      if (isa<UndefValue>(BI->getCondition())) {
-        BI->setCondition(ConstantInt::getFalse(BI->getContext()));
-        markEdgeExecutable(&BB, TI->getSuccessor(1));
-        MadeChange = true;
-        continue;
-      }
-
-      // Otherwise, it is a branch on a symbolic value which is currently
-      // considered to be undef.  Make sure some edge is executable, so a
-      // branch on "undef" always flows somewhere.
-      // FIXME: Distinguish between dead code and an LLVM "undef" value.
-      BasicBlock *DefaultSuccessor = TI->getSuccessor(1);
-      if (markEdgeExecutable(&BB, DefaultSuccessor))
-        MadeChange = true;
-
-      continue;
-    }
-
-   if (auto *IBR = dyn_cast<IndirectBrInst>(TI)) {
-      // Indirect branch with no successor ?. Its ok to assume it branches
-      // to no target.
-      if (IBR->getNumSuccessors() < 1)
-        continue;
-
-      if (!getValueState(IBR->getAddress()).isUnknownOrUndef())
-        continue;
-
-      // If the input to SCCP is actually branch on undef, fix the undef to
-      // the first successor of the indirect branch.
-      if (isa<UndefValue>(IBR->getAddress())) {
-        IBR->setAddress(BlockAddress::get(IBR->getSuccessor(0)));
-        markEdgeExecutable(&BB, IBR->getSuccessor(0));
-        MadeChange = true;
-        continue;
-      }
-
-      // Otherwise, it is a branch on a symbolic value which is currently
-      // considered to be undef.  Make sure some edge is executable, so a
-      // branch on "undef" always flows somewhere.
-      // FIXME: IndirectBr on "undef" doesn't actually need to go anywhere:
-      // we can assume the branch has undefined behavior instead.
-      BasicBlock *DefaultSuccessor = IBR->getSuccessor(0);
-      if (markEdgeExecutable(&BB, DefaultSuccessor))
-        MadeChange = true;
-
-      continue;
-    }
-
-    if (auto *SI = dyn_cast<SwitchInst>(TI)) {
-      if (!SI->getNumCases() ||
-          !getValueState(SI->getCondition()).isUnknownOrUndef())
-        continue;
-
-      // If the input to SCCP is actually switch on undef, fix the undef to
-      // the first constant.
-      if (isa<UndefValue>(SI->getCondition())) {
-        SI->setCondition(SI->case_begin()->getCaseValue());
-        markEdgeExecutable(&BB, SI->case_begin()->getCaseSuccessor());
-        MadeChange = true;
-        continue;
-      }
-
-      // Otherwise, it is a branch on a symbolic value which is currently
-      // considered to be undef.  Make sure some edge is executable, so a
-      // branch on "undef" always flows somewhere.
-      // FIXME: Distinguish between dead code and an LLVM "undef" value.
-      BasicBlock *DefaultSuccessor = SI->case_begin()->getCaseSuccessor();
-      if (markEdgeExecutable(&BB, DefaultSuccessor))
-        MadeChange = true;
-
-      continue;
-    }
-  }
-
-  return MadeChange;
-}
-
 static bool tryToReplaceWithConstant(SCCPSolver &Solver, Value *V) {
   Constant *Const = nullptr;
   if (V->getType()->isStructTy()) {
@@ -1650,16 +124,19 @@ static bool tryToReplaceWithConstant(SCCPSolver &Solver, Value *V) {
   assert(Const && "Constant is nullptr here!");
 
   // Replacing `musttail` instructions with constant breaks `musttail` invariant
-  // unless the call itself can be removed
-  CallInst *CI = dyn_cast<CallInst>(V);
-  if (CI && CI->isMustTailCall() && !CI->isSafeToRemove()) {
-    Function *F = CI->getCalledFunction();
+  // unless the call itself can be removed.
+  // Calls with "clang.arc.attachedcall" implicitly use the return value and
+  // those uses cannot be updated with a constant.
+  CallBase *CB = dyn_cast<CallBase>(V);
+  if (CB && ((CB->isMustTailCall() && !CB->isSafeToRemove()) ||
+             CB->getOperandBundle(LLVMContext::OB_clang_arc_attachedcall))) {
+    Function *F = CB->getCalledFunction();
 
     // Don't zap returns of the callee
     if (F)
-      Solver.AddMustTailCallee(F);
+      Solver.addToMustPreserveReturnsInFunctions(F);
 
-    LLVM_DEBUG(dbgs() << "  Can\'t treat the result of musttail call : " << *CI
+    LLVM_DEBUG(dbgs() << "  Can\'t treat the result of call " << *CB
                       << " as a constant\n");
     return false;
   }
@@ -1682,7 +159,7 @@ static bool simplifyInstsInBlock(SCCPSolver &Solver, BasicBlock &BB,
     if (tryToReplaceWithConstant(Solver, &Inst)) {
       if (Inst.isSafeToRemove())
         Inst.eraseFromParent();
-      // Hey, we just changed something!
+
       MadeChanges = true;
       ++InstRemovedStat;
     } else if (isa<SExtInst>(&Inst)) {
@@ -1716,7 +193,7 @@ static bool runSCCP(Function &F, const DataLayout &DL,
       F.getContext());
 
   // Mark the first block of the function as being executable.
-  Solver.MarkBlockExecutable(&F.front());
+  Solver.markBlockExecutable(&F.front());
 
   // Mark all arguments to the function as being overdefined.
   for (Argument &AI : F.args())
@@ -1725,9 +202,9 @@ static bool runSCCP(Function &F, const DataLayout &DL,
   // Solve for constants.
   bool ResolvedUndefs = true;
   while (ResolvedUndefs) {
-    Solver.Solve();
+    Solver.solve();
     LLVM_DEBUG(dbgs() << "RESOLVING UNDEFs\n");
-    ResolvedUndefs = Solver.ResolvedUndefsIn(F);
+    ResolvedUndefs = Solver.resolvedUndefsIn(F);
   }
 
   bool MadeChanges = false;
@@ -1762,7 +239,6 @@ PreservedAnalyses SCCPPass::run(Function &F, FunctionAnalysisManager &AM) {
     return PreservedAnalyses::all();
 
   auto PA = PreservedAnalyses();
-  PA.preserve<GlobalsAA>();
   PA.preserveSet<CFGAnalyses>();
   return PA;
 }
@@ -1821,11 +297,12 @@ static void findReturnsToZap(Function &F,
   if (!Solver.isArgumentTrackedFunction(&F))
     return;
 
-  // There is a non-removable musttail call site of this function. Zapping
-  // returns is not allowed.
-  if (Solver.isMustTailCallee(&F)) {
-    LLVM_DEBUG(dbgs() << "Can't zap returns of the function : " << F.getName()
-                      << " due to present musttail call of it\n");
+  if (Solver.mustPreserveReturn(&F)) {
+    LLVM_DEBUG(
+        dbgs()
+        << "Can't zap returns of the function : " << F.getName()
+        << " due to present musttail or \"clang.arc.attachedcall\" call of "
+           "it\n");
     return;
   }
 
@@ -1946,17 +423,17 @@ bool llvm::runIPSCCP(
     // Determine if we can track the function's return values. If so, add the
     // function to the solver's set of return-tracked functions.
     if (canTrackReturnsInterprocedurally(&F))
-      Solver.AddTrackedFunction(&F);
+      Solver.addTrackedFunction(&F);
 
     // Determine if we can track the function's arguments. If so, add the
     // function to the solver's set of argument-tracked functions.
     if (canTrackArgumentsInterprocedurally(&F)) {
-      Solver.AddArgumentTrackedFunction(&F);
+      Solver.addArgumentTrackedFunction(&F);
       continue;
     }
 
     // Assume the function is called.
-    Solver.MarkBlockExecutable(&F.front());
+    Solver.markBlockExecutable(&F.front());
 
     // Assume nothing about the incoming arguments.
     for (Argument &AI : F.args())
@@ -1969,21 +446,21 @@ bool llvm::runIPSCCP(
   for (GlobalVariable &G : M.globals()) {
     G.removeDeadConstantUsers();
     if (canTrackGlobalVariableInterprocedurally(&G))
-      Solver.TrackValueOfGlobalVariable(&G);
+      Solver.trackValueOfGlobalVariable(&G);
   }
 
   // Solve for constants.
   bool ResolvedUndefs = true;
-  Solver.Solve();
+  Solver.solve();
   while (ResolvedUndefs) {
     LLVM_DEBUG(dbgs() << "RESOLVING UNDEFS\n");
     ResolvedUndefs = false;
     for (Function &F : M) {
-      if (Solver.ResolvedUndefsIn(F))
+      if (Solver.resolvedUndefsIn(F))
         ResolvedUndefs = true;
     }
     if (ResolvedUndefs)
-      Solver.Solve();
+      Solver.solve();
   }
 
   bool MadeChanges = false;
@@ -2049,13 +526,11 @@ bool llvm::runIPSCCP(
     // nodes in executable blocks we found values for. The function's entry
     // block is not part of BlocksToErase, so we have to handle it separately.
     for (BasicBlock *BB : BlocksToErase) {
-      NumInstRemoved +=
-          changeToUnreachable(BB->getFirstNonPHI(), /*UseLLVMTrap=*/false,
-                              /*PreserveLCSSA=*/false, &DTU);
+      NumInstRemoved += changeToUnreachable(BB->getFirstNonPHI(),
+                                            /*PreserveLCSSA=*/false, &DTU);
     }
     if (!Solver.isBlockExecutable(&F.front()))
       NumInstRemoved += changeToUnreachable(F.front().getFirstNonPHI(),
-                                            /*UseLLVMTrap=*/false,
                                             /*PreserveLCSSA=*/false, &DTU);
 
     for (BasicBlock &BB : F)
diff --git a/llvm/lib/Transforms/Scalar/SROA.cpp b/llvm/lib/Transforms/Scalar/SROA.cpp
index d111a6ba4241..5ec01454e5b2 100644
--- a/llvm/lib/Transforms/Scalar/SROA.cpp
+++ b/llvm/lib/Transforms/Scalar/SROA.cpp
@@ -768,7 +768,8 @@ private:
     // We allow splitting of non-volatile loads and stores where the type is an
     // integer type. These may be used to implement 'memcpy' or other "transfer
     // of bits" patterns.
-    bool IsSplittable = Ty->isIntegerTy() && !IsVolatile;
+    bool IsSplittable =
+        Ty->isIntegerTy() && !IsVolatile && DL.typeSizeEqualsStoreSize(Ty);
 
     insertUse(I, Offset, Size, IsSplittable);
   }
@@ -926,7 +927,8 @@ private:
            "Map index doesn't point back to a slice with this user.");
   }
 
-  // Disable SRoA for any intrinsics except for lifetime invariants.
+  // Disable SRoA for any intrinsics except for lifetime invariants and
+  // invariant group.
   // FIXME: What about debug intrinsics? This matches old behavior, but
   // doesn't make sense.
   void visitIntrinsicInst(IntrinsicInst &II) {
@@ -946,6 +948,11 @@ private:
       return;
     }
 
+    if (II.isLaunderOrStripInvariantGroup()) {
+      enqueueUsers(II);
+      return;
+    }
+
     Base::visitIntrinsicInst(II);
   }
 
@@ -2461,14 +2468,17 @@ private:
       Pass.DeadInsts.push_back(I);
   }
 
-  Value *rewriteVectorizedLoadInst() {
+  Value *rewriteVectorizedLoadInst(LoadInst &LI) {
     unsigned BeginIndex = getIndex(NewBeginOffset);
     unsigned EndIndex = getIndex(NewEndOffset);
     assert(EndIndex > BeginIndex && "Empty vector!");
 
-    Value *V = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
-                                     NewAI.getAlign(), "load");
-    return extractVector(IRB, V, BeginIndex, EndIndex, "vec");
+    LoadInst *Load = IRB.CreateAlignedLoad(NewAI.getAllocatedType(), &NewAI,
+                                           NewAI.getAlign(), "load");
+
+    Load->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
+                            LLVMContext::MD_access_group});
+    return extractVector(IRB, Load, BeginIndex, EndIndex, "vec");
   }
 
   Value *rewriteIntegerLoad(LoadInst &LI) {
@@ -2512,7 +2522,7 @@ private:
     bool IsPtrAdjusted = false;
     Value *V;
     if (VecTy) {
-      V = rewriteVectorizedLoadInst();
+      V = rewriteVectorizedLoadInst(LI);
     } else if (IntTy && LI.getType()->isIntegerTy()) {
       V = rewriteIntegerLoad(LI);
     } else if (NewBeginOffset == NewAllocaBeginOffset &&
@@ -2524,7 +2534,7 @@ private:
                                               NewAI.getAlign(), LI.isVolatile(),
                                               LI.getName());
       if (AATags)
-        NewLI->setAAMetadata(AATags);
+        NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
       if (LI.isVolatile())
         NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
       if (NewLI->isAtomic())
@@ -2563,9 +2573,11 @@ private:
           IRB.CreateAlignedLoad(TargetTy, getNewAllocaSlicePtr(IRB, LTy),
                                 getSliceAlign(), LI.isVolatile(), LI.getName());
       if (AATags)
-        NewLI->setAAMetadata(AATags);
+        NewLI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
       if (LI.isVolatile())
         NewLI->setAtomic(LI.getOrdering(), LI.getSyncScopeID());
+      NewLI->copyMetadata(LI, {LLVMContext::MD_mem_parallel_loop_access,
+                               LLVMContext::MD_access_group});
 
       V = NewLI;
       IsPtrAdjusted = true;
@@ -2625,8 +2637,10 @@ private:
       V = insertVector(IRB, Old, V, BeginIndex, "vec");
     }
     StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign());
+    Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
+                             LLVMContext::MD_access_group});
     if (AATags)
-      Store->setAAMetadata(AATags);
+      Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
     Pass.DeadInsts.push_back(&SI);
 
     LLVM_DEBUG(dbgs() << "          to: " << *Store << "\n");
@@ -2650,7 +2664,7 @@ private:
     Store->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
                              LLVMContext::MD_access_group});
     if (AATags)
-      Store->setAAMetadata(AATags);
+      Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
     Pass.DeadInsts.push_back(&SI);
     LLVM_DEBUG(dbgs() << "          to: " << *Store << "\n");
     return true;
@@ -2720,7 +2734,7 @@ private:
     NewSI->copyMetadata(SI, {LLVMContext::MD_mem_parallel_loop_access,
                              LLVMContext::MD_access_group});
     if (AATags)
-      NewSI->setAAMetadata(AATags);
+      NewSI->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
     if (SI.isVolatile())
       NewSI->setAtomic(SI.getOrdering(), SI.getSyncScopeID());
     if (NewSI->isAtomic())
@@ -2775,7 +2789,7 @@ private:
 
     // If the memset has a variable size, it cannot be split, just adjust the
     // pointer to the new alloca.
-    if (!isa<Constant>(II.getLength())) {
+    if (!isa<ConstantInt>(II.getLength())) {
       assert(!IsSplit);
       assert(NewBeginOffset == BeginOffset);
       II.setDest(getNewAllocaSlicePtr(IRB, OldPtr->getType()));
@@ -2816,7 +2830,7 @@ private:
           getNewAllocaSlicePtr(IRB, OldPtr->getType()), II.getValue(), Size,
           MaybeAlign(getSliceAlign()), II.isVolatile());
       if (AATags)
-        New->setAAMetadata(AATags);
+        New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
       LLVM_DEBUG(dbgs() << "          to: " << *New << "\n");
       return false;
     }
@@ -2884,8 +2898,10 @@ private:
 
     StoreInst *New =
         IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlign(), II.isVolatile());
+    New->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
+                           LLVMContext::MD_access_group});
     if (AATags)
-      New->setAAMetadata(AATags);
+      New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
     LLVM_DEBUG(dbgs() << "          to: " << *New << "\n");
     return !II.isVolatile();
   }
@@ -3006,7 +3022,7 @@ private:
       CallInst *New = IRB.CreateMemCpy(DestPtr, DestAlign, SrcPtr, SrcAlign,
                                        Size, II.isVolatile());
       if (AATags)
-        New->setAAMetadata(AATags);
+        New->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
       LLVM_DEBUG(dbgs() << "          to: " << *New << "\n");
       return false;
     }
@@ -3059,8 +3075,10 @@ private:
     } else {
       LoadInst *Load = IRB.CreateAlignedLoad(OtherTy, SrcPtr, SrcAlign,
                                              II.isVolatile(), "copyload");
+      Load->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
+                              LLVMContext::MD_access_group});
       if (AATags)
-        Load->setAAMetadata(AATags);
+        Load->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
       Src = Load;
     }
 
@@ -3079,8 +3097,10 @@ private:
 
     StoreInst *Store = cast<StoreInst>(
         IRB.CreateAlignedStore(Src, DstPtr, DstAlign, II.isVolatile()));
+    Store->copyMetadata(II, {LLVMContext::MD_mem_parallel_loop_access,
+                             LLVMContext::MD_access_group});
     if (AATags)
-      Store->setAAMetadata(AATags);
+      Store->setAAMetadata(AATags.shift(NewBeginOffset - BeginOffset));
     LLVM_DEBUG(dbgs() << "          to: " << *Store << "\n");
     return !II.isVolatile();
   }
@@ -3381,8 +3401,13 @@ private:
           IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
       LoadInst *Load =
           IRB.CreateAlignedLoad(Ty, GEP, Alignment, Name + ".load");
-      if (AATags)
-        Load->setAAMetadata(AATags);
+
+      APInt Offset(
+          DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
+      if (AATags &&
+          GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset))
+        Load->setAAMetadata(AATags.shift(Offset.getZExtValue()));
+
       Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert");
       LLVM_DEBUG(dbgs() << "          to: " << *Load << "\n");
     }
@@ -3428,8 +3453,13 @@ private:
           IRB.CreateInBoundsGEP(BaseTy, Ptr, GEPIndices, Name + ".gep");
       StoreInst *Store =
           IRB.CreateAlignedStore(ExtractValue, InBoundsGEP, Alignment);
-      if (AATags)
-        Store->setAAMetadata(AATags);
+
+      APInt Offset(
+          DL.getIndexSizeInBits(Ptr->getType()->getPointerAddressSpace()), 0);
+      if (AATags &&
+          GEPOperator::accumulateConstantOffset(BaseTy, GEPIndices, DL, Offset))
+        Store->setAAMetadata(AATags.shift(Offset.getZExtValue()));
+
       LLVM_DEBUG(dbgs() << "          to: " << *Store << "\n");
     }
   };
@@ -3478,20 +3508,22 @@ private:
     SmallVector<Value *, 4> Index(GEPI.indices());
     bool IsInBounds = GEPI.isInBounds();
 
+    Type *Ty = GEPI.getSourceElementType();
     Value *True = Sel->getTrueValue();
     Value *NTrue =
         IsInBounds
-            ? Builder.CreateInBoundsGEP(True, Index,
+            ? Builder.CreateInBoundsGEP(Ty, True, Index,
                                         True->getName() + ".sroa.gep")
-            : Builder.CreateGEP(True, Index, True->getName() + ".sroa.gep");
+            : Builder.CreateGEP(Ty, True, Index, True->getName() + ".sroa.gep");
 
     Value *False = Sel->getFalseValue();
 
     Value *NFalse =
         IsInBounds
-            ? Builder.CreateInBoundsGEP(False, Index,
+            ? Builder.CreateInBoundsGEP(Ty, False, Index,
                                         False->getName() + ".sroa.gep")
-            : Builder.CreateGEP(False, Index, False->getName() + ".sroa.gep");
+            : Builder.CreateGEP(Ty, False, Index,
+                                False->getName() + ".sroa.gep");
 
     Value *NSel = Builder.CreateSelect(Sel->getCondition(), NTrue, NFalse,
                                        Sel->getName() + ".sroa.sel");
@@ -3545,9 +3577,10 @@ private:
         Instruction *In = cast<Instruction>(PHI->getIncomingValue(I));
 
         IRBuilderTy B(In->getParent(), std::next(In->getIterator()));
+        Type *Ty = GEPI.getSourceElementType();
         NewVal = IsInBounds
-            ? B.CreateInBoundsGEP(In, Index, In->getName() + ".sroa.gep")
-            : B.CreateGEP(In, Index, In->getName() + ".sroa.gep");
+            ? B.CreateInBoundsGEP(Ty, In, Index, In->getName() + ".sroa.gep")
+            : B.CreateGEP(Ty, In, Index, In->getName() + ".sroa.gep");
       }
       NewPN->addIncoming(NewVal, B);
     }
@@ -3970,6 +4003,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
     SplitLoads.clear();
 
     IntegerType *Ty = cast<IntegerType>(LI->getType());
+    assert(Ty->getBitWidth() % 8 == 0);
     uint64_t LoadSize = Ty->getBitWidth() / 8;
     assert(LoadSize > 0 && "Cannot have a zero-sized integer load!");
 
@@ -4056,7 +4090,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
                            PartPtrTy, StoreBasePtr->getName() + "."),
             getAdjustedAlignment(SI, PartOffset),
             /*IsVolatile*/ false);
-        PStore->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
+        PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access,
                                    LLVMContext::MD_access_group});
         LLVM_DEBUG(dbgs() << "      +" << PartOffset << ":" << *PStore << "\n");
       }
@@ -4094,6 +4128,7 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
   for (StoreInst *SI : Stores) {
     auto *LI = cast<LoadInst>(SI->getValueOperand());
     IntegerType *Ty = cast<IntegerType>(LI->getType());
+    assert(Ty->getBitWidth() % 8 == 0);
     uint64_t StoreSize = Ty->getBitWidth() / 8;
     assert(StoreSize > 0 && "Cannot have a zero-sized integer store!");
 
@@ -4141,6 +4176,8 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
                            LoadPartPtrTy, LoadBasePtr->getName() + "."),
             getAdjustedAlignment(LI, PartOffset),
             /*IsVolatile*/ false, LI->getName());
+        PLoad->copyMetadata(*LI, {LLVMContext::MD_mem_parallel_loop_access,
+                                  LLVMContext::MD_access_group});
       }
 
       // And store this partition.
@@ -4153,6 +4190,8 @@ bool SROA::presplitLoadsAndStores(AllocaInst &AI, AllocaSlices &AS) {
                          StorePartPtrTy, StoreBasePtr->getName() + "."),
           getAdjustedAlignment(SI, PartOffset),
           /*IsVolatile*/ false);
+      PStore->copyMetadata(*SI, {LLVMContext::MD_mem_parallel_loop_access,
+                                 LLVMContext::MD_access_group});
 
       // Now build a new slice for the alloca.
       NewSlices.push_back(
@@ -4761,7 +4800,6 @@ PreservedAnalyses SROA::runImpl(Function &F, DominatorTree &RunDT,
 
   PreservedAnalyses PA;
   PA.preserveSet<CFGAnalyses>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index dba3dba24e25..a041af0d70d0 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -60,6 +60,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeInferAddressSpacesPass(Registry);
   initializeInstSimplifyLegacyPassPass(Registry);
   initializeJumpThreadingPass(Registry);
+  initializeDFAJumpThreadingLegacyPassPass(Registry);
   initializeLegacyLICMPassPass(Registry);
   initializeLegacyLoopSinkPassPass(Registry);
   initializeLoopFuseLegacyPass(Registry);
diff --git a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
index afa2d1bc7966..ca288a533f46 100644
--- a/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
+++ b/llvm/lib/Transforms/Scalar/ScalarizeMaskedMemIntrin.cpp
@@ -15,11 +15,13 @@
 
 #include "llvm/Transforms/Scalar/ScalarizeMaskedMemIntrin.h"
 #include "llvm/ADT/Twine.h"
+#include "llvm/Analysis/DomTreeUpdater.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
 #include "llvm/IR/BasicBlock.h"
 #include "llvm/IR/Constant.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DerivedTypes.h"
+#include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstrTypes.h"
@@ -33,6 +35,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Support/Casting.h"
 #include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include <algorithm>
 #include <cassert>
 
@@ -59,16 +62,18 @@ public:
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<TargetTransformInfoWrapperPass>();
+    AU.addPreserved<DominatorTreeWrapperPass>();
   }
 };
 
 } // end anonymous namespace
 
 static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT,
-                          const TargetTransformInfo &TTI, const DataLayout &DL);
+                          const TargetTransformInfo &TTI, const DataLayout &DL,
+                          DomTreeUpdater *DTU);
 static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
                              const TargetTransformInfo &TTI,
-                             const DataLayout &DL);
+                             const DataLayout &DL, DomTreeUpdater *DTU);
 
 char ScalarizeMaskedMemIntrinLegacyPass::ID = 0;
 
@@ -76,6 +81,7 @@ INITIALIZE_PASS_BEGIN(ScalarizeMaskedMemIntrinLegacyPass, DEBUG_TYPE,
                       "Scalarize unsupported masked memory intrinsics", false,
                       false)
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
 INITIALIZE_PASS_END(ScalarizeMaskedMemIntrinLegacyPass, DEBUG_TYPE,
                     "Scalarize unsupported masked memory intrinsics", false,
                     false)
@@ -99,6 +105,11 @@ static bool isConstantIntVector(Value *Mask) {
   return true;
 }
 
+static unsigned adjustForEndian(const DataLayout &DL, unsigned VectorWidth,
+                                unsigned Idx) {
+  return DL.isBigEndian() ? VectorWidth - 1 - Idx : Idx;
+}
+
 // Translate a masked load intrinsic like
 // <16 x i32 > @llvm.masked.load( <16 x i32>* %addr, i32 align,
 //                               <16 x i1> %mask, <16 x i32> %passthru)
@@ -131,7 +142,8 @@ static bool isConstantIntVector(Value *Mask) {
 //  %10 = extractelement <16 x i1> %mask, i32 2
 //  br i1 %10, label %cond.load4, label %else5
 //
-static void scalarizeMaskedLoad(CallInst *CI, bool &ModifiedDT) {
+static void scalarizeMaskedLoad(const DataLayout &DL, CallInst *CI,
+                                DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Ptr = CI->getArgOperand(0);
   Value *Alignment = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
@@ -200,7 +212,8 @@ static void scalarizeMaskedLoad(CallInst *CI, bool &ModifiedDT) {
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -213,25 +226,26 @@ static void scalarizeMaskedLoad(CallInst *CI, bool &ModifiedDT) {
     //  %Elt = load i32* %EltAddr
     //  VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
     //
-    BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt->getIterator(),
-                                                     "cond.load");
-    Builder.SetInsertPoint(InsertPt);
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
+
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.load");
 
+    Builder.SetInsertPoint(CondBlock->getTerminator());
     Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, FirstEltPtr, Idx);
     LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AdjustedAlignVal);
     Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
 
     // Create "else" block, fill it in the next iteration
-    BasicBlock *NewIfBlock =
-        CondBlock->splitBasicBlock(InsertPt->getIterator(), "else");
-    Builder.SetInsertPoint(InsertPt);
-    Instruction *OldBr = IfBlock->getTerminator();
-    BranchInst::Create(CondBlock, NewIfBlock, Predicate, OldBr);
-    OldBr->eraseFromParent();
+    BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
+    NewIfBlock->setName("else");
     BasicBlock *PrevIfBlock = IfBlock;
     IfBlock = NewIfBlock;
 
     // Create the phi to join the new and previous value.
+    Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());
     PHINode *Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");
     Phi->addIncoming(NewVResult, CondBlock);
     Phi->addIncoming(VResult, PrevIfBlock);
@@ -270,7 +284,8 @@ static void scalarizeMaskedLoad(CallInst *CI, bool &ModifiedDT) {
 //   store i32 %6, i32* %7
 //   br label %else2
 //   . . .
-static void scalarizeMaskedStore(CallInst *CI, bool &ModifiedDT) {
+static void scalarizeMaskedStore(const DataLayout &DL, CallInst *CI,
+                                 DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptr = CI->getArgOperand(1);
   Value *Alignment = CI->getArgOperand(2);
@@ -283,7 +298,6 @@ static void scalarizeMaskedStore(CallInst *CI, bool &ModifiedDT) {
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
-  BasicBlock *IfBlock = CI->getParent();
   Builder.SetInsertPoint(InsertPt);
   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
 
@@ -332,7 +346,8 @@ static void scalarizeMaskedStore(CallInst *CI, bool &ModifiedDT) {
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -345,22 +360,23 @@ static void scalarizeMaskedStore(CallInst *CI, bool &ModifiedDT) {
     //  %EltAddr = getelementptr i32* %1, i32 0
     //  %store i32 %OneElt, i32* %EltAddr
     //
-    BasicBlock *CondBlock =
-        IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.store");
-    Builder.SetInsertPoint(InsertPt);
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
+
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.store");
 
+    Builder.SetInsertPoint(CondBlock->getTerminator());
     Value *OneElt = Builder.CreateExtractElement(Src, Idx);
     Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, FirstEltPtr, Idx);
     Builder.CreateAlignedStore(OneElt, Gep, AdjustedAlignVal);
 
     // Create "else" block, fill it in the next iteration
-    BasicBlock *NewIfBlock =
-        CondBlock->splitBasicBlock(InsertPt->getIterator(), "else");
-    Builder.SetInsertPoint(InsertPt);
-    Instruction *OldBr = IfBlock->getTerminator();
-    BranchInst::Create(CondBlock, NewIfBlock, Predicate, OldBr);
-    OldBr->eraseFromParent();
-    IfBlock = NewIfBlock;
+    BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
+    NewIfBlock->setName("else");
+
+    Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());
   }
   CI->eraseFromParent();
 
@@ -396,7 +412,8 @@ static void scalarizeMaskedStore(CallInst *CI, bool &ModifiedDT) {
 // . . .
 // %Result = select <16 x i1> %Mask, <16 x i32> %res.phi.select, <16 x i32> %Src
 // ret <16 x i32> %Result
-static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) {
+static void scalarizeMaskedGather(const DataLayout &DL, CallInst *CI,
+                                  DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Ptrs = CI->getArgOperand(0);
   Value *Alignment = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
@@ -451,7 +468,8 @@ static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) {
 
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -464,9 +482,14 @@ static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) {
     //  %Elt = load i32* %EltAddr
     //  VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
     //
-    BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.load");
-    Builder.SetInsertPoint(InsertPt);
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
 
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.load");
+
+    Builder.SetInsertPoint(CondBlock->getTerminator());
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
     LoadInst *Load =
         Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx));
@@ -474,14 +497,13 @@ static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) {
         Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx));
 
     // Create "else" block, fill it in the next iteration
-    BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else");
-    Builder.SetInsertPoint(InsertPt);
-    Instruction *OldBr = IfBlock->getTerminator();
-    BranchInst::Create(CondBlock, NewIfBlock, Predicate, OldBr);
-    OldBr->eraseFromParent();
+    BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
+    NewIfBlock->setName("else");
     BasicBlock *PrevIfBlock = IfBlock;
     IfBlock = NewIfBlock;
 
+    // Create the phi to join the new and previous value.
+    Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());
     PHINode *Phi = Builder.CreatePHI(VecType, 2, "res.phi.else");
     Phi->addIncoming(NewVResult, CondBlock);
     Phi->addIncoming(VResult, PrevIfBlock);
@@ -520,7 +542,8 @@ static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) {
 // store i32 %Elt1, i32* %Ptr1, align 4
 // br label %else2
 //   . . .
-static void scalarizeMaskedScatter(CallInst *CI, bool &ModifiedDT) {
+static void scalarizeMaskedScatter(const DataLayout &DL, CallInst *CI,
+                                   DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptrs = CI->getArgOperand(1);
   Value *Alignment = CI->getArgOperand(2);
@@ -535,7 +558,6 @@ static void scalarizeMaskedScatter(CallInst *CI, bool &ModifiedDT) {
 
   IRBuilder<> Builder(CI->getContext());
   Instruction *InsertPt = CI;
-  BasicBlock *IfBlock = CI->getParent();
   Builder.SetInsertPoint(InsertPt);
   Builder.SetCurrentDebugLocation(CI->getDebugLoc());
 
@@ -573,7 +595,8 @@ static void scalarizeMaskedScatter(CallInst *CI, bool &ModifiedDT) {
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -586,27 +609,31 @@ static void scalarizeMaskedScatter(CallInst *CI, bool &ModifiedDT) {
     //  %Ptr1 = extractelement <16 x i32*> %Ptrs, i32 1
     //  %store i32 %Elt1, i32* %Ptr1
     //
-    BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.store");
-    Builder.SetInsertPoint(InsertPt);
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
+
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.store");
 
+    Builder.SetInsertPoint(CondBlock->getTerminator());
     Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx));
     Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx));
     Builder.CreateAlignedStore(OneElt, Ptr, AlignVal);
 
     // Create "else" block, fill it in the next iteration
-    BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else");
-    Builder.SetInsertPoint(InsertPt);
-    Instruction *OldBr = IfBlock->getTerminator();
-    BranchInst::Create(CondBlock, NewIfBlock, Predicate, OldBr);
-    OldBr->eraseFromParent();
-    IfBlock = NewIfBlock;
+    BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
+    NewIfBlock->setName("else");
+
+    Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());
   }
   CI->eraseFromParent();
 
   ModifiedDT = true;
 }
 
-static void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) {
+static void scalarizeMaskedExpandLoad(const DataLayout &DL, CallInst *CI,
+                                      DomTreeUpdater *DTU, bool &ModifiedDT) {
   Value *Ptr = CI->getArgOperand(0);
   Value *Mask = CI->getArgOperand(1);
   Value *PassThru = CI->getArgOperand(2);
@@ -674,7 +701,8 @@ static void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) {
 
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -687,10 +715,14 @@ static void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) {
     //  %Elt = load i32* %EltAddr
     //  VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx
     //
-    BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt->getIterator(),
-                                                     "cond.load");
-    Builder.SetInsertPoint(InsertPt);
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
+
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.load");
 
+    Builder.SetInsertPoint(CondBlock->getTerminator());
     LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, Align(1));
     Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx);
 
@@ -700,16 +732,13 @@ static void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) {
       NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, 1);
 
     // Create "else" block, fill it in the next iteration
-    BasicBlock *NewIfBlock =
-        CondBlock->splitBasicBlock(InsertPt->getIterator(), "else");
-    Builder.SetInsertPoint(InsertPt);
-    Instruction *OldBr = IfBlock->getTerminator();
-    BranchInst::Create(CondBlock, NewIfBlock, Predicate, OldBr);
-    OldBr->eraseFromParent();
+    BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
+    NewIfBlock->setName("else");
     BasicBlock *PrevIfBlock = IfBlock;
     IfBlock = NewIfBlock;
 
     // Create the phi to join the new and previous value.
+    Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());
     PHINode *ResultPhi = Builder.CreatePHI(VecType, 2, "res.phi.else");
     ResultPhi->addIncoming(NewVResult, CondBlock);
     ResultPhi->addIncoming(VResult, PrevIfBlock);
@@ -730,7 +759,9 @@ static void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) {
   ModifiedDT = true;
 }
 
-static void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) {
+static void scalarizeMaskedCompressStore(const DataLayout &DL, CallInst *CI,
+                                         DomTreeUpdater *DTU,
+                                         bool &ModifiedDT) {
   Value *Src = CI->getArgOperand(0);
   Value *Ptr = CI->getArgOperand(1);
   Value *Mask = CI->getArgOperand(2);
@@ -780,7 +811,8 @@ static void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) {
     //
     Value *Predicate;
     if (VectorWidth != 1) {
-      Value *Mask = Builder.getInt(APInt::getOneBitSet(VectorWidth, Idx));
+      Value *Mask = Builder.getInt(APInt::getOneBitSet(
+          VectorWidth, adjustForEndian(DL, VectorWidth, Idx)));
       Predicate = Builder.CreateICmpNE(Builder.CreateAnd(SclrMask, Mask),
                                        Builder.getIntN(VectorWidth, 0));
     } else {
@@ -793,10 +825,14 @@ static void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) {
     //  %EltAddr = getelementptr i32* %1, i32 0
     //  %store i32 %OneElt, i32* %EltAddr
     //
-    BasicBlock *CondBlock =
-        IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.store");
-    Builder.SetInsertPoint(InsertPt);
+    Instruction *ThenTerm =
+        SplitBlockAndInsertIfThen(Predicate, InsertPt, /*Unreachable=*/false,
+                                  /*BranchWeights=*/nullptr, DTU);
 
+    BasicBlock *CondBlock = ThenTerm->getParent();
+    CondBlock->setName("cond.store");
+
+    Builder.SetInsertPoint(CondBlock->getTerminator());
     Value *OneElt = Builder.CreateExtractElement(Src, Idx);
     Builder.CreateAlignedStore(OneElt, Ptr, Align(1));
 
@@ -806,15 +842,13 @@ static void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) {
       NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, 1);
 
     // Create "else" block, fill it in the next iteration
-    BasicBlock *NewIfBlock =
-        CondBlock->splitBasicBlock(InsertPt->getIterator(), "else");
-    Builder.SetInsertPoint(InsertPt);
-    Instruction *OldBr = IfBlock->getTerminator();
-    BranchInst::Create(CondBlock, NewIfBlock, Predicate, OldBr);
-    OldBr->eraseFromParent();
+    BasicBlock *NewIfBlock = ThenTerm->getSuccessor(0);
+    NewIfBlock->setName("else");
     BasicBlock *PrevIfBlock = IfBlock;
     IfBlock = NewIfBlock;
 
+    Builder.SetInsertPoint(NewIfBlock, NewIfBlock->begin());
+
     // Add a PHI for the pointer if this isn't the last iteration.
     if ((Idx + 1) != VectorWidth) {
       PHINode *PtrPhi = Builder.CreatePHI(Ptr->getType(), 2, "ptr.phi.else");
@@ -828,7 +862,12 @@ static void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) {
   ModifiedDT = true;
 }
 
-static bool runImpl(Function &F, const TargetTransformInfo &TTI) {
+static bool runImpl(Function &F, const TargetTransformInfo &TTI,
+                    DominatorTree *DT) {
+  Optional<DomTreeUpdater> DTU;
+  if (DT)
+    DTU.emplace(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+
   bool EverMadeChange = false;
   bool MadeChange = true;
   auto &DL = F.getParent()->getDataLayout();
@@ -837,7 +876,9 @@ static bool runImpl(Function &F, const TargetTransformInfo &TTI) {
     for (Function::iterator I = F.begin(); I != F.end();) {
       BasicBlock *BB = &*I++;
       bool ModifiedDTOnIteration = false;
-      MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration, TTI, DL);
+      MadeChange |= optimizeBlock(*BB, ModifiedDTOnIteration, TTI, DL,
+                                  DTU.hasValue() ? DTU.getPointer() : nullptr);
+
 
       // Restart BB iteration if the dominator tree of the Function was changed
       if (ModifiedDTOnIteration)
@@ -851,28 +892,33 @@ static bool runImpl(Function &F, const TargetTransformInfo &TTI) {
 
 bool ScalarizeMaskedMemIntrinLegacyPass::runOnFunction(Function &F) {
   auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
-  return runImpl(F, TTI);
+  DominatorTree *DT = nullptr;
+  if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
+    DT = &DTWP->getDomTree();
+  return runImpl(F, TTI, DT);
 }
 
 PreservedAnalyses
 ScalarizeMaskedMemIntrinPass::run(Function &F, FunctionAnalysisManager &AM) {
   auto &TTI = AM.getResult<TargetIRAnalysis>(F);
-  if (!runImpl(F, TTI))
+  auto *DT = AM.getCachedResult<DominatorTreeAnalysis>(F);
+  if (!runImpl(F, TTI, DT))
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
   PA.preserve<TargetIRAnalysis>();
+  PA.preserve<DominatorTreeAnalysis>();
   return PA;
 }
 
 static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT,
-                          const TargetTransformInfo &TTI,
-                          const DataLayout &DL) {
+                          const TargetTransformInfo &TTI, const DataLayout &DL,
+                          DomTreeUpdater *DTU) {
   bool MadeChange = false;
 
   BasicBlock::iterator CurInstIterator = BB.begin();
   while (CurInstIterator != BB.end()) {
     if (CallInst *CI = dyn_cast<CallInst>(&*CurInstIterator++))
-      MadeChange |= optimizeCallInst(CI, ModifiedDT, TTI, DL);
+      MadeChange |= optimizeCallInst(CI, ModifiedDT, TTI, DL, DTU);
     if (ModifiedDT)
       return true;
   }
@@ -882,7 +928,7 @@ static bool optimizeBlock(BasicBlock &BB, bool &ModifiedDT,
 
 static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
                              const TargetTransformInfo &TTI,
-                             const DataLayout &DL) {
+                             const DataLayout &DL, DomTreeUpdater *DTU) {
   IntrinsicInst *II = dyn_cast<IntrinsicInst>(CI);
   if (II) {
     // The scalarization code below does not work for scalable vectors.
@@ -900,46 +946,46 @@ static bool optimizeCallInst(CallInst *CI, bool &ModifiedDT,
               CI->getType(),
               cast<ConstantInt>(CI->getArgOperand(1))->getAlignValue()))
         return false;
-      scalarizeMaskedLoad(CI, ModifiedDT);
+      scalarizeMaskedLoad(DL, CI, DTU, ModifiedDT);
       return true;
     case Intrinsic::masked_store:
       if (TTI.isLegalMaskedStore(
               CI->getArgOperand(0)->getType(),
               cast<ConstantInt>(CI->getArgOperand(2))->getAlignValue()))
         return false;
-      scalarizeMaskedStore(CI, ModifiedDT);
+      scalarizeMaskedStore(DL, CI, DTU, ModifiedDT);
       return true;
     case Intrinsic::masked_gather: {
-      unsigned AlignmentInt =
-          cast<ConstantInt>(CI->getArgOperand(1))->getZExtValue();
+      MaybeAlign MA =
+          cast<ConstantInt>(CI->getArgOperand(1))->getMaybeAlignValue();
       Type *LoadTy = CI->getType();
-      Align Alignment =
-          DL.getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), LoadTy);
+      Align Alignment = DL.getValueOrABITypeAlignment(MA,
+                                                      LoadTy->getScalarType());
       if (TTI.isLegalMaskedGather(LoadTy, Alignment))
         return false;
-      scalarizeMaskedGather(CI, ModifiedDT);
+      scalarizeMaskedGather(DL, CI, DTU, ModifiedDT);
       return true;
     }
     case Intrinsic::masked_scatter: {
-      unsigned AlignmentInt =
-          cast<ConstantInt>(CI->getArgOperand(2))->getZExtValue();
+      MaybeAlign MA =
+          cast<ConstantInt>(CI->getArgOperand(2))->getMaybeAlignValue();
       Type *StoreTy = CI->getArgOperand(0)->getType();
-      Align Alignment =
-          DL.getValueOrABITypeAlignment(MaybeAlign(AlignmentInt), StoreTy);
+      Align Alignment = DL.getValueOrABITypeAlignment(MA,
+                                                      StoreTy->getScalarType());
       if (TTI.isLegalMaskedScatter(StoreTy, Alignment))
         return false;
-      scalarizeMaskedScatter(CI, ModifiedDT);
+      scalarizeMaskedScatter(DL, CI, DTU, ModifiedDT);
       return true;
     }
     case Intrinsic::masked_expandload:
       if (TTI.isLegalMaskedExpandLoad(CI->getType()))
         return false;
-      scalarizeMaskedExpandLoad(CI, ModifiedDT);
+      scalarizeMaskedExpandLoad(DL, CI, DTU, ModifiedDT);
       return true;
     case Intrinsic::masked_compressstore:
       if (TTI.isLegalMaskedCompressStore(CI->getArgOperand(0)->getType()))
         return false;
-      scalarizeMaskedCompressStore(CI, ModifiedDT);
+      scalarizeMaskedCompressStore(DL, CI, DTU, ModifiedDT);
       return true;
     }
   }
diff --git a/llvm/lib/Transforms/Scalar/Scalarizer.cpp b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
index c95984fe198f..8ef6b69673be 100644
--- a/llvm/lib/Transforms/Scalar/Scalarizer.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalarizer.cpp
@@ -510,8 +510,8 @@ static bool isTriviallyScalariable(Intrinsic::ID ID) {
 // All of the current scalarizable intrinsics only have one mangled type.
 static Function *getScalarIntrinsicDeclaration(Module *M,
                                                Intrinsic::ID ID,
-                                               VectorType *Ty) {
-  return Intrinsic::getDeclaration(M, ID, { Ty->getScalarType() });
+                                               ArrayRef<Type*> Tys) {
+  return Intrinsic::getDeclaration(M, ID, Tys);
 }
 
 /// If a call to a vector typed intrinsic function, split into a scalar call per
@@ -537,6 +537,9 @@ bool ScalarizerVisitor::splitCall(CallInst &CI) {
 
   Scattered.resize(NumArgs);
 
+  SmallVector<llvm::Type *, 3> Tys;
+  Tys.push_back(VT->getScalarType());
+
   // Assumes that any vector type has the same number of elements as the return
   // vector type, which is true for all current intrinsics.
   for (unsigned I = 0; I != NumArgs; ++I) {
@@ -546,13 +549,15 @@ bool ScalarizerVisitor::splitCall(CallInst &CI) {
       assert(Scattered[I].size() == NumElems && "mismatched call operands");
     } else {
       ScalarOperands[I] = OpI;
+      if (hasVectorInstrinsicOverloadedScalarOpd(ID, I))
+        Tys.push_back(OpI->getType());
     }
   }
 
   ValueVector Res(NumElems);
   ValueVector ScalarCallOps(NumArgs);
 
-  Function *NewIntrin = getScalarIntrinsicDeclaration(F->getParent(), ID, VT);
+  Function *NewIntrin = getScalarIntrinsicDeclaration(F->getParent(), ID, Tys);
   IRBuilder<> Builder(&CI);
 
   // Perform actual scalarization, taking care to preserve any scalar operands.
diff --git a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
index 9d3c8d0f3739..b9cccc2af309 100644
--- a/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
+++ b/llvm/lib/Transforms/Scalar/SimpleLoopUnswitch.cpp
@@ -38,6 +38,7 @@
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/PatternMatch.h"
 #include "llvm/IR/Use.h"
 #include "llvm/IR/Value.h"
 #include "llvm/InitializePasses.h"
@@ -63,6 +64,7 @@
 #define DEBUG_TYPE "simple-loop-unswitch"
 
 using namespace llvm;
+using namespace llvm::PatternMatch;
 
 STATISTIC(NumBranches, "Number of branches unswitched");
 STATISTIC(NumSwitches, "Number of switches unswitched");
@@ -101,6 +103,11 @@ static cl::opt<bool> DropNonTrivialImplicitNullChecks(
     cl::init(false), cl::Hidden,
     cl::desc("If enabled, drop make.implicit metadata in unswitched implicit "
              "null checks to save time analyzing if we can keep it."));
+static cl::opt<unsigned>
+    MSSAThreshold("simple-loop-unswitch-memoryssa-threshold",
+                  cl::desc("Max number of memory uses to explore during "
+                           "partial unswitching analysis"),
+                  cl::init(100), cl::Hidden);
 
 /// Collect all of the loop invariant input values transitively used by the
 /// homogeneous instruction graph from a given root.
@@ -116,6 +123,9 @@ collectHomogenousInstGraphLoopInvariants(Loop &L, Instruction &Root,
          "Only need to walk the graph if root itself is not invariant.");
   TinyPtrVector<Value *> Invariants;
 
+  bool IsRootAnd = match(&Root, m_LogicalAnd());
+  bool IsRootOr  = match(&Root, m_LogicalOr());
+
   // Build a worklist and recurse through operators collecting invariants.
   SmallVector<Instruction *, 4> Worklist;
   SmallPtrSet<Instruction *, 8> Visited;
@@ -136,12 +146,13 @@ collectHomogenousInstGraphLoopInvariants(Loop &L, Instruction &Root,
 
       // If not an instruction with the same opcode, nothing we can do.
       Instruction *OpI = dyn_cast<Instruction>(OpV);
-      if (!OpI || OpI->getOpcode() != Root.getOpcode())
-        continue;
 
-      // Visit this operand.
-      if (Visited.insert(OpI).second)
-        Worklist.push_back(OpI);
+      if (OpI && ((IsRootAnd && match(OpI, m_LogicalAnd())) ||
+                  (IsRootOr  && match(OpI, m_LogicalOr())))) {
+        // Visit this operand.
+        if (Visited.insert(OpI).second)
+          Worklist.push_back(OpI);
+      }
     }
   } while (!Worklist.empty());
 
@@ -153,14 +164,13 @@ static void replaceLoopInvariantUses(Loop &L, Value *Invariant,
   assert(!isa<Constant>(Invariant) && "Why are we unswitching on a constant?");
 
   // Replace uses of LIC in the loop with the given constant.
-  for (auto UI = Invariant->use_begin(), UE = Invariant->use_end(); UI != UE;) {
-    // Grab the use and walk past it so we can clobber it in the use list.
-    Use *U = &*UI++;
-    Instruction *UserI = dyn_cast<Instruction>(U->getUser());
+  // We use make_early_inc_range as set invalidates the iterator.
+  for (Use &U : llvm::make_early_inc_range(Invariant->uses())) {
+    Instruction *UserI = dyn_cast<Instruction>(U.getUser());
 
     // Replace this use within the loop body.
     if (UserI && L.contains(UserI))
-      U->set(&Replacement);
+      U.set(&Replacement);
   }
 }
 
@@ -182,8 +192,9 @@ static bool areLoopExitPHIsLoopInvariant(Loop &L, BasicBlock &ExitingBB,
   llvm_unreachable("Basic blocks should never be empty!");
 }
 
-/// Insert code to test a set of loop invariant values, and conditionally branch
-/// on them.
+/// Copy a set of loop invariant values \p ToDuplicate and insert them at the
+/// end of \p BB and conditionally branch on the copied condition. We only
+/// branch on a single value.
 static void buildPartialUnswitchConditionalBranch(BasicBlock &BB,
                                                   ArrayRef<Value *> Invariants,
                                                   bool Direction,
@@ -197,6 +208,49 @@ static void buildPartialUnswitchConditionalBranch(BasicBlock &BB,
                    Direction ? &NormalSucc : &UnswitchedSucc);
 }
 
+/// Copy a set of loop invariant values, and conditionally branch on them.
+static void buildPartialInvariantUnswitchConditionalBranch(
+    BasicBlock &BB, ArrayRef<Value *> ToDuplicate, bool Direction,
+    BasicBlock &UnswitchedSucc, BasicBlock &NormalSucc, Loop &L,
+    MemorySSAUpdater *MSSAU) {
+  ValueToValueMapTy VMap;
+  for (auto *Val : reverse(ToDuplicate)) {
+    Instruction *Inst = cast<Instruction>(Val);
+    Instruction *NewInst = Inst->clone();
+    BB.getInstList().insert(BB.end(), NewInst);
+    RemapInstruction(NewInst, VMap,
+                     RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
+    VMap[Val] = NewInst;
+
+    if (!MSSAU)
+      continue;
+
+    MemorySSA *MSSA = MSSAU->getMemorySSA();
+    if (auto *MemUse =
+            dyn_cast_or_null<MemoryUse>(MSSA->getMemoryAccess(Inst))) {
+      auto *DefiningAccess = MemUse->getDefiningAccess();
+      // Get the first defining access before the loop.
+      while (L.contains(DefiningAccess->getBlock())) {
+        // If the defining access is a MemoryPhi, get the incoming
+        // value for the pre-header as defining access.
+        if (auto *MemPhi = dyn_cast<MemoryPhi>(DefiningAccess))
+          DefiningAccess =
+              MemPhi->getIncomingValueForBlock(L.getLoopPreheader());
+        else
+          DefiningAccess = cast<MemoryDef>(DefiningAccess)->getDefiningAccess();
+      }
+      MSSAU->createMemoryAccessInBB(NewInst, DefiningAccess,
+                                    NewInst->getParent(),
+                                    MemorySSA::BeforeTerminator);
+    }
+  }
+
+  IRBuilder<> IRB(&BB);
+  Value *Cond = VMap[ToDuplicate[0]];
+  IRB.CreateCondBr(Cond, Direction ? &UnswitchedSucc : &NormalSucc,
+                   Direction ? &NormalSucc : &UnswitchedSucc);
+}
+
 /// Rewrite the PHI nodes in an unswitched loop exit basic block.
 ///
 /// Requires that the loop exit and unswitched basic block are the same, and
@@ -366,7 +420,7 @@ static Loop *getTopMostExitingLoop(BasicBlock *ExitBB, LoopInfo &LI) {
 /// hoists the branch above that split. Preserves loop simplified form
 /// (splitting the exit block as necessary). It simplifies the branch within
 /// the loop to an unconditional branch but doesn't remove it entirely. Further
-/// cleanup can be done with some simplify-cfg like pass.
+/// cleanup can be done with some simplifycfg like pass.
 ///
 /// If `SE` is not null, it will be updated based on the potential loop SCEVs
 /// invalidated by this.
@@ -389,9 +443,10 @@ static bool unswitchTrivialBranch(Loop &L, BranchInst &BI, DominatorTree &DT,
   } else {
     if (auto *CondInst = dyn_cast<Instruction>(BI.getCondition()))
       Invariants = collectHomogenousInstGraphLoopInvariants(L, *CondInst, LI);
-    if (Invariants.empty())
-      // Couldn't find invariant inputs!
+    if (Invariants.empty()) {
+      LLVM_DEBUG(dbgs() << "   Couldn't find invariant inputs!\n");
       return false;
+    }
   }
 
   // Check that one of the branch's successors exits, and which one.
@@ -402,13 +457,17 @@ static bool unswitchTrivialBranch(Loop &L, BranchInst &BI, DominatorTree &DT,
     ExitDirection = false;
     LoopExitSuccIdx = 1;
     LoopExitBB = BI.getSuccessor(1);
-    if (L.contains(LoopExitBB))
+    if (L.contains(LoopExitBB)) {
+      LLVM_DEBUG(dbgs() << "   Branch doesn't exit the loop!\n");
       return false;
+    }
   }
   auto *ContinueBB = BI.getSuccessor(1 - LoopExitSuccIdx);
   auto *ParentBB = BI.getParent();
-  if (!areLoopExitPHIsLoopInvariant(L, *ParentBB, *LoopExitBB))
+  if (!areLoopExitPHIsLoopInvariant(L, *ParentBB, *LoopExitBB)) {
+    LLVM_DEBUG(dbgs() << "   Loop exit PHI's aren't loop-invariant!\n");
     return false;
+  }
 
   // When unswitching only part of the branch's condition, we need the exit
   // block to be reached directly from the partially unswitched input. This can
@@ -416,12 +475,11 @@ static bool unswitchTrivialBranch(Loop &L, BranchInst &BI, DominatorTree &DT,
   // is a graph of `or` operations, or the exit block is along the false edge
   // and the condition is a graph of `and` operations.
   if (!FullUnswitch) {
-    if (ExitDirection) {
-      if (cast<Instruction>(BI.getCondition())->getOpcode() != Instruction::Or)
-        return false;
-    } else {
-      if (cast<Instruction>(BI.getCondition())->getOpcode() != Instruction::And)
-        return false;
+    if (ExitDirection ? !match(BI.getCondition(), m_LogicalOr())
+                      : !match(BI.getCondition(), m_LogicalAnd())) {
+      LLVM_DEBUG(dbgs() << "   Branch condition is in improper form for "
+                           "non-full unswitch!\n");
+      return false;
     }
   }
 
@@ -498,13 +556,13 @@ static bool unswitchTrivialBranch(Loop &L, BranchInst &BI, DominatorTree &DT,
     // Only unswitching a subset of inputs to the condition, so we will need to
     // build a new branch that merges the invariant inputs.
     if (ExitDirection)
-      assert(cast<Instruction>(BI.getCondition())->getOpcode() ==
-                 Instruction::Or &&
-             "Must have an `or` of `i1`s for the condition!");
+      assert(match(BI.getCondition(), m_LogicalOr()) &&
+             "Must have an `or` of `i1`s or `select i1 X, true, Y`s for the "
+             "condition!");
     else
-      assert(cast<Instruction>(BI.getCondition())->getOpcode() ==
-                 Instruction::And &&
-             "Must have an `and` of `i1`s for the condition!");
+      assert(match(BI.getCondition(), m_LogicalAnd()) &&
+             "Must have an `and` of `i1`s or `select i1 X, Y, false`s for the"
+             " condition!");
     buildPartialUnswitchConditionalBranch(*OldPH, Invariants, ExitDirection,
                                           *UnswitchedBB, *NewPH);
   }
@@ -590,7 +648,7 @@ static bool unswitchTrivialBranch(Loop &L, BranchInst &BI, DominatorTree &DT,
 /// considered for unswitching so this is a stable transform and the same
 /// switch will not be revisited. If after unswitching there is only a single
 /// in-loop successor, the switch is further simplified to an unconditional
-/// branch. Still more cleanup can be done with some simplify-cfg like pass.
+/// branch. Still more cleanup can be done with some simplifycfg like pass.
 ///
 /// If `SE` is not null, it will be updated based on the potential loop SCEVs
 /// invalidated by this.
@@ -925,7 +983,7 @@ static bool unswitchAllTrivialConditions(Loop &L, DominatorTree &DT,
     if (auto *SI = dyn_cast<SwitchInst>(CurrentTerm)) {
       // Don't bother trying to unswitch past a switch with a constant
       // condition. This should be removed prior to running this pass by
-      // simplify-cfg.
+      // simplifycfg.
       if (isa<Constant>(SI->getCondition()))
         return Changed;
 
@@ -954,7 +1012,7 @@ static bool unswitchAllTrivialConditions(Loop &L, DominatorTree &DT,
       return Changed;
 
     // Don't bother trying to unswitch past an unconditional branch or a branch
-    // with a constant value. These should be removed by simplify-cfg prior to
+    // with a constant value. These should be removed by simplifycfg prior to
     // running this pass.
     if (!BI->isConditional() || isa<Constant>(BI->getCondition()))
       return Changed;
@@ -1108,9 +1166,8 @@ static BasicBlock *buildClonedLoopBlocks(
     for (Instruction &I : *ClonedBB) {
       RemapInstruction(&I, VMap,
                        RF_NoModuleLevelChanges | RF_IgnoreMissingLocals);
-      if (auto *II = dyn_cast<IntrinsicInst>(&I))
-        if (II->getIntrinsicID() == Intrinsic::assume)
-          AC.registerAssumption(II);
+      if (auto *II = dyn_cast<AssumeInst>(&I))
+        AC.registerAssumption(II);
     }
 
   // Update any PHI nodes in the cloned successors of the skipped blocks to not
@@ -1959,18 +2016,22 @@ void visitDomSubTree(DominatorTree &DT, BasicBlock *BB, CallableT Callable) {
 
 static void unswitchNontrivialInvariants(
     Loop &L, Instruction &TI, ArrayRef<Value *> Invariants,
-    SmallVectorImpl<BasicBlock *> &ExitBlocks, DominatorTree &DT, LoopInfo &LI,
-    AssumptionCache &AC, function_ref<void(bool, ArrayRef<Loop *>)> UnswitchCB,
+    SmallVectorImpl<BasicBlock *> &ExitBlocks, IVConditionInfo &PartialIVInfo,
+    DominatorTree &DT, LoopInfo &LI, AssumptionCache &AC,
+    function_ref<void(bool, bool, ArrayRef<Loop *>)> UnswitchCB,
     ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
   auto *ParentBB = TI.getParent();
   BranchInst *BI = dyn_cast<BranchInst>(&TI);
   SwitchInst *SI = BI ? nullptr : cast<SwitchInst>(&TI);
 
   // We can only unswitch switches, conditional branches with an invariant
-  // condition, or combining invariant conditions with an instruction.
+  // condition, or combining invariant conditions with an instruction or
+  // partially invariant instructions.
   assert((SI || (BI && BI->isConditional())) &&
          "Can only unswitch switches and conditional branch!");
-  bool FullUnswitch = SI || BI->getCondition() == Invariants[0];
+  bool PartiallyInvariant = !PartialIVInfo.InstToDuplicate.empty();
+  bool FullUnswitch =
+      SI || (BI->getCondition() == Invariants[0] && !PartiallyInvariant);
   if (FullUnswitch)
     assert(Invariants.size() == 1 &&
            "Cannot have other invariants with full unswitching!");
@@ -1984,19 +2045,24 @@ static void unswitchNontrivialInvariants(
   // Constant and BBs tracking the cloned and continuing successor. When we are
   // unswitching the entire condition, this can just be trivially chosen to
   // unswitch towards `true`. However, when we are unswitching a set of
-  // invariants combined with `and` or `or`, the combining operation determines
-  // the best direction to unswitch: we want to unswitch the direction that will
-  // collapse the branch.
+  // invariants combined with `and` or `or` or partially invariant instructions,
+  // the combining operation determines the best direction to unswitch: we want
+  // to unswitch the direction that will collapse the branch.
   bool Direction = true;
   int ClonedSucc = 0;
   if (!FullUnswitch) {
-    if (cast<Instruction>(BI->getCondition())->getOpcode() != Instruction::Or) {
-      assert(cast<Instruction>(BI->getCondition())->getOpcode() ==
-                 Instruction::And &&
-             "Only `or` and `and` instructions can combine invariants being "
-             "unswitched.");
-      Direction = false;
-      ClonedSucc = 1;
+    Value *Cond = BI->getCondition();
+    (void)Cond;
+    assert(((match(Cond, m_LogicalAnd()) ^ match(Cond, m_LogicalOr())) ||
+            PartiallyInvariant) &&
+           "Only `or`, `and`, an `select`, partially invariant instructions "
+           "can combine invariants being unswitched.");
+    if (!match(BI->getCondition(), m_LogicalOr())) {
+      if (match(BI->getCondition(), m_LogicalAnd()) ||
+          (PartiallyInvariant && !PartialIVInfo.KnownValue->isOneValue())) {
+        Direction = false;
+        ClonedSucc = 1;
+      }
     }
   }
 
@@ -2214,8 +2280,12 @@ static void unswitchNontrivialInvariants(
     BasicBlock *ClonedPH = ClonedPHs.begin()->second;
     // When doing a partial unswitch, we have to do a bit more work to build up
     // the branch in the split block.
-    buildPartialUnswitchConditionalBranch(*SplitBB, Invariants, Direction,
-                                          *ClonedPH, *LoopPH);
+    if (PartiallyInvariant)
+      buildPartialInvariantUnswitchConditionalBranch(
+          *SplitBB, Invariants, Direction, *ClonedPH, *LoopPH, L, MSSAU);
+    else
+      buildPartialUnswitchConditionalBranch(*SplitBB, Invariants, Direction,
+                                            *ClonedPH, *LoopPH);
     DTUpdates.push_back({DominatorTree::Insert, SplitBB, ClonedPH});
 
     if (MSSAU) {
@@ -2267,7 +2337,7 @@ static void unswitchNontrivialInvariants(
   // verification steps.
   assert(DT.verify(DominatorTree::VerificationLevel::Fast));
 
-  if (BI) {
+  if (BI && !PartiallyInvariant) {
     // If we unswitched a branch which collapses the condition to a known
     // constant we want to replace all the uses of the invariants within both
     // the original and cloned blocks. We do this here so that we can use the
@@ -2285,7 +2355,8 @@ static void unswitchNontrivialInvariants(
     // for each invariant operand.
     // So it happens that for multiple-partial case we dont replace
     // in the unswitched branch.
-    bool ReplaceUnswitched = FullUnswitch || (Invariants.size() == 1);
+    bool ReplaceUnswitched =
+        FullUnswitch || (Invariants.size() == 1) || PartiallyInvariant;
 
     ConstantInt *UnswitchedReplacement =
         Direction ? ConstantInt::getTrue(BI->getContext())
@@ -2294,21 +2365,19 @@ static void unswitchNontrivialInvariants(
         Direction ? ConstantInt::getFalse(BI->getContext())
                   : ConstantInt::getTrue(BI->getContext());
     for (Value *Invariant : Invariants)
-      for (auto UI = Invariant->use_begin(), UE = Invariant->use_end();
-           UI != UE;) {
-        // Grab the use and walk past it so we can clobber it in the use list.
-        Use *U = &*UI++;
-        Instruction *UserI = dyn_cast<Instruction>(U->getUser());
+      // Use make_early_inc_range here as set invalidates the iterator.
+      for (Use &U : llvm::make_early_inc_range(Invariant->uses())) {
+        Instruction *UserI = dyn_cast<Instruction>(U.getUser());
         if (!UserI)
           continue;
 
         // Replace it with the 'continue' side if in the main loop body, and the
         // unswitched if in the cloned blocks.
         if (DT.dominates(LoopPH, UserI->getParent()))
-          U->set(ContinueReplacement);
+          U.set(ContinueReplacement);
         else if (ReplaceUnswitched &&
                  DT.dominates(ClonedPH, UserI->getParent()))
-          U->set(UnswitchedReplacement);
+          U.set(UnswitchedReplacement);
       }
   }
 
@@ -2382,7 +2451,7 @@ static void unswitchNontrivialInvariants(
   for (Loop *UpdatedL : llvm::concat<Loop *>(NonChildClonedLoops, HoistedLoops))
     if (UpdatedL->getParentLoop() == ParentL)
       SibLoops.push_back(UpdatedL);
-  UnswitchCB(IsStillLoop, SibLoops);
+  UnswitchCB(IsStillLoop, PartiallyInvariant, SibLoops);
 
   if (MSSAU && VerifyMemorySSA)
     MSSAU->getMemorySSA()->verifyMemorySSA();
@@ -2399,10 +2468,10 @@ static void unswitchNontrivialInvariants(
 /// The recursive computation is memozied into the provided DT-indexed cost map
 /// to allow querying it for most nodes in the domtree without it becoming
 /// quadratic.
-static int
-computeDomSubtreeCost(DomTreeNode &N,
-                      const SmallDenseMap<BasicBlock *, int, 4> &BBCostMap,
-                      SmallDenseMap<DomTreeNode *, int, 4> &DTCostMap) {
+static InstructionCost computeDomSubtreeCost(
+    DomTreeNode &N,
+    const SmallDenseMap<BasicBlock *, InstructionCost, 4> &BBCostMap,
+    SmallDenseMap<DomTreeNode *, InstructionCost, 4> &DTCostMap) {
   // Don't accumulate cost (or recurse through) blocks not in our block cost
   // map and thus not part of the duplication cost being considered.
   auto BBCostIt = BBCostMap.find(N.getBlock());
@@ -2416,8 +2485,9 @@ computeDomSubtreeCost(DomTreeNode &N,
 
   // If not, we have to compute it. We can't use insert above and update
   // because computing the cost may insert more things into the map.
-  int Cost = std::accumulate(
-      N.begin(), N.end(), BBCostIt->second, [&](int Sum, DomTreeNode *ChildN) {
+  InstructionCost Cost = std::accumulate(
+      N.begin(), N.end(), BBCostIt->second,
+      [&](InstructionCost Sum, DomTreeNode *ChildN) -> InstructionCost {
         return Sum + computeDomSubtreeCost(*ChildN, BBCostMap, DTCostMap);
       });
   bool Inserted = DTCostMap.insert({&N, Cost}).second;
@@ -2596,11 +2666,11 @@ static int CalculateUnswitchCostMultiplier(
   return CostMultiplier;
 }
 
-static bool
-unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
-                      AssumptionCache &AC, TargetTransformInfo &TTI,
-                      function_ref<void(bool, ArrayRef<Loop *>)> UnswitchCB,
-                      ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
+static bool unswitchBestCondition(
+    Loop &L, DominatorTree &DT, LoopInfo &LI, AssumptionCache &AC,
+    AAResults &AA, TargetTransformInfo &TTI,
+    function_ref<void(bool, bool, ArrayRef<Loop *>)> UnswitchCB,
+    ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
   // Collect all invariant conditions within this loop (as opposed to an inner
   // loop which would be handled when visiting that inner loop).
   SmallVector<std::pair<Instruction *, TinyPtrVector<Value *>>, 4>
@@ -2615,6 +2685,7 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
       CollectGuards = true;
   }
 
+  IVConditionInfo PartialIVInfo;
   for (auto *BB : L.blocks()) {
     if (LI.getLoopFor(BB) != &L)
       continue;
@@ -2642,22 +2713,48 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
         BI->getSuccessor(0) == BI->getSuccessor(1))
       continue;
 
+    // If BI's condition is 'select _, true, false', simplify it to confuse
+    // matchers
+    Value *Cond = BI->getCondition(), *CondNext;
+    while (match(Cond, m_Select(m_Value(CondNext), m_One(), m_Zero())))
+      Cond = CondNext;
+    BI->setCondition(Cond);
+
     if (L.isLoopInvariant(BI->getCondition())) {
       UnswitchCandidates.push_back({BI, {BI->getCondition()}});
       continue;
     }
 
     Instruction &CondI = *cast<Instruction>(BI->getCondition());
-    if (CondI.getOpcode() != Instruction::And &&
-      CondI.getOpcode() != Instruction::Or)
-      continue;
+    if (match(&CondI, m_CombineOr(m_LogicalAnd(), m_LogicalOr()))) {
+      TinyPtrVector<Value *> Invariants =
+          collectHomogenousInstGraphLoopInvariants(L, CondI, LI);
+      if (Invariants.empty())
+        continue;
 
-    TinyPtrVector<Value *> Invariants =
-        collectHomogenousInstGraphLoopInvariants(L, CondI, LI);
-    if (Invariants.empty())
+      UnswitchCandidates.push_back({BI, std::move(Invariants)});
       continue;
+    }
+  }
 
-    UnswitchCandidates.push_back({BI, std::move(Invariants)});
+  Instruction *PartialIVCondBranch = nullptr;
+  if (MSSAU && !findOptionMDForLoop(&L, "llvm.loop.unswitch.partial.disable") &&
+      !any_of(UnswitchCandidates, [&L](auto &TerminatorAndInvariants) {
+        return TerminatorAndInvariants.first == L.getHeader()->getTerminator();
+      })) {
+    MemorySSA *MSSA = MSSAU->getMemorySSA();
+    if (auto Info = hasPartialIVCondition(L, MSSAThreshold, *MSSA, AA)) {
+      LLVM_DEBUG(
+          dbgs() << "simple-loop-unswitch: Found partially invariant condition "
+                 << *Info->InstToDuplicate[0] << "\n");
+      PartialIVInfo = *Info;
+      PartialIVCondBranch = L.getHeader()->getTerminator();
+      TinyPtrVector<Value *> ValsToDuplicate;
+      for (auto *Inst : Info->InstToDuplicate)
+        ValsToDuplicate.push_back(Inst);
+      UnswitchCandidates.push_back(
+          {L.getHeader()->getTerminator(), std::move(ValsToDuplicate)});
+    }
   }
 
   // If we didn't find any candidates, we're done.
@@ -2678,15 +2775,18 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
   SmallVector<BasicBlock *, 4> ExitBlocks;
   L.getUniqueExitBlocks(ExitBlocks);
 
-  // We cannot unswitch if exit blocks contain a cleanuppad instruction as we
-  // don't know how to split those exit blocks.
+  // We cannot unswitch if exit blocks contain a cleanuppad/catchswitch
+  // instruction as we don't know how to split those exit blocks.
   // FIXME: We should teach SplitBlock to handle this and remove this
   // restriction.
-  for (auto *ExitBB : ExitBlocks)
-    if (isa<CleanupPadInst>(ExitBB->getFirstNonPHI())) {
-      dbgs() << "Cannot unswitch because of cleanuppad in exit block\n";
+  for (auto *ExitBB : ExitBlocks) {
+    auto *I = ExitBB->getFirstNonPHI();
+    if (isa<CleanupPadInst>(I) || isa<CatchSwitchInst>(I)) {
+      LLVM_DEBUG(dbgs() << "Cannot unswitch because of cleanuppad/catchswitch "
+                           "in exit block\n");
       return false;
     }
+  }
 
   LLVM_DEBUG(
       dbgs() << "Considering " << UnswitchCandidates.size()
@@ -2699,7 +2799,7 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
   // subsets of the loop for duplication during unswitching.
   SmallPtrSet<const Value *, 4> EphValues;
   CodeMetrics::collectEphemeralValues(&L, &AC, EphValues);
-  SmallDenseMap<BasicBlock *, int, 4> BBCostMap;
+  SmallDenseMap<BasicBlock *, InstructionCost, 4> BBCostMap;
 
   // Compute the cost of each block, as well as the total loop cost. Also, bail
   // out if we see instructions which are incompatible with loop unswitching
@@ -2710,9 +2810,9 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
       L.getHeader()->getParent()->hasMinSize()
       ? TargetTransformInfo::TCK_CodeSize
       : TargetTransformInfo::TCK_SizeAndLatency;
-  int LoopCost = 0;
+  InstructionCost LoopCost = 0;
   for (auto *BB : L.blocks()) {
-    int Cost = 0;
+    InstructionCost Cost = 0;
     for (auto &I : *BB) {
       if (EphValues.count(&I))
         continue;
@@ -2746,37 +2846,38 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
   // This requires memoizing each dominator subtree to avoid redundant work.
   //
   // FIXME: Need to actually do the number of candidates part above.
-  SmallDenseMap<DomTreeNode *, int, 4> DTCostMap;
+  SmallDenseMap<DomTreeNode *, InstructionCost, 4> DTCostMap;
   // Given a terminator which might be unswitched, computes the non-duplicated
   // cost for that terminator.
-  auto ComputeUnswitchedCost = [&](Instruction &TI, bool FullUnswitch) {
+  auto ComputeUnswitchedCost = [&](Instruction &TI,
+                                   bool FullUnswitch) -> InstructionCost {
     BasicBlock &BB = *TI.getParent();
     SmallPtrSet<BasicBlock *, 4> Visited;
 
-    int Cost = LoopCost;
+    InstructionCost Cost = 0;
     for (BasicBlock *SuccBB : successors(&BB)) {
       // Don't count successors more than once.
       if (!Visited.insert(SuccBB).second)
         continue;
 
       // If this is a partial unswitch candidate, then it must be a conditional
-      // branch with a condition of either `or` or `and`. In that case, one of
+      // branch with a condition of either `or`, `and`, their corresponding
+      // select forms or partially invariant instructions. In that case, one of
       // the successors is necessarily duplicated, so don't even try to remove
       // its cost.
       if (!FullUnswitch) {
         auto &BI = cast<BranchInst>(TI);
-        if (cast<Instruction>(BI.getCondition())->getOpcode() ==
-            Instruction::And) {
+        if (match(BI.getCondition(), m_LogicalAnd())) {
           if (SuccBB == BI.getSuccessor(1))
             continue;
-        } else {
-          assert(cast<Instruction>(BI.getCondition())->getOpcode() ==
-                     Instruction::Or &&
-                 "Only `and` and `or` conditions can result in a partial "
-                 "unswitch!");
+        } else if (match(BI.getCondition(), m_LogicalOr())) {
           if (SuccBB == BI.getSuccessor(0))
             continue;
-        }
+        } else if ((PartialIVInfo.KnownValue->isOneValue() &&
+                    SuccBB == BI.getSuccessor(0)) ||
+                   (!PartialIVInfo.KnownValue->isOneValue() &&
+                    SuccBB == BI.getSuccessor(1)))
+          continue;
       }
 
       // This successor's domtree will not need to be duplicated after
@@ -2787,8 +2888,8 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
           llvm::all_of(predecessors(SuccBB), [&](BasicBlock *PredBB) {
             return PredBB == &BB || DT.dominates(SuccBB, PredBB);
           })) {
-        Cost -= computeDomSubtreeCost(*DT[SuccBB], BBCostMap, DTCostMap);
-        assert(Cost >= 0 &&
+        Cost += computeDomSubtreeCost(*DT[SuccBB], BBCostMap, DTCostMap);
+        assert(Cost <= LoopCost &&
                "Non-duplicated cost should never exceed total loop cost!");
       }
     }
@@ -2801,16 +2902,16 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
     int SuccessorsCount = isGuard(&TI) ? 2 : Visited.size();
     assert(SuccessorsCount > 1 &&
            "Cannot unswitch a condition without multiple distinct successors!");
-    return Cost * (SuccessorsCount - 1);
+    return (LoopCost - Cost) * (SuccessorsCount - 1);
   };
   Instruction *BestUnswitchTI = nullptr;
-  int BestUnswitchCost = 0;
+  InstructionCost BestUnswitchCost = 0;
   ArrayRef<Value *> BestUnswitchInvariants;
   for (auto &TerminatorAndInvariants : UnswitchCandidates) {
     Instruction &TI = *TerminatorAndInvariants.first;
     ArrayRef<Value *> Invariants = TerminatorAndInvariants.second;
     BranchInst *BI = dyn_cast<BranchInst>(&TI);
-    int CandidateCost = ComputeUnswitchedCost(
+    InstructionCost CandidateCost = ComputeUnswitchedCost(
         TI, /*FullUnswitch*/ !BI || (Invariants.size() == 1 &&
                                      Invariants[0] == BI->getCondition()));
     // Calculate cost multiplier which is a tool to limit potentially
@@ -2844,6 +2945,9 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
     return false;
   }
 
+  if (BestUnswitchTI != PartialIVCondBranch)
+    PartialIVInfo.InstToDuplicate.clear();
+
   // If the best candidate is a guard, turn it into a branch.
   if (isGuard(BestUnswitchTI))
     BestUnswitchTI = turnGuardIntoBranch(cast<IntrinsicInst>(BestUnswitchTI), L,
@@ -2853,7 +2957,8 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
                     << BestUnswitchCost << ") terminator: " << *BestUnswitchTI
                     << "\n");
   unswitchNontrivialInvariants(L, *BestUnswitchTI, BestUnswitchInvariants,
-                               ExitBlocks, DT, LI, AC, UnswitchCB, SE, MSSAU);
+                               ExitBlocks, PartialIVInfo, DT, LI, AC,
+                               UnswitchCB, SE, MSSAU);
   return true;
 }
 
@@ -2864,9 +2969,9 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
 /// looks at other loop invariant control flows and tries to unswitch those as
 /// well by cloning the loop if the result is small enough.
 ///
-/// The `DT`, `LI`, `AC`, `TTI` parameters are required analyses that are also
-/// updated based on the unswitch.
-/// The `MSSA` analysis is also updated if valid (i.e. its use is enabled).
+/// The `DT`, `LI`, `AC`, `AA`, `TTI` parameters are required analyses that are
+/// also updated based on the unswitch. The `MSSA` analysis is also updated if
+/// valid (i.e. its use is enabled).
 ///
 /// If either `NonTrivial` is true or the flag `EnableNonTrivialUnswitch` is
 /// true, we will attempt to do non-trivial unswitching as well as trivial
@@ -2878,11 +2983,12 @@ unswitchBestCondition(Loop &L, DominatorTree &DT, LoopInfo &LI,
 ///
 /// If `SE` is non-null, we will update that analysis based on the unswitching
 /// done.
-static bool unswitchLoop(Loop &L, DominatorTree &DT, LoopInfo &LI,
-                         AssumptionCache &AC, TargetTransformInfo &TTI,
-                         bool NonTrivial,
-                         function_ref<void(bool, ArrayRef<Loop *>)> UnswitchCB,
-                         ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
+static bool
+unswitchLoop(Loop &L, DominatorTree &DT, LoopInfo &LI, AssumptionCache &AC,
+             AAResults &AA, TargetTransformInfo &TTI, bool Trivial,
+             bool NonTrivial,
+             function_ref<void(bool, bool, ArrayRef<Loop *>)> UnswitchCB,
+             ScalarEvolution *SE, MemorySSAUpdater *MSSAU) {
   assert(L.isRecursivelyLCSSAForm(DT, LI) &&
          "Loops must be in LCSSA form before unswitching.");
 
@@ -2891,23 +2997,37 @@ static bool unswitchLoop(Loop &L, DominatorTree &DT, LoopInfo &LI,
     return false;
 
   // Try trivial unswitch first before loop over other basic blocks in the loop.
-  if (unswitchAllTrivialConditions(L, DT, LI, SE, MSSAU)) {
+  if (Trivial && unswitchAllTrivialConditions(L, DT, LI, SE, MSSAU)) {
     // If we unswitched successfully we will want to clean up the loop before
     // processing it further so just mark it as unswitched and return.
-    UnswitchCB(/*CurrentLoopValid*/ true, {});
+    UnswitchCB(/*CurrentLoopValid*/ true, false, {});
     return true;
   }
 
-  // If we're not doing non-trivial unswitching, we're done. We both accept
-  // a parameter but also check a local flag that can be used for testing
-  // a debugging.
-  if (!NonTrivial && !EnableNonTrivialUnswitch)
+  // Check whether we should continue with non-trivial conditions.
+  // EnableNonTrivialUnswitch: Global variable that forces non-trivial
+  //                           unswitching for testing and debugging.
+  // NonTrivial: Parameter that enables non-trivial unswitching for this
+  //             invocation of the transform. But this should be allowed only
+  //             for targets without branch divergence.
+  //
+  // FIXME: If divergence analysis becomes available to a loop
+  // transform, we should allow unswitching for non-trivial uniform
+  // branches even on targets that have divergence.
+  // https://bugs.llvm.org/show_bug.cgi?id=48819
+  bool ContinueWithNonTrivial =
+      EnableNonTrivialUnswitch || (NonTrivial && !TTI.hasBranchDivergence());
+  if (!ContinueWithNonTrivial)
     return false;
 
   // Skip non-trivial unswitching for optsize functions.
   if (L.getHeader()->getParent()->hasOptSize())
     return false;
 
+  // Skip non-trivial unswitching for loops that cannot be cloned.
+  if (!L.isSafeToClone())
+    return false;
+
   // For non-trivial unswitching, because it often creates new loops, we rely on
   // the pass manager to iterate on the loops rather than trying to immediately
   // reach a fixed point. There is no substantial advantage to iterating
@@ -2916,7 +3036,7 @@ static bool unswitchLoop(Loop &L, DominatorTree &DT, LoopInfo &LI,
 
   // Try to unswitch the best invariant condition. We prefer this full unswitch to
   // a partial unswitch when possible below the threshold.
-  if (unswitchBestCondition(L, DT, LI, AC, TTI, UnswitchCB, SE, MSSAU))
+  if (unswitchBestCondition(L, DT, LI, AC, AA, TTI, UnswitchCB, SE, MSSAU))
     return true;
 
   // No other opportunities to unswitch.
@@ -2937,6 +3057,7 @@ PreservedAnalyses SimpleLoopUnswitchPass::run(Loop &L, LoopAnalysisManager &AM,
   std::string LoopName = std::string(L.getName());
 
   auto UnswitchCB = [&L, &U, &LoopName](bool CurrentLoopValid,
+                                        bool PartiallyInvariant,
                                         ArrayRef<Loop *> NewLoops) {
     // If we did a non-trivial unswitch, we have added new (cloned) loops.
     if (!NewLoops.empty())
@@ -2944,9 +3065,21 @@ PreservedAnalyses SimpleLoopUnswitchPass::run(Loop &L, LoopAnalysisManager &AM,
 
     // If the current loop remains valid, we should revisit it to catch any
     // other unswitch opportunities. Otherwise, we need to mark it as deleted.
-    if (CurrentLoopValid)
-      U.revisitCurrentLoop();
-    else
+    if (CurrentLoopValid) {
+      if (PartiallyInvariant) {
+        // Mark the new loop as partially unswitched, to avoid unswitching on
+        // the same condition again.
+        auto &Context = L.getHeader()->getContext();
+        MDNode *DisableUnswitchMD = MDNode::get(
+            Context,
+            MDString::get(Context, "llvm.loop.unswitch.partial.disable"));
+        MDNode *NewLoopID = makePostTransformationMetadata(
+            Context, L.getLoopID(), {"llvm.loop.unswitch.partial"},
+            {DisableUnswitchMD});
+        L.setLoopID(NewLoopID);
+      } else
+        U.revisitCurrentLoop();
+    } else
       U.markLoopAsDeleted(L, LoopName);
   };
 
@@ -2956,8 +3089,9 @@ PreservedAnalyses SimpleLoopUnswitchPass::run(Loop &L, LoopAnalysisManager &AM,
     if (VerifyMemorySSA)
       AR.MSSA->verifyMemorySSA();
   }
-  if (!unswitchLoop(L, AR.DT, AR.LI, AR.AC, AR.TTI, NonTrivial, UnswitchCB,
-                    &AR.SE, MSSAU.hasValue() ? MSSAU.getPointer() : nullptr))
+  if (!unswitchLoop(L, AR.DT, AR.LI, AR.AC, AR.AA, AR.TTI, Trivial, NonTrivial,
+                    UnswitchCB, &AR.SE,
+                    MSSAU.hasValue() ? MSSAU.getPointer() : nullptr))
     return PreservedAnalyses::all();
 
   if (AR.MSSA && VerifyMemorySSA)
@@ -3014,6 +3148,7 @@ bool SimpleLoopUnswitchLegacyPass::runOnLoop(Loop *L, LPPassManager &LPM) {
   auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
   auto &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
   auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+  auto &AA = getAnalysis<AAResultsWrapperPass>().getAAResults();
   auto &TTI = getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
   MemorySSA *MSSA = nullptr;
   Optional<MemorySSAUpdater> MSSAU;
@@ -3025,7 +3160,7 @@ bool SimpleLoopUnswitchLegacyPass::runOnLoop(Loop *L, LPPassManager &LPM) {
   auto *SEWP = getAnalysisIfAvailable<ScalarEvolutionWrapperPass>();
   auto *SE = SEWP ? &SEWP->getSE() : nullptr;
 
-  auto UnswitchCB = [&L, &LPM](bool CurrentLoopValid,
+  auto UnswitchCB = [&L, &LPM](bool CurrentLoopValid, bool PartiallyInvariant,
                                ArrayRef<Loop *> NewLoops) {
     // If we did a non-trivial unswitch, we have added new (cloned) loops.
     for (auto *NewL : NewLoops)
@@ -3034,17 +3169,22 @@ bool SimpleLoopUnswitchLegacyPass::runOnLoop(Loop *L, LPPassManager &LPM) {
     // If the current loop remains valid, re-add it to the queue. This is
     // a little wasteful as we'll finish processing the current loop as well,
     // but it is the best we can do in the old PM.
-    if (CurrentLoopValid)
-      LPM.addLoop(*L);
-    else
+    if (CurrentLoopValid) {
+      // If the current loop has been unswitched using a partially invariant
+      // condition, we should not re-add the current loop to avoid unswitching
+      // on the same condition again.
+      if (!PartiallyInvariant)
+        LPM.addLoop(*L);
+    } else
       LPM.markLoopAsDeleted(*L);
   };
 
   if (MSSA && VerifyMemorySSA)
     MSSA->verifyMemorySSA();
 
-  bool Changed = unswitchLoop(*L, DT, LI, AC, TTI, NonTrivial, UnswitchCB, SE,
-                              MSSAU.hasValue() ? MSSAU.getPointer() : nullptr);
+  bool Changed =
+      unswitchLoop(*L, DT, LI, AC, AA, TTI, true, NonTrivial, UnswitchCB, SE,
+                   MSSAU.hasValue() ? MSSAU.getPointer() : nullptr);
 
   if (MSSA && VerifyMemorySSA)
     MSSA->verifyMemorySSA();
diff --git a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
index 38e7109ead57..09d59b0e884a 100644
--- a/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
+++ b/llvm/lib/Transforms/Scalar/SimplifyCFGPass.cpp
@@ -20,6 +20,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
@@ -42,6 +43,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Scalar/SimplifyCFG.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/SimplifyCFGOptions.h"
 #include <utility>
@@ -76,117 +78,131 @@ static cl::opt<bool> UserSinkCommonInsts(
 
 STATISTIC(NumSimpl, "Number of blocks simplified");
 
-/// If we have more than one empty (other than phi node) return blocks,
-/// merge them together to promote recursive block merging.
-static bool mergeEmptyReturnBlocks(Function &F, DomTreeUpdater *DTU) {
-  bool Changed = false;
-
-  std::vector<DominatorTree::UpdateType> Updates;
-  SmallVector<BasicBlock *, 8> DeadBlocks;
-
-  BasicBlock *RetBlock = nullptr;
+static bool tailMergeBlocksWithSimilarFunctionTerminators(Function &F,
+                                                          DomTreeUpdater *DTU) {
+  SmallMapVector<unsigned /*TerminatorOpcode*/, SmallVector<BasicBlock *, 2>, 4>
+      Structure;
 
-  // Scan all the blocks in the function, looking for empty return blocks.
-  for (BasicBlock &BB : make_early_inc_range(F)) {
+  // Scan all the blocks in the function, record the interesting-ones.
+  for (BasicBlock &BB : F) {
     if (DTU && DTU->isBBPendingDeletion(&BB))
       continue;
 
-    // Only look at return blocks.
-    ReturnInst *Ret = dyn_cast<ReturnInst>(BB.getTerminator());
-    if (!Ret) continue;
+    // We are only interested in function-terminating blocks.
+    if (!succ_empty(&BB))
+      continue;
 
-    // Only look at the block if it is empty or the only other thing in it is a
-    // single PHI node that is the operand to the return.
-    if (Ret != &BB.front()) {
-      // Check for something else in the block.
-      BasicBlock::iterator I(Ret);
-      --I;
-      // Skip over debug info.
-      while (isa<DbgInfoIntrinsic>(I) && I != BB.begin())
-        --I;
-      if (!isa<DbgInfoIntrinsic>(I) &&
-          (!isa<PHINode>(I) || I != BB.begin() || Ret->getNumOperands() == 0 ||
-           Ret->getOperand(0) != &*I))
-        continue;
-    }
+    auto *Term = BB.getTerminator();
 
-    // If this is the first returning block, remember it and keep going.
-    if (!RetBlock) {
-      RetBlock = &BB;
+    // Fow now only support `ret`/`resume` function terminators.
+    // FIXME: lift this restriction.
+    switch (Term->getOpcode()) {
+    case Instruction::Ret:
+    case Instruction::Resume:
+      break;
+    default:
       continue;
     }
 
-    // Skip merging if this would result in a CallBr instruction with a
-    // duplicate destination. FIXME: See note in CodeGenPrepare.cpp.
-    bool SkipCallBr = false;
-    for (pred_iterator PI = pred_begin(&BB), E = pred_end(&BB);
-         PI != E && !SkipCallBr; ++PI) {
-      if (auto *CBI = dyn_cast<CallBrInst>((*PI)->getTerminator()))
-        for (unsigned i = 0, e = CBI->getNumSuccessors(); i != e; ++i)
-          if (RetBlock == CBI->getSuccessor(i)) {
-            SkipCallBr = true;
-            break;
-          }
+    // We can't tail-merge block that contains a musttail call.
+    if (BB.getTerminatingMustTailCall())
+      continue;
+
+    // Calls to experimental_deoptimize must be followed by a return
+    // of the value computed by experimental_deoptimize.
+    // I.e., we can not change `ret` to `br` for this block.
+    if (auto *CI =
+            dyn_cast_or_null<CallInst>(Term->getPrevNonDebugInstruction())) {
+      if (Function *F = CI->getCalledFunction())
+        if (Intrinsic::ID ID = F->getIntrinsicID())
+          if (ID == Intrinsic::experimental_deoptimize)
+            continue;
     }
-    if (SkipCallBr)
+
+    // PHI nodes cannot have token type, so if the terminator has an operand
+    // with token type, we can not tail-merge this kind of function terminators.
+    if (any_of(Term->operands(),
+               [](Value *Op) { return Op->getType()->isTokenTy(); }))
+      continue;
+
+    // Canonical blocks are uniqued based on the terminator type (opcode).
+    Structure[Term->getOpcode()].emplace_back(&BB);
+  }
+
+  bool Changed = false;
+
+  std::vector<DominatorTree::UpdateType> Updates;
+
+  for (ArrayRef<BasicBlock *> BBs : make_second_range(Structure)) {
+    SmallVector<PHINode *, 1> NewOps;
+
+    // We don't want to change IR just because we can.
+    // Only do that if there are at least two blocks we'll tail-merge.
+    if (BBs.size() < 2)
       continue;
 
-    // Otherwise, we found a duplicate return block.  Merge the two.
     Changed = true;
 
-    // Case when there is no input to the return or when the returned values
-    // agree is trivial.  Note that they can't agree if there are phis in the
-    // blocks.
-    if (Ret->getNumOperands() == 0 ||
-        Ret->getOperand(0) ==
-          cast<ReturnInst>(RetBlock->getTerminator())->getOperand(0)) {
-      // All predecessors of BB should now branch to RetBlock instead.
-      if (DTU) {
-        for (auto *Predecessor : predecessors(&BB)) {
-          // But, iff Predecessor already branches to RetBlock,
-          // don't (re-)add DomTree edge, because it already exists.
-          if (!is_contained(successors(Predecessor), RetBlock))
-            Updates.push_back({DominatorTree::Insert, Predecessor, RetBlock});
-          Updates.push_back({DominatorTree::Delete, Predecessor, &BB});
-        }
+    if (DTU)
+      Updates.reserve(Updates.size() + BBs.size());
+
+    BasicBlock *CanonicalBB;
+    Instruction *CanonicalTerm;
+    {
+      auto *Term = BBs[0]->getTerminator();
+
+      // Create a canonical block for this function terminator type now,
+      // placing it *before* the first block that will branch to it.
+      CanonicalBB = BasicBlock::Create(
+          F.getContext(), Twine("common.") + Term->getOpcodeName(), &F, BBs[0]);
+      // We'll also need a PHI node per each operand of the terminator.
+      NewOps.resize(Term->getNumOperands());
+      for (auto I : zip(Term->operands(), NewOps)) {
+        std::get<1>(I) = PHINode::Create(std::get<0>(I)->getType(),
+                                         /*NumReservedValues=*/BBs.size(),
+                                         CanonicalBB->getName() + ".op");
+        CanonicalBB->getInstList().push_back(std::get<1>(I));
       }
-      BB.replaceAllUsesWith(RetBlock);
-      DeadBlocks.emplace_back(&BB);
-      continue;
+      // Make it so that this canonical block actually has the right
+      // terminator.
+      CanonicalTerm = Term->clone();
+      CanonicalBB->getInstList().push_back(CanonicalTerm);
+      // If the canonical terminator has operands, rewrite it to take PHI's.
+      for (auto I : zip(NewOps, CanonicalTerm->operands()))
+        std::get<1>(I) = std::get<0>(I);
     }
 
-    // If the canonical return block has no PHI node, create one now.
-    PHINode *RetBlockPHI = dyn_cast<PHINode>(RetBlock->begin());
-    if (!RetBlockPHI) {
-      Value *InVal = cast<ReturnInst>(RetBlock->getTerminator())->getOperand(0);
-      pred_iterator PB = pred_begin(RetBlock), PE = pred_end(RetBlock);
-      RetBlockPHI = PHINode::Create(Ret->getOperand(0)->getType(),
-                                    std::distance(PB, PE), "merge",
-                                    &RetBlock->front());
+    // Now, go through each block (with the current terminator type)
+    // we've recorded, and rewrite it to branch to the new common block.
+    const DILocation *CommonDebugLoc = nullptr;
+    for (BasicBlock *BB : BBs) {
+      auto *Term = BB->getTerminator();
+
+      // Aha, found a new non-canonical function terminator. If it has operands,
+      // forward them to the PHI nodes in the canonical block.
+      for (auto I : zip(Term->operands(), NewOps))
+        std::get<1>(I)->addIncoming(std::get<0>(I), BB);
+
+      // Compute the debug location common to all the original terminators.
+      if (!CommonDebugLoc)
+        CommonDebugLoc = Term->getDebugLoc();
+      else
+        CommonDebugLoc =
+            DILocation::getMergedLocation(CommonDebugLoc, Term->getDebugLoc());
 
-      for (pred_iterator PI = PB; PI != PE; ++PI)
-        RetBlockPHI->addIncoming(InVal, *PI);
-      RetBlock->getTerminator()->setOperand(0, RetBlockPHI);
+      // And turn BB into a block that just unconditionally branches
+      // to the canonical block.
+      Term->eraseFromParent();
+      BranchInst::Create(CanonicalBB, BB);
+      if (DTU)
+        Updates.push_back({DominatorTree::Insert, BB, CanonicalBB});
     }
 
-    // Turn BB into a block that just unconditionally branches to the return
-    // block.  This handles the case when the two return blocks have a common
-    // predecessor but that return different things.
-    RetBlockPHI->addIncoming(Ret->getOperand(0), &BB);
-    BB.getTerminator()->eraseFromParent();
-    BranchInst::Create(RetBlock, &BB);
-    if (DTU)
-      Updates.push_back({DominatorTree::Insert, &BB, RetBlock});
+    CanonicalTerm->setDebugLoc(CommonDebugLoc);
   }
 
-  if (DTU) {
+  if (DTU)
     DTU->applyUpdates(Updates);
-    for (auto *BB : DeadBlocks)
-      DTU->deleteBB(BB);
-  } else {
-    for (auto *BB : DeadBlocks)
-      BB->eraseFromParent();
-  }
 
   return Changed;
 }
@@ -239,7 +255,8 @@ static bool simplifyFunctionCFGImpl(Function &F, const TargetTransformInfo &TTI,
   DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Eager);
 
   bool EverChanged = removeUnreachableBlocks(F, DT ? &DTU : nullptr);
-  EverChanged |= mergeEmptyReturnBlocks(F, DT ? &DTU : nullptr);
+  EverChanged |=
+      tailMergeBlocksWithSimilarFunctionTerminators(F, DT ? &DTU : nullptr);
   EverChanged |= iterativelySimplifyCFG(F, TTI, DT ? &DTU : nullptr, Options);
 
   // If neither pass changed anything, we're done.
@@ -319,7 +336,6 @@ PreservedAnalyses SimplifyCFGPass::run(Function &F,
   PreservedAnalyses PA;
   if (RequireAndPreserveDomTree)
     PA.preserve<DominatorTreeAnalysis>();
-  PA.preserve<GlobalsAA>();
   return PA;
 }
 
diff --git a/llvm/lib/Transforms/Scalar/Sink.cpp b/llvm/lib/Transforms/Scalar/Sink.cpp
index 89cfbe384be4..8600aacdb056 100644
--- a/llvm/lib/Transforms/Scalar/Sink.cpp
+++ b/llvm/lib/Transforms/Scalar/Sink.cpp
@@ -202,7 +202,7 @@ static bool ProcessBlock(BasicBlock &BB, DominatorTree &DT, LoopInfo &LI,
     if (!ProcessedBegin)
       --I;
 
-    if (isa<DbgInfoIntrinsic>(Inst))
+    if (Inst->isDebugOrPseudoInst())
       continue;
 
     if (SinkInstruction(Inst, Stores, DT, LI, AA)) {
diff --git a/llvm/lib/Transforms/Scalar/SpeculateAroundPHIs.cpp b/llvm/lib/Transforms/Scalar/SpeculateAroundPHIs.cpp
deleted file mode 100644
index 9b18c945d950..000000000000
--- a/llvm/lib/Transforms/Scalar/SpeculateAroundPHIs.cpp
+++ /dev/null
@@ -1,830 +0,0 @@
-//===- SpeculateAroundPHIs.cpp --------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Transforms/Scalar/SpeculateAroundPHIs.h"
-#include "llvm/ADT/PostOrderIterator.h"
-#include "llvm/ADT/Sequence.h"
-#include "llvm/ADT/SetVector.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/TargetTransformInfo.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/IR/BasicBlock.h"
-#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/Instructions.h"
-#include "llvm/IR/IntrinsicInst.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Transforms/Utils/BasicBlockUtils.h"
-
-using namespace llvm;
-
-#define DEBUG_TYPE "spec-phis"
-
-STATISTIC(NumPHIsSpeculated, "Number of PHI nodes we speculated around");
-STATISTIC(NumEdgesSplit,
-          "Number of critical edges which were split for speculation");
-STATISTIC(NumSpeculatedInstructions,
-          "Number of instructions we speculated around the PHI nodes");
-STATISTIC(NumNewRedundantInstructions,
-          "Number of new, redundant instructions inserted");
-
-/// Check whether speculating the users of a PHI node around the PHI
-/// will be safe.
-///
-/// This checks both that all of the users are safe and also that all of their
-/// operands are either recursively safe or already available along an incoming
-/// edge to the PHI.
-///
-/// This routine caches both all the safe nodes explored in `PotentialSpecSet`
-/// and the chain of nodes that definitively reach any unsafe node in
-/// `UnsafeSet`. By preserving these between repeated calls to this routine for
-/// PHIs in the same basic block, the exploration here can be reused. However,
-/// these caches must no be reused for PHIs in a different basic block as they
-/// reflect what is available along incoming edges.
-static bool
-isSafeToSpeculatePHIUsers(PHINode &PN, DominatorTree &DT,
-                          SmallPtrSetImpl<Instruction *> &PotentialSpecSet,
-                          SmallPtrSetImpl<Instruction *> &UnsafeSet) {
-  auto *PhiBB = PN.getParent();
-  SmallPtrSet<Instruction *, 4> Visited;
-  SmallVector<std::pair<Instruction *, User::value_op_iterator>, 16> DFSStack;
-
-  // Walk each user of the PHI node.
-  for (Use &U : PN.uses()) {
-    auto *UI = cast<Instruction>(U.getUser());
-
-    // Ensure the use post-dominates the PHI node. This ensures that, in the
-    // absence of unwinding, the use will actually be reached.
-    // FIXME: We use a blunt hammer of requiring them to be in the same basic
-    // block. We should consider using actual post-dominance here in the
-    // future.
-    if (UI->getParent() != PhiBB) {
-      LLVM_DEBUG(dbgs() << "  Unsafe: use in a different BB: " << *UI << "\n");
-      return false;
-    }
-
-    if (const auto *CS = dyn_cast<CallBase>(UI)) {
-      if (CS->isConvergent() || CS->cannotDuplicate()) {
-        LLVM_DEBUG(dbgs() << "  Unsafe: convergent "
-                   "callsite cannot de duplicated: " << *UI << '\n');
-        return false;
-      }
-    }
-
-    // FIXME: This check is much too conservative. We're not going to move these
-    // instructions onto new dynamic paths through the program unless there is
-    // a call instruction between the use and the PHI node. And memory isn't
-    // changing unless there is a store in that same sequence. We should
-    // probably change this to do at least a limited scan of the intervening
-    // instructions and allow handling stores in easily proven safe cases.
-    if (mayBeMemoryDependent(*UI)) {
-      LLVM_DEBUG(dbgs() << "  Unsafe: can't speculate use: " << *UI << "\n");
-      return false;
-    }
-
-    // Now do a depth-first search of everything these users depend on to make
-    // sure they are transitively safe. This is a depth-first search, but we
-    // check nodes in preorder to minimize the amount of checking.
-    Visited.insert(UI);
-    DFSStack.push_back({UI, UI->value_op_begin()});
-    do {
-      User::value_op_iterator OpIt;
-      std::tie(UI, OpIt) = DFSStack.pop_back_val();
-
-      while (OpIt != UI->value_op_end()) {
-        auto *OpI = dyn_cast<Instruction>(*OpIt);
-        // Increment to the next operand for whenever we continue.
-        ++OpIt;
-        // No need to visit non-instructions, which can't form dependencies.
-        if (!OpI)
-          continue;
-
-        // Now do the main pre-order checks that this operand is a viable
-        // dependency of something we want to speculate.
-
-        // First do a few checks for instructions that won't require
-        // speculation at all because they are trivially available on the
-        // incoming edge (either through dominance or through an incoming value
-        // to a PHI).
-        //
-        // The cases in the current block will be trivially dominated by the
-        // edge.
-        auto *ParentBB = OpI->getParent();
-        if (ParentBB == PhiBB) {
-          if (isa<PHINode>(OpI)) {
-            // We can trivially map through phi nodes in the same block.
-            continue;
-          }
-        } else if (DT.dominates(ParentBB, PhiBB)) {
-          // Instructions from dominating blocks are already available.
-          continue;
-        }
-
-        // Once we know that we're considering speculating the operand, check
-        // if we've already explored this subgraph and found it to be safe.
-        if (PotentialSpecSet.count(OpI))
-          continue;
-
-        // If we've already explored this subgraph and found it unsafe, bail.
-        // If when we directly test whether this is safe it fails, bail.
-        if (UnsafeSet.count(OpI) || ParentBB != PhiBB ||
-            mayBeMemoryDependent(*OpI)) {
-          LLVM_DEBUG(dbgs() << "  Unsafe: can't speculate transitive use: "
-                            << *OpI << "\n");
-          // Record the stack of instructions which reach this node as unsafe
-          // so we prune subsequent searches.
-          UnsafeSet.insert(OpI);
-          for (auto &StackPair : DFSStack) {
-            Instruction *I = StackPair.first;
-            UnsafeSet.insert(I);
-          }
-          return false;
-        }
-
-        // Skip any operands we're already recursively checking.
-        if (!Visited.insert(OpI).second)
-          continue;
-
-        // Push onto the stack and descend. We can directly continue this
-        // loop when ascending.
-        DFSStack.push_back({UI, OpIt});
-        UI = OpI;
-        OpIt = OpI->value_op_begin();
-      }
-
-      // This node and all its operands are safe. Go ahead and cache that for
-      // reuse later.
-      PotentialSpecSet.insert(UI);
-
-      // Continue with the next node on the stack.
-    } while (!DFSStack.empty());
-  }
-
-#ifndef NDEBUG
-  // Every visited operand should have been marked as safe for speculation at
-  // this point. Verify this and return success.
-  for (auto *I : Visited)
-    assert(PotentialSpecSet.count(I) &&
-           "Failed to mark a visited instruction as safe!");
-#endif
-  return true;
-}
-
-/// Check whether, in isolation, a given PHI node is both safe and profitable
-/// to speculate users around.
-///
-/// This handles checking whether there are any constant operands to a PHI
-/// which could represent a useful speculation candidate, whether the users of
-/// the PHI are safe to speculate including all their transitive dependencies,
-/// and whether after speculation there will be some cost savings (profit) to
-/// folding the operands into the users of the PHI node. Returns true if both
-/// safe and profitable with relevant cost savings updated in the map and with
-/// an update to the `PotentialSpecSet`. Returns false if either safety or
-/// profitability are absent. Some new entries may be made to the
-/// `PotentialSpecSet` even when this routine returns false, but they remain
-/// conservatively correct.
-///
-/// The profitability check here is a local one, but it checks this in an
-/// interesting way. Beyond checking that the total cost of materializing the
-/// constants will be less than the cost of folding them into their users, it
-/// also checks that no one incoming constant will have a higher cost when
-/// folded into its users rather than materialized. This higher cost could
-/// result in a dynamic *path* that is more expensive even when the total cost
-/// is lower. Currently, all of the interesting cases where this optimization
-/// should fire are ones where it is a no-loss operation in this sense. If we
-/// ever want to be more aggressive here, we would need to balance the
-/// different incoming edges' cost by looking at their respective
-/// probabilities.
-static bool isSafeAndProfitableToSpeculateAroundPHI(
-    PHINode &PN, SmallDenseMap<PHINode *, int, 16> &CostSavingsMap,
-    SmallPtrSetImpl<Instruction *> &PotentialSpecSet,
-    SmallPtrSetImpl<Instruction *> &UnsafeSet, DominatorTree &DT,
-    TargetTransformInfo &TTI) {
-  // First see whether there is any cost savings to speculating around this
-  // PHI, and build up a map of the constant inputs to how many times they
-  // occur.
-  bool NonFreeMat = false;
-  struct CostsAndCount {
-    int MatCost = TargetTransformInfo::TCC_Free;
-    int FoldedCost = TargetTransformInfo::TCC_Free;
-    int Count = 0;
-  };
-  SmallDenseMap<ConstantInt *, CostsAndCount, 16> CostsAndCounts;
-  SmallPtrSet<BasicBlock *, 16> IncomingConstantBlocks;
-  for (int i : llvm::seq<int>(0, PN.getNumIncomingValues())) {
-    auto *IncomingC = dyn_cast<ConstantInt>(PN.getIncomingValue(i));
-    if (!IncomingC)
-      continue;
-
-    // Only visit each incoming edge with a constant input once.
-    if (!IncomingConstantBlocks.insert(PN.getIncomingBlock(i)).second)
-      continue;
-
-    auto InsertResult = CostsAndCounts.insert({IncomingC, {}});
-    // Count how many edges share a given incoming costant.
-    ++InsertResult.first->second.Count;
-    // Only compute the cost the first time we see a particular constant.
-    if (!InsertResult.second)
-      continue;
-
-    int &MatCost = InsertResult.first->second.MatCost;
-    MatCost = TTI.getIntImmCost(IncomingC->getValue(), IncomingC->getType(),
-                                TargetTransformInfo::TCK_SizeAndLatency);
-    NonFreeMat |= MatCost != TTI.TCC_Free;
-  }
-  if (!NonFreeMat) {
-    LLVM_DEBUG(dbgs() << "    Free: " << PN << "\n");
-    // No profit in free materialization.
-    return false;
-  }
-
-  // Now check that the uses of this PHI can actually be speculated,
-  // otherwise we'll still have to materialize the PHI value.
-  if (!isSafeToSpeculatePHIUsers(PN, DT, PotentialSpecSet, UnsafeSet)) {
-    LLVM_DEBUG(dbgs() << "    Unsafe PHI: " << PN << "\n");
-    return false;
-  }
-
-  // Compute how much (if any) savings are available by speculating around this
-  // PHI.
-  for (Use &U : PN.uses()) {
-    auto *UserI = cast<Instruction>(U.getUser());
-    // Now check whether there is any savings to folding the incoming constants
-    // into this use.
-    unsigned Idx = U.getOperandNo();
-
-    // If we have a binary operator that is commutative, an actual constant
-    // operand would end up on the RHS, so pretend the use of the PHI is on the
-    // RHS.
-    //
-    // Technically, this is a bit weird if *both* operands are PHIs we're
-    // speculating. But if that is the case, giving an "optimistic" cost isn't
-    // a bad thing because after speculation it will constant fold. And
-    // moreover, such cases should likely have been constant folded already by
-    // some other pass, so we shouldn't worry about "modeling" them terribly
-    // accurately here. Similarly, if the other operand is a constant, it still
-    // seems fine to be "optimistic" in our cost modeling, because when the
-    // incoming operand from the PHI node is also a constant, we will end up
-    // constant folding.
-    if (UserI->isBinaryOp() && UserI->isCommutative() && Idx != 1)
-      // Assume we will commute the constant to the RHS to be canonical.
-      Idx = 1;
-
-    // Get the intrinsic ID if this user is an intrinsic.
-    Intrinsic::ID IID = Intrinsic::not_intrinsic;
-    if (auto *UserII = dyn_cast<IntrinsicInst>(UserI))
-      IID = UserII->getIntrinsicID();
-
-    for (auto &IncomingConstantAndCostsAndCount : CostsAndCounts) {
-      ConstantInt *IncomingC = IncomingConstantAndCostsAndCount.first;
-      int MatCost = IncomingConstantAndCostsAndCount.second.MatCost;
-      int &FoldedCost = IncomingConstantAndCostsAndCount.second.FoldedCost;
-      if (IID)
-        FoldedCost +=
-          TTI.getIntImmCostIntrin(IID, Idx, IncomingC->getValue(),
-                                  IncomingC->getType(),
-                                  TargetTransformInfo::TCK_SizeAndLatency);
-      else
-        FoldedCost +=
-            TTI.getIntImmCostInst(UserI->getOpcode(), Idx,
-                                  IncomingC->getValue(), IncomingC->getType(),
-                                  TargetTransformInfo::TCK_SizeAndLatency);
-
-      // If we accumulate more folded cost for this incoming constant than
-      // materialized cost, then we'll regress any edge with this constant so
-      // just bail. We're only interested in cases where folding the incoming
-      // constants is at least break-even on all paths.
-      if (FoldedCost > MatCost) {
-        LLVM_DEBUG(dbgs() << "  Not profitable to fold imm: " << *IncomingC
-                          << "\n"
-                             "    Materializing cost:    "
-                          << MatCost
-                          << "\n"
-                             "    Accumulated folded cost: "
-                          << FoldedCost << "\n");
-        return false;
-      }
-    }
-  }
-
-  // Compute the total cost savings afforded by this PHI node.
-  int TotalMatCost = TTI.TCC_Free, TotalFoldedCost = TTI.TCC_Free;
-  for (auto IncomingConstantAndCostsAndCount : CostsAndCounts) {
-    int MatCost = IncomingConstantAndCostsAndCount.second.MatCost;
-    int FoldedCost = IncomingConstantAndCostsAndCount.second.FoldedCost;
-    int Count = IncomingConstantAndCostsAndCount.second.Count;
-
-    TotalMatCost += MatCost * Count;
-    TotalFoldedCost += FoldedCost * Count;
-  }
-  assert(TotalFoldedCost <= TotalMatCost && "If each constant's folded cost is "
-                                            "less that its materialized cost, "
-                                            "the sum must be as well.");
-
-  LLVM_DEBUG(dbgs() << "    Cost savings " << (TotalMatCost - TotalFoldedCost)
-                    << ": " << PN << "\n");
-  CostSavingsMap[&PN] = TotalMatCost - TotalFoldedCost;
-  return true;
-}
-
-/// Simple helper to walk all the users of a list of phis depth first, and call
-/// a visit function on each one in post-order.
-///
-/// All of the PHIs should be in the same basic block, and this is primarily
-/// used to make a single depth-first walk across their collective users
-/// without revisiting any subgraphs. Callers should provide a fast, idempotent
-/// callable to test whether a node has been visited and the more important
-/// callable to actually visit a particular node.
-///
-/// Depth-first and postorder here refer to the *operand* graph -- we start
-/// from a collection of users of PHI nodes and walk "up" the operands
-/// depth-first.
-template <typename IsVisitedT, typename VisitT>
-static void visitPHIUsersAndDepsInPostOrder(ArrayRef<PHINode *> PNs,
-                                            IsVisitedT IsVisited,
-                                            VisitT Visit) {
-  SmallVector<std::pair<Instruction *, User::value_op_iterator>, 16> DFSStack;
-  for (auto *PN : PNs)
-    for (Use &U : PN->uses()) {
-      auto *UI = cast<Instruction>(U.getUser());
-      if (IsVisited(UI))
-        // Already visited this user, continue across the roots.
-        continue;
-
-      // Otherwise, walk the operand graph depth-first and visit each
-      // dependency in postorder.
-      DFSStack.push_back({UI, UI->value_op_begin()});
-      do {
-        User::value_op_iterator OpIt;
-        std::tie(UI, OpIt) = DFSStack.pop_back_val();
-        while (OpIt != UI->value_op_end()) {
-          auto *OpI = dyn_cast<Instruction>(*OpIt);
-          // Increment to the next operand for whenever we continue.
-          ++OpIt;
-          // No need to visit non-instructions, which can't form dependencies,
-          // or instructions outside of our potential dependency set that we
-          // were given. Finally, if we've already visited the node, continue
-          // to the next.
-          if (!OpI || IsVisited(OpI))
-            continue;
-
-          // Push onto the stack and descend. We can directly continue this
-          // loop when ascending.
-          DFSStack.push_back({UI, OpIt});
-          UI = OpI;
-          OpIt = OpI->value_op_begin();
-        }
-
-        // Finished visiting children, visit this node.
-        assert(!IsVisited(UI) && "Should not have already visited a node!");
-        Visit(UI);
-      } while (!DFSStack.empty());
-    }
-}
-
-/// Find profitable PHIs to speculate.
-///
-/// For a PHI node to be profitable, we need the cost of speculating its users
-/// (and their dependencies) to not exceed the savings of folding the PHI's
-/// constant operands into the speculated users.
-///
-/// Computing this is surprisingly challenging. Because users of two different
-/// PHI nodes can depend on each other or on common other instructions, it may
-/// be profitable to speculate two PHI nodes together even though neither one
-/// in isolation is profitable. The straightforward way to find all the
-/// profitable PHIs would be to check each combination of PHIs' cost, but this
-/// is exponential in complexity.
-///
-/// Even if we assume that we only care about cases where we can consider each
-/// PHI node in isolation (rather than considering cases where none are
-/// profitable in isolation but some subset are profitable as a set), we still
-/// have a challenge. The obvious way to find all individually profitable PHIs
-/// is to iterate until reaching a fixed point, but this will be quadratic in
-/// complexity. =/
-///
-/// This code currently uses a linear-to-compute order for a greedy approach.
-/// It won't find cases where a set of PHIs must be considered together, but it
-/// handles most cases of order dependence without quadratic iteration. The
-/// specific order used is the post-order across the operand DAG. When the last
-/// user of a PHI is visited in this postorder walk, we check it for
-/// profitability.
-///
-/// There is an orthogonal extra complexity to all of this: computing the cost
-/// itself can easily become a linear computation making everything again (at
-/// best) quadratic. Using a postorder over the operand graph makes it
-/// particularly easy to avoid this through dynamic programming. As we do the
-/// postorder walk, we build the transitive cost of that subgraph. It is also
-/// straightforward to then update these costs when we mark a PHI for
-/// speculation so that subsequent PHIs don't re-pay the cost of already
-/// speculated instructions.
-static SmallVector<PHINode *, 16>
-findProfitablePHIs(ArrayRef<PHINode *> PNs,
-                   const SmallDenseMap<PHINode *, int, 16> &CostSavingsMap,
-                   const SmallPtrSetImpl<Instruction *> &PotentialSpecSet,
-                   int NumPreds, DominatorTree &DT, TargetTransformInfo &TTI) {
-  SmallVector<PHINode *, 16> SpecPNs;
-
-  // First, establish a reverse mapping from immediate users of the PHI nodes
-  // to the nodes themselves, and count how many users each PHI node has in
-  // a way we can update while processing them.
-  SmallDenseMap<Instruction *, TinyPtrVector<PHINode *>, 16> UserToPNMap;
-  SmallDenseMap<PHINode *, int, 16> PNUserCountMap;
-  SmallPtrSet<Instruction *, 16> UserSet;
-  for (auto *PN : PNs) {
-    assert(UserSet.empty() && "Must start with an empty user set!");
-    for (Use &U : PN->uses())
-      UserSet.insert(cast<Instruction>(U.getUser()));
-    PNUserCountMap[PN] = UserSet.size();
-    for (auto *UI : UserSet)
-      UserToPNMap.insert({UI, {}}).first->second.push_back(PN);
-    UserSet.clear();
-  }
-
-  // Now do a DFS across the operand graph of the users, computing cost as we
-  // go and when all costs for a given PHI are known, checking that PHI for
-  // profitability.
-  SmallDenseMap<Instruction *, int, 16> SpecCostMap;
-  visitPHIUsersAndDepsInPostOrder(
-      PNs,
-      /*IsVisited*/
-      [&](Instruction *I) {
-        // We consider anything that isn't potentially speculated to be
-        // "visited" as it is already handled. Similarly, anything that *is*
-        // potentially speculated but for which we have an entry in our cost
-        // map, we're done.
-        return !PotentialSpecSet.count(I) || SpecCostMap.count(I);
-      },
-      /*Visit*/
-      [&](Instruction *I) {
-        // We've fully visited the operands, so sum their cost with this node
-        // and update the cost map.
-        int Cost = TTI.TCC_Free;
-        for (Value *OpV : I->operand_values())
-          if (auto *OpI = dyn_cast<Instruction>(OpV)) {
-            auto CostMapIt = SpecCostMap.find(OpI);
-            if (CostMapIt != SpecCostMap.end())
-              Cost += CostMapIt->second;
-          }
-        Cost += TTI.getUserCost(I, TargetTransformInfo::TCK_SizeAndLatency);
-        bool Inserted = SpecCostMap.insert({I, Cost}).second;
-        (void)Inserted;
-        assert(Inserted && "Must not re-insert a cost during the DFS!");
-
-        // Now check if this node had a corresponding PHI node using it. If so,
-        // we need to decrement the outstanding user count for it.
-        auto UserPNsIt = UserToPNMap.find(I);
-        if (UserPNsIt == UserToPNMap.end())
-          return;
-        auto &UserPNs = UserPNsIt->second;
-        auto UserPNsSplitIt = std::stable_partition(
-            UserPNs.begin(), UserPNs.end(), [&](PHINode *UserPN) {
-              int &PNUserCount = PNUserCountMap.find(UserPN)->second;
-              assert(
-                  PNUserCount > 0 &&
-                  "Should never re-visit a PN after its user count hits zero!");
-              --PNUserCount;
-              return PNUserCount != 0;
-            });
-
-        // FIXME: Rather than one at a time, we should sum the savings as the
-        // cost will be completely shared.
-        SmallVector<Instruction *, 16> SpecWorklist;
-        for (auto *PN : llvm::make_range(UserPNsSplitIt, UserPNs.end())) {
-          int SpecCost = TTI.TCC_Free;
-          for (Use &U : PN->uses())
-            SpecCost +=
-                SpecCostMap.find(cast<Instruction>(U.getUser()))->second;
-          SpecCost *= (NumPreds - 1);
-          // When the user count of a PHI node hits zero, we should check its
-          // profitability. If profitable, we should mark it for speculation
-          // and zero out the cost of everything it depends on.
-          int CostSavings = CostSavingsMap.find(PN)->second;
-          if (SpecCost > CostSavings) {
-            LLVM_DEBUG(dbgs() << "  Not profitable, speculation cost: " << *PN
-                              << "\n"
-                                 "    Cost savings:     "
-                              << CostSavings
-                              << "\n"
-                                 "    Speculation cost: "
-                              << SpecCost << "\n");
-            continue;
-          }
-
-          // We're going to speculate this user-associated PHI. Copy it out and
-          // add its users to the worklist to update their cost.
-          SpecPNs.push_back(PN);
-          for (Use &U : PN->uses()) {
-            auto *UI = cast<Instruction>(U.getUser());
-            auto CostMapIt = SpecCostMap.find(UI);
-            if (CostMapIt->second == 0)
-              continue;
-            // Zero out this cost entry to avoid duplicates.
-            CostMapIt->second = 0;
-            SpecWorklist.push_back(UI);
-          }
-        }
-
-        // Now walk all the operands of the users in the worklist transitively
-        // to zero out all the memoized costs.
-        while (!SpecWorklist.empty()) {
-          Instruction *SpecI = SpecWorklist.pop_back_val();
-          assert(SpecCostMap.find(SpecI)->second == 0 &&
-                 "Didn't zero out a cost!");
-
-          // Walk the operands recursively to zero out their cost as well.
-          for (auto *OpV : SpecI->operand_values()) {
-            auto *OpI = dyn_cast<Instruction>(OpV);
-            if (!OpI)
-              continue;
-            auto CostMapIt = SpecCostMap.find(OpI);
-            if (CostMapIt == SpecCostMap.end() || CostMapIt->second == 0)
-              continue;
-            CostMapIt->second = 0;
-            SpecWorklist.push_back(OpI);
-          }
-        }
-      });
-
-  return SpecPNs;
-}
-
-/// Speculate users around a set of PHI nodes.
-///
-/// This routine does the actual speculation around a set of PHI nodes where we
-/// have determined this to be both safe and profitable.
-///
-/// This routine handles any spliting of critical edges necessary to create
-/// a safe block to speculate into as well as cloning the instructions and
-/// rewriting all uses.
-static void speculatePHIs(ArrayRef<PHINode *> SpecPNs,
-                          SmallPtrSetImpl<Instruction *> &PotentialSpecSet,
-                          SmallSetVector<BasicBlock *, 16> &PredSet,
-                          DominatorTree &DT) {
-  LLVM_DEBUG(dbgs() << "  Speculating around " << SpecPNs.size() << " PHIs!\n");
-  NumPHIsSpeculated += SpecPNs.size();
-
-  // Split any critical edges so that we have a block to hoist into.
-  auto *ParentBB = SpecPNs[0]->getParent();
-  SmallVector<BasicBlock *, 16> SpecPreds;
-  SpecPreds.reserve(PredSet.size());
-  for (auto *PredBB : PredSet) {
-    auto *NewPredBB = SplitCriticalEdge(
-        PredBB, ParentBB,
-        CriticalEdgeSplittingOptions(&DT).setMergeIdenticalEdges());
-    if (NewPredBB) {
-      ++NumEdgesSplit;
-      LLVM_DEBUG(dbgs() << "  Split critical edge from: " << PredBB->getName()
-                        << "\n");
-      SpecPreds.push_back(NewPredBB);
-    } else {
-      assert(PredBB->getSingleSuccessor() == ParentBB &&
-             "We need a non-critical predecessor to speculate into.");
-      assert(!isa<InvokeInst>(PredBB->getTerminator()) &&
-             "Cannot have a non-critical invoke!");
-
-      // Already non-critical, use existing pred.
-      SpecPreds.push_back(PredBB);
-    }
-  }
-
-  SmallPtrSet<Instruction *, 16> SpecSet;
-  SmallVector<Instruction *, 16> SpecList;
-  visitPHIUsersAndDepsInPostOrder(SpecPNs,
-                                  /*IsVisited*/
-                                  [&](Instruction *I) {
-                                    // This is visited if we don't need to
-                                    // speculate it or we already have
-                                    // speculated it.
-                                    return !PotentialSpecSet.count(I) ||
-                                           SpecSet.count(I);
-                                  },
-                                  /*Visit*/
-                                  [&](Instruction *I) {
-                                    // All operands scheduled, schedule this
-                                    // node.
-                                    SpecSet.insert(I);
-                                    SpecList.push_back(I);
-                                  });
-
-  int NumSpecInsts = SpecList.size() * SpecPreds.size();
-  int NumRedundantInsts = NumSpecInsts - SpecList.size();
-  LLVM_DEBUG(dbgs() << "  Inserting " << NumSpecInsts
-                    << " speculated instructions, " << NumRedundantInsts
-                    << " redundancies\n");
-  NumSpeculatedInstructions += NumSpecInsts;
-  NumNewRedundantInstructions += NumRedundantInsts;
-
-  // Each predecessor is numbered by its index in `SpecPreds`, so for each
-  // instruction we speculate, the speculated instruction is stored in that
-  // index of the vector associated with the original instruction. We also
-  // store the incoming values for each predecessor from any PHIs used.
-  SmallDenseMap<Instruction *, SmallVector<Value *, 2>, 16> SpeculatedValueMap;
-
-  // Inject the synthetic mappings to rewrite PHIs to the appropriate incoming
-  // value. This handles both the PHIs we are speculating around and any other
-  // PHIs that happen to be used.
-  for (auto *OrigI : SpecList)
-    for (auto *OpV : OrigI->operand_values()) {
-      auto *OpPN = dyn_cast<PHINode>(OpV);
-      if (!OpPN || OpPN->getParent() != ParentBB)
-        continue;
-
-      auto InsertResult = SpeculatedValueMap.insert({OpPN, {}});
-      if (!InsertResult.second)
-        continue;
-
-      auto &SpeculatedVals = InsertResult.first->second;
-
-      // Populating our structure for mapping is particularly annoying because
-      // finding an incoming value for a particular predecessor block in a PHI
-      // node is a linear time operation! To avoid quadratic behavior, we build
-      // a map for this PHI node's incoming values and then translate it into
-      // the more compact representation used below.
-      SmallDenseMap<BasicBlock *, Value *, 16> IncomingValueMap;
-      for (int i : llvm::seq<int>(0, OpPN->getNumIncomingValues()))
-        IncomingValueMap[OpPN->getIncomingBlock(i)] = OpPN->getIncomingValue(i);
-
-      for (auto *PredBB : SpecPreds)
-        SpeculatedVals.push_back(IncomingValueMap.find(PredBB)->second);
-    }
-
-  // Speculate into each predecessor.
-  for (int PredIdx : llvm::seq<int>(0, SpecPreds.size())) {
-    auto *PredBB = SpecPreds[PredIdx];
-    assert(PredBB->getSingleSuccessor() == ParentBB &&
-           "We need a non-critical predecessor to speculate into.");
-
-    for (auto *OrigI : SpecList) {
-      auto *NewI = OrigI->clone();
-      NewI->setName(Twine(OrigI->getName()) + "." + Twine(PredIdx));
-      NewI->insertBefore(PredBB->getTerminator());
-
-      // Rewrite all the operands to the previously speculated instructions.
-      // Because we're walking in-order, the defs must precede the uses and we
-      // should already have these mappings.
-      for (Use &U : NewI->operands()) {
-        auto *OpI = dyn_cast<Instruction>(U.get());
-        if (!OpI)
-          continue;
-        auto MapIt = SpeculatedValueMap.find(OpI);
-        if (MapIt == SpeculatedValueMap.end())
-          continue;
-        const auto &SpeculatedVals = MapIt->second;
-        assert(SpeculatedVals[PredIdx] &&
-               "Must have a speculated value for this predecessor!");
-        assert(SpeculatedVals[PredIdx]->getType() == OpI->getType() &&
-               "Speculated value has the wrong type!");
-
-        // Rewrite the use to this predecessor's speculated instruction.
-        U.set(SpeculatedVals[PredIdx]);
-      }
-
-      // Commute instructions which now have a constant in the LHS but not the
-      // RHS.
-      if (NewI->isBinaryOp() && NewI->isCommutative() &&
-          isa<Constant>(NewI->getOperand(0)) &&
-          !isa<Constant>(NewI->getOperand(1)))
-        NewI->getOperandUse(0).swap(NewI->getOperandUse(1));
-
-      SpeculatedValueMap[OrigI].push_back(NewI);
-      assert(SpeculatedValueMap[OrigI][PredIdx] == NewI &&
-             "Mismatched speculated instruction index!");
-    }
-  }
-
-  // Walk the speculated instruction list and if they have uses, insert a PHI
-  // for them from the speculated versions, and replace the uses with the PHI.
-  // Then erase the instructions as they have been fully speculated. The walk
-  // needs to be in reverse so that we don't think there are users when we'll
-  // actually eventually remove them later.
-  IRBuilder<> IRB(SpecPNs[0]);
-  for (auto *OrigI : llvm::reverse(SpecList)) {
-    // Check if we need a PHI for any remaining users and if so, insert it.
-    if (!OrigI->use_empty()) {
-      auto *SpecIPN = IRB.CreatePHI(OrigI->getType(), SpecPreds.size(),
-                                    Twine(OrigI->getName()) + ".phi");
-      // Add the incoming values we speculated.
-      auto &SpeculatedVals = SpeculatedValueMap.find(OrigI)->second;
-      for (int PredIdx : llvm::seq<int>(0, SpecPreds.size()))
-        SpecIPN->addIncoming(SpeculatedVals[PredIdx], SpecPreds[PredIdx]);
-
-      // And replace the uses with the PHI node.
-      OrigI->replaceAllUsesWith(SpecIPN);
-    }
-
-    // It is important to immediately erase this so that it stops using other
-    // instructions. This avoids inserting needless PHIs of them.
-    OrigI->eraseFromParent();
-  }
-
-  // All of the uses of the speculated phi nodes should be removed at this
-  // point, so erase them.
-  for (auto *SpecPN : SpecPNs) {
-    assert(SpecPN->use_empty() && "All users should have been speculated!");
-    SpecPN->eraseFromParent();
-  }
-}
-
-/// Try to speculate around a series of PHIs from a single basic block.
-///
-/// This routine checks whether any of these PHIs are profitable to speculate
-/// users around. If safe and profitable, it does the speculation. It returns
-/// true when at least some speculation occurs.
-static bool tryToSpeculatePHIs(SmallVectorImpl<PHINode *> &PNs,
-                               DominatorTree &DT, TargetTransformInfo &TTI) {
-  LLVM_DEBUG(dbgs() << "Evaluating phi nodes for speculation:\n");
-
-  // Savings in cost from speculating around a PHI node.
-  SmallDenseMap<PHINode *, int, 16> CostSavingsMap;
-
-  // Remember the set of instructions that are candidates for speculation so
-  // that we can quickly walk things within that space. This prunes out
-  // instructions already available along edges, etc.
-  SmallPtrSet<Instruction *, 16> PotentialSpecSet;
-
-  // Remember the set of instructions that are (transitively) unsafe to
-  // speculate into the incoming edges of this basic block. This avoids
-  // recomputing them for each PHI node we check. This set is specific to this
-  // block though as things are pruned out of it based on what is available
-  // along incoming edges.
-  SmallPtrSet<Instruction *, 16> UnsafeSet;
-
-  // For each PHI node in this block, check whether there are immediate folding
-  // opportunities from speculation, and whether that speculation will be
-  // valid. This determise the set of safe PHIs to speculate.
-  llvm::erase_if(PNs, [&](PHINode *PN) {
-    return !isSafeAndProfitableToSpeculateAroundPHI(
-        *PN, CostSavingsMap, PotentialSpecSet, UnsafeSet, DT, TTI);
-  });
-  // If no PHIs were profitable, skip.
-  if (PNs.empty()) {
-    LLVM_DEBUG(dbgs() << "  No safe and profitable PHIs found!\n");
-    return false;
-  }
-
-  // We need to know how much speculation will cost which is determined by how
-  // many incoming edges will need a copy of each speculated instruction.
-  SmallSetVector<BasicBlock *, 16> PredSet;
-  for (auto *PredBB : PNs[0]->blocks()) {
-    if (!PredSet.insert(PredBB))
-      continue;
-
-    // We cannot speculate when a predecessor is an indirect branch.
-    // FIXME: We also can't reliably create a non-critical edge block for
-    // speculation if the predecessor is an invoke. This doesn't seem
-    // fundamental and we should probably be splitting critical edges
-    // differently.
-    const auto *TermInst = PredBB->getTerminator();
-    if (isa<IndirectBrInst>(TermInst) ||
-        isa<InvokeInst>(TermInst) ||
-        isa<CallBrInst>(TermInst)) {
-      LLVM_DEBUG(dbgs() << "  Invalid: predecessor terminator: "
-                        << PredBB->getName() << "\n");
-      return false;
-    }
-  }
-  if (PredSet.size() < 2) {
-    LLVM_DEBUG(dbgs() << "  Unimportant: phi with only one predecessor\n");
-    return false;
-  }
-
-  SmallVector<PHINode *, 16> SpecPNs = findProfitablePHIs(
-      PNs, CostSavingsMap, PotentialSpecSet, PredSet.size(), DT, TTI);
-  if (SpecPNs.empty())
-    // Nothing to do.
-    return false;
-
-  speculatePHIs(SpecPNs, PotentialSpecSet, PredSet, DT);
-  return true;
-}
-
-PreservedAnalyses SpeculateAroundPHIsPass::run(Function &F,
-                                               FunctionAnalysisManager &AM) {
-  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
-  auto &TTI = AM.getResult<TargetIRAnalysis>(F);
-
-  bool Changed = false;
-  for (auto *BB : ReversePostOrderTraversal<Function *>(&F)) {
-    SmallVector<PHINode *, 16> PNs;
-    auto BBI = BB->begin();
-    while (auto *PN = dyn_cast<PHINode>(&*BBI)) {
-      PNs.push_back(PN);
-      ++BBI;
-    }
-
-    if (PNs.empty())
-      continue;
-
-    Changed |= tryToSpeculatePHIs(PNs, DT, TTI);
-  }
-
-  if (!Changed)
-    return PreservedAnalyses::all();
-
-  PreservedAnalyses PA;
-  return PA;
-}
diff --git a/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp b/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp
index c78185f2a6ad..dfa30418ea01 100644
--- a/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp
+++ b/llvm/lib/Transforms/Scalar/SpeculativeExecution.cpp
@@ -210,8 +210,8 @@ bool SpeculativeExecutionPass::runOnBasicBlock(BasicBlock &B) {
   return false;
 }
 
-static unsigned ComputeSpeculationCost(const Instruction *I,
-                                       const TargetTransformInfo &TTI) {
+static InstructionCost ComputeSpeculationCost(const Instruction *I,
+                                              const TargetTransformInfo &TTI) {
   switch (Operator::getOpcode(I)) {
     case Instruction::GetElementPtr:
     case Instruction::Add:
@@ -255,7 +255,8 @@ static unsigned ComputeSpeculationCost(const Instruction *I,
       return TTI.getUserCost(I, TargetTransformInfo::TCK_SizeAndLatency);
 
     default:
-      return UINT_MAX; // Disallow anything not explicitly listed.
+      return InstructionCost::getInvalid(); // Disallow anything not explicitly
+                                            // listed.
   }
 }
 
@@ -265,11 +266,13 @@ bool SpeculativeExecutionPass::considerHoistingFromTo(
   const auto AllPrecedingUsesFromBlockHoisted = [&NotHoisted](const User *U) {
     // Debug variable has special operand to check it's not hoisted.
     if (const auto *DVI = dyn_cast<DbgVariableIntrinsic>(U)) {
-      if (const auto *I =
-              dyn_cast_or_null<Instruction>(DVI->getVariableLocation()))
-        if (NotHoisted.count(I) == 0)
-          return true;
-      return false;
+      return all_of(DVI->location_ops(), [&NotHoisted](Value *V) {
+        if (const auto *I = dyn_cast_or_null<Instruction>(V)) {
+          if (NotHoisted.count(I) == 0)
+            return true;
+        }
+        return false;
+      });
     }
 
     // Usially debug label instrinsic corresponds to label in LLVM IR. In these
@@ -288,11 +291,11 @@ bool SpeculativeExecutionPass::considerHoistingFromTo(
     return true;
   };
 
-  unsigned TotalSpeculationCost = 0;
+  InstructionCost TotalSpeculationCost = 0;
   unsigned NotHoistedInstCount = 0;
   for (const auto &I : FromBlock) {
-    const unsigned Cost = ComputeSpeculationCost(&I, *TTI);
-    if (Cost != UINT_MAX && isSafeToSpeculativelyExecute(&I) &&
+    const InstructionCost Cost = ComputeSpeculationCost(&I, *TTI);
+    if (Cost.isValid() && isSafeToSpeculativelyExecute(&I) &&
         AllPrecedingUsesFromBlockHoisted(&I)) {
       TotalSpeculationCost += Cost;
       if (TotalSpeculationCost > SpecExecMaxSpeculationCost)
@@ -340,7 +343,6 @@ PreservedAnalyses SpeculativeExecutionPass::run(Function &F,
   if (!Changed)
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
-  PA.preserve<GlobalsAA>();
   PA.preserveSet<CFGAnalyses>();
   return PA;
 }
diff --git a/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp b/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
index 577992ccb5f4..20b8b982e14b 100644
--- a/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
+++ b/llvm/lib/Transforms/Scalar/StraightLineStrengthReduce.cpp
@@ -312,8 +312,8 @@ bool StraightLineStrengthReduce::isFoldable(const Candidate &C,
 // Returns true if GEP has zero or one non-zero index.
 static bool hasOnlyOneNonZeroIndex(GetElementPtrInst *GEP) {
   unsigned NumNonZeroIndices = 0;
-  for (auto I = GEP->idx_begin(); I != GEP->idx_end(); ++I) {
-    ConstantInt *ConstIdx = dyn_cast<ConstantInt>(*I);
+  for (Use &Idx : GEP->indices()) {
+    ConstantInt *ConstIdx = dyn_cast<ConstantInt>(Idx);
     if (ConstIdx == nullptr || !ConstIdx->isZero())
       ++NumNonZeroIndices;
   }
@@ -533,8 +533,8 @@ void StraightLineStrengthReduce::allocateCandidatesAndFindBasisForGEP(
     return;
 
   SmallVector<const SCEV *, 4> IndexExprs;
-  for (auto I = GEP->idx_begin(); I != GEP->idx_end(); ++I)
-    IndexExprs.push_back(SE->getSCEV(*I));
+  for (Use &Idx : GEP->indices())
+    IndexExprs.push_back(SE->getSCEV(Idx));
 
   gep_type_iterator GTI = gep_type_begin(GEP);
   for (unsigned I = 1, E = GEP->getNumOperands(); I != E; ++I, ++GTI) {
diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
index 3e15cad5f3f3..ac580b4161f4 100644
--- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
+++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp
@@ -677,9 +677,8 @@ void StructurizeCFG::killTerminator(BasicBlock *BB) {
   if (!Term)
     return;
 
-  for (succ_iterator SI = succ_begin(BB), SE = succ_end(BB);
-       SI != SE; ++SI)
-    delPhiValues(BB, *SI);
+  for (BasicBlock *Succ : successors(BB))
+    delPhiValues(BB, Succ);
 
   if (DA)
     DA->removeValue(Term);
@@ -694,11 +693,9 @@ void StructurizeCFG::changeExit(RegionNode *Node, BasicBlock *NewExit,
     BasicBlock *OldExit = SubRegion->getExit();
     BasicBlock *Dominator = nullptr;
 
-    // Find all the edges from the sub region to the exit
-    for (auto BBI = pred_begin(OldExit), E = pred_end(OldExit); BBI != E;) {
-      // Incrememt BBI before mucking with BB's terminator.
-      BasicBlock *BB = *BBI++;
-
+    // Find all the edges from the sub region to the exit.
+    // We use make_early_inc_range here because we modify BB's terminator.
+    for (BasicBlock *BB : llvm::make_early_inc_range(predecessors(OldExit))) {
       if (!SubRegion->contains(BB))
         continue;
 
@@ -924,10 +921,9 @@ void StructurizeCFG::rebuildSSA() {
   for (BasicBlock *BB : ParentRegion->blocks())
     for (Instruction &I : *BB) {
       bool Initialized = false;
-      // We may modify the use list as we iterate over it, so be careful to
-      // compute the next element in the use list at the top of the loop.
-      for (auto UI = I.use_begin(), E = I.use_end(); UI != E;) {
-        Use &U = *UI++;
+      // We may modify the use list as we iterate over it, so we use
+      // make_early_inc_range.
+      for (Use &U : llvm::make_early_inc_range(I.uses())) {
         Instruction *User = cast<Instruction>(U.getUser());
         if (User->getParent() == BB) {
           continue;
diff --git a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
index 9e7cccc88412..846a9321f53e 100644
--- a/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/TailRecursionElimination.cpp
@@ -63,6 +63,7 @@
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/PostDominators.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/IR/CFG.h"
 #include "llvm/IR/Constants.h"
 #include "llvm/IR/DataLayout.h"
@@ -70,6 +71,7 @@
 #include "llvm/IR/DiagnosticInfo.h"
 #include "llvm/IR/Dominators.h"
 #include "llvm/IR/Function.h"
+#include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/InstIterator.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/IR/IntrinsicInst.h"
@@ -81,6 +83,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
 
 #define DEBUG_TYPE "tailcallelim"
@@ -92,10 +95,10 @@ STATISTIC(NumAccumAdded, "Number of accumulators introduced");
 /// Scan the specified function for alloca instructions.
 /// If it contains any dynamic allocas, returns false.
 static bool canTRE(Function &F) {
-  // FIXME: The code generator produces really bad code when an 'escaping
-  // alloca' is changed from being a static alloca to being a dynamic alloca.
-  // Until this is resolved, disable this transformation if that would ever
-  // happen.  This bug is PR962.
+  // TODO: We don't do TRE if dynamic allocas are used.
+  // Dynamic allocas allocate stack space which should be
+  // deallocated before new iteration started. That is
+  // currently not implemented.
   return llvm::all_of(instructions(F), [](Instruction &I) {
     auto *AI = dyn_cast<AllocaInst>(&I);
     return !AI || AI->isStaticAlloca();
@@ -188,11 +191,9 @@ struct AllocaDerivedValueTracker {
 };
 }
 
-static bool markTails(Function &F, bool &AllCallsAreTailCalls,
-                      OptimizationRemarkEmitter *ORE) {
+static bool markTails(Function &F, OptimizationRemarkEmitter *ORE) {
   if (F.callsFunctionThatReturnsTwice())
     return false;
-  AllCallsAreTailCalls = true;
 
   // The local stack holds all alloca instructions and all byval arguments.
   AllocaDerivedValueTracker Tracker;
@@ -247,7 +248,10 @@ static bool markTails(Function &F, bool &AllCallsAreTailCalls,
           isa<PseudoProbeInst>(&I))
         continue;
 
-      bool IsNoTail = CI->isNoTailCall() || CI->hasOperandBundles();
+      // Special-case operand bundle "clang.arc.attachedcall".
+      bool IsNoTail =
+          CI->isNoTailCall() || CI->hasOperandBundlesOtherThan(
+                                    LLVMContext::OB_clang_arc_attachedcall);
 
       if (!IsNoTail && CI->doesNotAccessMemory()) {
         // A call to a readnone function whose arguments are all things computed
@@ -279,11 +283,8 @@ static bool markTails(Function &F, bool &AllCallsAreTailCalls,
         }
       }
 
-      if (!IsNoTail && Escaped == UNESCAPED && !Tracker.AllocaUsers.count(CI)) {
+      if (!IsNoTail && Escaped == UNESCAPED && !Tracker.AllocaUsers.count(CI))
         DeferredTails.push_back(CI);
-      } else {
-        AllCallsAreTailCalls = false;
-      }
     }
 
     for (auto *SuccBB : successors(BB)) {
@@ -320,8 +321,6 @@ static bool markTails(Function &F, bool &AllCallsAreTailCalls,
       LLVM_DEBUG(dbgs() << "Marked as tail call candidate: " << *CI << "\n");
       CI->setTailCall();
       Modified = true;
-    } else {
-      AllCallsAreTailCalls = false;
     }
   }
 
@@ -333,6 +332,14 @@ static bool markTails(Function &F, bool &AllCallsAreTailCalls,
 /// instructions between the call and this instruction are movable.
 ///
 static bool canMoveAboveCall(Instruction *I, CallInst *CI, AliasAnalysis *AA) {
+  if (isa<DbgInfoIntrinsic>(I))
+    return true;
+
+  if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(I))
+    if (II->getIntrinsicID() == Intrinsic::lifetime_end &&
+        llvm::findAllocaForValue(II->getArgOperand(1)))
+      return true;
+
   // FIXME: We can move load/store/call/free instructions above the call if the
   // call does not mod/ref the memory location being processed.
   if (I->mayHaveSideEffects())  // This also handles volatile loads.
@@ -399,7 +406,6 @@ class TailRecursionEliminator {
   // createTailRecurseLoopHeader the first time we find a call we can eliminate.
   BasicBlock *HeaderBB = nullptr;
   SmallVector<PHINode *, 8> ArgumentPHIs;
-  bool RemovableCallsMustBeMarkedTail = false;
 
   // PHI node to store our return value.
   PHINode *RetPN = nullptr;
@@ -426,8 +432,7 @@ class TailRecursionEliminator {
                           DomTreeUpdater &DTU)
       : F(F), TTI(TTI), AA(AA), ORE(ORE), DTU(DTU) {}
 
-  CallInst *findTRECandidate(BasicBlock *BB,
-                             bool CannotTailCallElimCallsMarkedTail);
+  CallInst *findTRECandidate(BasicBlock *BB);
 
   void createTailRecurseLoopHeader(CallInst *CI);
 
@@ -437,7 +442,11 @@ class TailRecursionEliminator {
 
   void cleanupAndFinalize();
 
-  bool processBlock(BasicBlock &BB, bool CannotTailCallElimCallsMarkedTail);
+  bool processBlock(BasicBlock &BB);
+
+  void copyByValueOperandIntoLocalTemp(CallInst *CI, int OpndIdx);
+
+  void copyLocalTempOfByValueOperandIntoArguments(CallInst *CI, int OpndIdx);
 
 public:
   static bool eliminate(Function &F, const TargetTransformInfo *TTI,
@@ -446,8 +455,7 @@ public:
 };
 } // namespace
 
-CallInst *TailRecursionEliminator::findTRECandidate(
-    BasicBlock *BB, bool CannotTailCallElimCallsMarkedTail) {
+CallInst *TailRecursionEliminator::findTRECandidate(BasicBlock *BB) {
   Instruction *TI = BB->getTerminator();
 
   if (&BB->front() == TI) // Make sure there is something before the terminator.
@@ -467,9 +475,9 @@ CallInst *TailRecursionEliminator::findTRECandidate(
     --BBI;
   }
 
-  // If this call is marked as a tail call, and if there are dynamic allocas in
-  // the function, we cannot perform this optimization.
-  if (CI->isTailCall() && CannotTailCallElimCallsMarkedTail)
+  assert((!CI->isTailCall() || !CI->isNoTailCall()) &&
+         "Incompatible call site attributes(Tail,NoTail)");
+  if (!CI->isTailCall())
     return nullptr;
 
   // As a special case, detect code like this:
@@ -501,26 +509,13 @@ void TailRecursionEliminator::createTailRecurseLoopHeader(CallInst *CI) {
   BranchInst *BI = BranchInst::Create(HeaderBB, NewEntry);
   BI->setDebugLoc(CI->getDebugLoc());
 
-  // If this function has self recursive calls in the tail position where some
-  // are marked tail and some are not, only transform one flavor or another.
-  // We have to choose whether we move allocas in the entry block to the new
-  // entry block or not, so we can't make a good choice for both. We make this
-  // decision here based on whether the first call we found to remove is
-  // marked tail.
-  // NOTE: We could do slightly better here in the case that the function has
-  // no entry block allocas.
-  RemovableCallsMustBeMarkedTail = CI->isTailCall();
-
-  // If this tail call is marked 'tail' and if there are any allocas in the
-  // entry block, move them up to the new entry block.
-  if (RemovableCallsMustBeMarkedTail)
-    // Move all fixed sized allocas from HeaderBB to NewEntry.
-    for (BasicBlock::iterator OEBI = HeaderBB->begin(), E = HeaderBB->end(),
-                              NEBI = NewEntry->begin();
-         OEBI != E;)
-      if (AllocaInst *AI = dyn_cast<AllocaInst>(OEBI++))
-        if (isa<ConstantInt>(AI->getArraySize()))
-          AI->moveBefore(&*NEBI);
+  // Move all fixed sized allocas from HeaderBB to NewEntry.
+  for (BasicBlock::iterator OEBI = HeaderBB->begin(), E = HeaderBB->end(),
+                            NEBI = NewEntry->begin();
+       OEBI != E;)
+    if (AllocaInst *AI = dyn_cast<AllocaInst>(OEBI++))
+      if (isa<ConstantInt>(AI->getArraySize()))
+        AI->moveBefore(&*NEBI);
 
   // Now that we have created a new block, which jumps to the entry
   // block, insert a PHI node for each argument of the function.
@@ -585,6 +580,54 @@ void TailRecursionEliminator::insertAccumulator(Instruction *AccRecInstr) {
   ++NumAccumAdded;
 }
 
+// Creates a copy of contents of ByValue operand of the specified
+// call instruction into the newly created temporarily variable.
+void TailRecursionEliminator::copyByValueOperandIntoLocalTemp(CallInst *CI,
+                                                              int OpndIdx) {
+  PointerType *ArgTy = cast<PointerType>(CI->getArgOperand(OpndIdx)->getType());
+  Type *AggTy = ArgTy->getElementType();
+  const DataLayout &DL = F.getParent()->getDataLayout();
+
+  // Get alignment of byVal operand.
+  Align Alignment(CI->getParamAlign(OpndIdx).valueOrOne());
+
+  // Create alloca for temporarily byval operands.
+  // Put alloca into the entry block.
+  Value *NewAlloca = new AllocaInst(
+      AggTy, DL.getAllocaAddrSpace(), nullptr, Alignment,
+      CI->getArgOperand(OpndIdx)->getName(), &*F.getEntryBlock().begin());
+
+  IRBuilder<> Builder(CI);
+  Value *Size = Builder.getInt64(DL.getTypeAllocSize(AggTy));
+
+  // Copy data from byvalue operand into the temporarily variable.
+  Builder.CreateMemCpy(NewAlloca, /*DstAlign*/ Alignment,
+                       CI->getArgOperand(OpndIdx),
+                       /*SrcAlign*/ Alignment, Size);
+  CI->setArgOperand(OpndIdx, NewAlloca);
+}
+
+// Creates a copy from temporarily variable(keeping value of ByVal argument)
+// into the corresponding function argument location.
+void TailRecursionEliminator::copyLocalTempOfByValueOperandIntoArguments(
+    CallInst *CI, int OpndIdx) {
+  PointerType *ArgTy = cast<PointerType>(CI->getArgOperand(OpndIdx)->getType());
+  Type *AggTy = ArgTy->getElementType();
+  const DataLayout &DL = F.getParent()->getDataLayout();
+
+  // Get alignment of byVal operand.
+  Align Alignment(CI->getParamAlign(OpndIdx).valueOrOne());
+
+  IRBuilder<> Builder(CI);
+  Value *Size = Builder.getInt64(DL.getTypeAllocSize(AggTy));
+
+  // Copy data from the temporarily variable into corresponding
+  // function argument location.
+  Builder.CreateMemCpy(F.getArg(OpndIdx), /*DstAlign*/ Alignment,
+                       CI->getArgOperand(OpndIdx),
+                       /*SrcAlign*/ Alignment, Size);
+}
+
 bool TailRecursionEliminator::eliminateCall(CallInst *CI) {
   ReturnInst *Ret = cast<ReturnInst>(CI->getParent()->getTerminator());
 
@@ -623,14 +666,22 @@ bool TailRecursionEliminator::eliminateCall(CallInst *CI) {
   if (!HeaderBB)
     createTailRecurseLoopHeader(CI);
 
-  if (RemovableCallsMustBeMarkedTail && !CI->isTailCall())
-    return false;
+  // Copy values of ByVal operands into local temporarily variables.
+  for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) {
+    if (CI->isByValArgument(I))
+      copyByValueOperandIntoLocalTemp(CI, I);
+  }
 
   // Ok, now that we know we have a pseudo-entry block WITH all of the
   // required PHI nodes, add entries into the PHI node for the actual
   // parameters passed into the tail-recursive call.
-  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i)
-    ArgumentPHIs[i]->addIncoming(CI->getArgOperand(i), BB);
+  for (unsigned I = 0, E = CI->getNumArgOperands(); I != E; ++I) {
+    if (CI->isByValArgument(I)) {
+      copyLocalTempOfByValueOperandIntoArguments(CI, I);
+      ArgumentPHIs[I]->addIncoming(F.getArg(I), BB);
+    } else
+      ArgumentPHIs[I]->addIncoming(CI->getArgOperand(I), BB);
+  }
 
   if (AccRecInstr) {
     insertAccumulator(AccRecInstr);
@@ -747,8 +798,7 @@ void TailRecursionEliminator::cleanupAndFinalize() {
   }
 }
 
-bool TailRecursionEliminator::processBlock(
-    BasicBlock &BB, bool CannotTailCallElimCallsMarkedTail) {
+bool TailRecursionEliminator::processBlock(BasicBlock &BB) {
   Instruction *TI = BB.getTerminator();
 
   if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
@@ -761,7 +811,7 @@ bool TailRecursionEliminator::processBlock(
     if (!Ret)
       return false;
 
-    CallInst *CI = findTRECandidate(&BB, CannotTailCallElimCallsMarkedTail);
+    CallInst *CI = findTRECandidate(&BB);
 
     if (!CI)
       return false;
@@ -782,7 +832,7 @@ bool TailRecursionEliminator::processBlock(
     eliminateCall(CI);
     return true;
   } else if (isa<ReturnInst>(TI)) {
-    CallInst *CI = findTRECandidate(&BB, CannotTailCallElimCallsMarkedTail);
+    CallInst *CI = findTRECandidate(&BB);
 
     if (CI)
       return eliminateCall(CI);
@@ -796,30 +846,25 @@ bool TailRecursionEliminator::eliminate(Function &F,
                                         AliasAnalysis *AA,
                                         OptimizationRemarkEmitter *ORE,
                                         DomTreeUpdater &DTU) {
-  if (F.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
+  if (F.getFnAttribute("disable-tail-calls").getValueAsBool())
     return false;
 
   bool MadeChange = false;
-  bool AllCallsAreTailCalls = false;
-  MadeChange |= markTails(F, AllCallsAreTailCalls, ORE);
-  if (!AllCallsAreTailCalls)
-    return MadeChange;
+  MadeChange |= markTails(F, ORE);
 
   // If this function is a varargs function, we won't be able to PHI the args
   // right, so don't even try to convert it...
   if (F.getFunctionType()->isVarArg())
     return MadeChange;
 
-  // If false, we cannot perform TRE on tail calls marked with the 'tail'
-  // attribute, because doing so would cause the stack size to increase (real
-  // TRE would deallocate variable sized allocas, TRE doesn't).
-  bool CanTRETailMarkedCall = canTRE(F);
+  if (!canTRE(F))
+    return MadeChange;
 
   // Change any tail recursive calls to loops.
   TailRecursionEliminator TRE(F, TTI, AA, ORE, DTU);
 
   for (BasicBlock &BB : F)
-    MadeChange |= TRE.processBlock(BB, !CanTRETailMarkedCall);
+    MadeChange |= TRE.processBlock(BB);
 
   TRE.cleanupAndFinalize();
 
@@ -893,7 +938,6 @@ PreservedAnalyses TailCallElimPass::run(Function &F,
   if (!Changed)
     return PreservedAnalyses::all();
   PreservedAnalyses PA;
-  PA.preserve<GlobalsAA>();
   PA.preserve<DominatorTreeAnalysis>();
   PA.preserve<PostDominatorTreeAnalysis>();
   return PA;