vendor/llvm/llvm-trunk-r238337

66 files changed, 2977 insertions, 2037 deletions

diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h
index e3ce57ad18502..47201e2564e3c 100644
--- a/include/llvm/CodeGen/AsmPrinter.h
+++ b/include/llvm/CodeGen/AsmPrinter.h
@@ -16,8 +16,10 @@
 #ifndef LLVM_CODEGEN_ASMPRINTER_H
 #define LLVM_CODEGEN_ASMPRINTER_H
 
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/DwarfStringPoolEntry.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -29,6 +31,8 @@ class ByteStreamer;
 class GCStrategy;
 class Constant;
 class ConstantArray;
+class DIE;
+class DIEAbbrev;
 class GCMetadataPrinter;
 class GlobalValue;
 class GlobalVariable;
@@ -46,11 +50,11 @@ class MCCFIInstruction;
 class MCContext;
 class MCExpr;
 class MCInst;
-class MCInstrInfo;
 class MCSection;
 class MCStreamer;
 class MCSubtargetInfo;
 class MCSymbol;
+class MCTargetOptions;
 class MDNode;
 class DwarfDebug;
 class Mangler;
@@ -69,7 +73,6 @@ public:
   ///
   const MCAsmInfo *MAI;
 
-  const MCInstrInfo *MII;
   /// This is the context for the output file that we are streaming. This owns
   /// all of the global MC-related objects for the generated translation unit.
   MCContext &OutContext;
@@ -77,7 +80,7 @@ public:
   /// This is the MCStreamer object for the file we are generating. This
   /// contains the transient state for the current translation unit that we are
   /// generating (such as the current section etc).
-  MCStreamer &OutStreamer;
+  std::unique_ptr<MCStreamer> OutStreamer;
 
   /// The current machine function.
   const MachineFunction *MF;
@@ -99,7 +102,16 @@ public:
   /// default, this is equal to CurrentFnSym.
   MCSymbol *CurrentFnSymForSize;
 
+  /// Map global GOT equivalent MCSymbols to GlobalVariables and keep track of
+  /// its number of uses by other globals.
+  typedef std::pair<const GlobalVariable *, unsigned> GOTEquivUsePair;
+  MapVector<const MCSymbol *, GOTEquivUsePair> GlobalGOTEquivs;
+
 private:
+  MCSymbol *CurrentFnBegin;
+  MCSymbol *CurrentFnEnd;
+  MCSymbol *CurExceptionSym;
+
   // The garbage collection metadata printer table.
   void *GCMetadataPrinters; // Really a DenseMap.
 
@@ -127,10 +139,10 @@ private:
   DwarfDebug *DD;
 
 protected:
-  explicit AsmPrinter(TargetMachine &TM, MCStreamer &Streamer);
+  explicit AsmPrinter(TargetMachine &TM, std::unique_ptr<MCStreamer> Streamer);
 
 public:
-  virtual ~AsmPrinter();
+  ~AsmPrinter() override;
 
   DwarfDebug *getDwarfDebug() { return DD; }
   DwarfDebug *getDwarfDebug() const { return DD; }
@@ -143,6 +155,10 @@ public:
   ///
   unsigned getFunctionNumber() const;
 
+  MCSymbol *getFunctionBegin() const { return CurrentFnBegin; }
+  MCSymbol *getFunctionEnd() const { return CurrentFnEnd; }
+  MCSymbol *getCurExceptionSym();
+
   /// Return information about object file lowering.
   const TargetLoweringObjectFile &getObjFileLowering() const;
 
@@ -184,7 +200,6 @@ public:
   /// Emit the specified function out to the OutStreamer.
   bool runOnMachineFunction(MachineFunction &MF) override {
     SetupMachineFunction(MF);
-    EmitFunctionHeader();
     EmitFunctionBody();
     return false;
   }
@@ -197,9 +212,6 @@ public:
   /// runOnMachineFunction.
   void SetupMachineFunction(MachineFunction &MF);
 
-  /// This method emits the header for the current function.
-  void EmitFunctionHeader();
-
   /// This method emits the body and trailer for a function.
   void EmitFunctionBody();
 
@@ -238,16 +250,27 @@ public:
   ///
   void EmitAlignment(unsigned NumBits, const GlobalObject *GO = nullptr) const;
 
-  /// This method prints the label for the specified MachineBasicBlock, an
-  /// alignment (if present) and a comment describing it if appropriate.
-  void EmitBasicBlockStart(const MachineBasicBlock &MBB) const;
-
   /// Lower the specified LLVM Constant to an MCExpr.
   const MCExpr *lowerConstant(const Constant *CV);
 
   /// \brief Print a general LLVM constant to the .s file.
   void EmitGlobalConstant(const Constant *CV);
 
+  /// \brief Unnamed constant global variables solely contaning a pointer to
+  /// another globals variable act like a global variable "proxy", or GOT
+  /// equivalents, i.e., it's only used to hold the address of the latter. One
+  /// optimization is to replace accesses to these proxies by using the GOT
+  /// entry for the final global instead. Hence, we select GOT equivalent
+  /// candidates among all the module global variables, avoid emitting them
+  /// unnecessarily and finally replace references to them by pc relative
+  /// accesses to GOT entries.
+  void computeGlobalGOTEquivs(Module &M);
+
+  /// \brief Constant expressions using GOT equivalent globals may not be
+  /// eligible for PC relative GOT entry conversion, in such cases we need to
+  /// emit the proxies we previously omitted in EmitGlobalVariable.
+  void emitGlobalGOTEquivs();
+
   //===------------------------------------------------------------------===//
   // Overridable Hooks
   //===------------------------------------------------------------------===//
@@ -271,6 +294,12 @@ public:
   /// function.
   virtual void EmitFunctionBodyEnd() {}
 
+  /// Targets can override this to emit stuff at the start of a basic block.
+  /// By default, this method prints the label for the specified
+  /// MachineBasicBlock, an alignment (if present) and a comment describing it
+  /// if appropriate.
+  virtual void EmitBasicBlockStart(const MachineBasicBlock &MBB) const;
+
   /// Targets can override this to emit stuff at the end of a basic block.
   virtual void EmitBasicBlockEnd(const MachineBasicBlock &MBB) {}
 
@@ -304,12 +333,7 @@ public:
   // Symbol Lowering Routines.
   //===------------------------------------------------------------------===//
 public:
-  /// Return the MCSymbol corresponding to the assembler temporary label with
-  /// the specified stem and unique ID.
-  MCSymbol *GetTempSymbol(Twine Name, unsigned ID) const;
-
-  /// Return an assembler temporary label with the specified stem.
-  MCSymbol *GetTempSymbol(Twine Name) const;
+  MCSymbol *createTempSymbol(const Twine &Name) const;
 
   /// Return the MCSymbol for a private symbol with global value name as its
   /// base, with the specified suffix.
@@ -397,42 +421,21 @@ public:
   /// Emit the 4-byte offset of Label from the start of its section.  This can
   /// be done with a special directive if the target supports it (e.g. cygwin)
   /// or by emitting it as an offset from a label at the start of the section.
+  void emitSectionOffset(const MCSymbol *Label) const;
+
+  /// Emit the 4-byte offset of a string from the start of its section.
   ///
-  /// SectionLabel is a temporary label emitted at the start of the section
-  /// that Label lives in.
-  void EmitSectionOffset(const MCSymbol *Label,
-                         const MCSymbol *SectionLabel) const;
+  /// When possible, emit a DwarfStringPool section offset without any
+  /// relocations, and without using the symbol.  Otherwise, defers to \a
+  /// emitSectionOffset().
+  void emitDwarfStringOffset(DwarfStringPoolEntryRef S) const;
 
   /// Get the value for DW_AT_APPLE_isa. Zero if no isa encoding specified.
   virtual unsigned getISAEncoding() { return 0; }
 
-  /// Emit a dwarf register operation for describing
-  /// - a small value occupying only part of a register or
-  /// - a register representing only part of a value.
-  void EmitDwarfOpPiece(ByteStreamer &Streamer, unsigned SizeInBits,
-                        unsigned OffsetInBits = 0) const;
-
-
-  /// \brief Emit a partial DWARF register operation.
-  /// \param MLoc             the register
-  /// \param PieceSize        size and
-  /// \param PieceOffset      offset of the piece in bits, if this is one
-  ///                         piece of an aggregate value.
-  ///
-  /// If size and offset is zero an operation for the entire
-  /// register is emitted: Some targets do not provide a DWARF
-  /// register number for every register.  If this is the case, this
-  /// function will attempt to emit a DWARF register by emitting a
-  /// piece of a super-register or by piecing together multiple
-  /// subregisters that alias the register.
-  void EmitDwarfRegOpPiece(ByteStreamer &BS, const MachineLocation &MLoc,
-                           unsigned PieceSize = 0,
-                           unsigned PieceOffset = 0) const;
-
   /// EmitDwarfRegOp - Emit a dwarf register operation.
-  /// \param Indirect   whether this is a register-indirect address
-  virtual void EmitDwarfRegOp(ByteStreamer &BS, const MachineLocation &MLoc,
-                              bool Indirect) const;
+  virtual void EmitDwarfRegOp(ByteStreamer &BS,
+                              const MachineLocation &MLoc) const;
 
   //===------------------------------------------------------------------===//
   // Dwarf Lowering Routines
@@ -441,6 +444,12 @@ public:
   /// \brief Emit frame instruction to describe the layout of the frame.
   void emitCFIInstruction(const MCCFIInstruction &Inst) const;
 
+  /// \brief Emit Dwarf abbreviation table.
+  void emitDwarfAbbrevs(const std::vector<DIEAbbrev *>& Abbrevs) const;
+
+  /// \brief Recursively emit Dwarf DIE tree.
+  void emitDwarfDIE(const DIE &Die) const;
+
   //===------------------------------------------------------------------===//
   // Inline Asm Support
   //===------------------------------------------------------------------===//
@@ -474,7 +483,7 @@ public:
 
   /// Let the target do anything it needs to do before emitting inlineasm.
   /// \p StartInfo - the subtarget info before parsing inline asm
-  virtual void emitInlineAsmStart(const MCSubtargetInfo &StartInfo) const;
+  virtual void emitInlineAsmStart() const;
 
   /// Let the target do anything it needs to do after emitting inlineasm.
   /// This callback can be used restore the original mode in case the
@@ -491,11 +500,15 @@ private:
   mutable const MachineInstr *LastMI;
   mutable unsigned LastFn;
   mutable unsigned Counter;
-  mutable unsigned SetCounter;
+
+  /// This method emits the header for the current function.
+  virtual void EmitFunctionHeader();
 
   /// Emit a blob of inline asm to the output streamer.
   void
-  EmitInlineAsm(StringRef Str, const MDNode *LocMDNode = nullptr,
+  EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI,
+                const MCTargetOptions &MCOptions,
+                const MDNode *LocMDNode = nullptr,
                 InlineAsm::AsmDialect AsmDialect = InlineAsm::AD_ATT) const;
 
   /// This method formats and emits the specified machine instruction that is an
diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h
new file mode 100644
index 0000000000000..d07265560430e
--- /dev/null
+++ b/include/llvm/CodeGen/BasicTTIImpl.h
@@ -0,0 +1,767 @@
+//===- BasicTTIImpl.h -------------------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This file provides a helper that implements much of the TTI interface in
+/// terms of the target-independent code generator and TargetLowering
+/// interfaces.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_BASICTTIIMPL_H
+#define LLVM_CODEGEN_BASICTTIIMPL_H
+
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/TargetTransformInfoImpl.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+#include "llvm/Analysis/TargetLibraryInfo.h"
+
+namespace llvm {
+
+extern cl::opt<unsigned> PartialUnrollingThreshold;
+
+/// \brief Base class which can be used to help build a TTI implementation.
+///
+/// This class provides as much implementation of the TTI interface as is
+/// possible using the target independent parts of the code generator.
+///
+/// In order to subclass it, your class must implement a getST() method to
+/// return the subtarget, and a getTLI() method to return the target lowering.
+/// We need these methods implemented in the derived class so that this class
+/// doesn't have to duplicate storage for them.
+template <typename T>
+class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
+private:
+  typedef TargetTransformInfoImplCRTPBase<T> BaseT;
+  typedef TargetTransformInfo TTI;
+
+  /// Estimate the overhead of scalarizing an instruction. Insert and Extract
+  /// are set if the result needs to be inserted and/or extracted from vectors.
+  unsigned getScalarizationOverhead(Type *Ty, bool Insert, bool Extract) {
+    assert(Ty->isVectorTy() && "Can only scalarize vectors");
+    unsigned Cost = 0;
+
+    for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) {
+      if (Insert)
+        Cost += static_cast<T *>(this)
+                    ->getVectorInstrCost(Instruction::InsertElement, Ty, i);
+      if (Extract)
+        Cost += static_cast<T *>(this)
+                    ->getVectorInstrCost(Instruction::ExtractElement, Ty, i);
+    }
+
+    return Cost;
+  }
+
+  /// Estimate the cost overhead of SK_Alternate shuffle.
+  unsigned getAltShuffleOverhead(Type *Ty) {
+    assert(Ty->isVectorTy() && "Can only shuffle vectors");
+    unsigned Cost = 0;
+    // Shuffle cost is equal to the cost of extracting element from its argument
+    // plus the cost of inserting them onto the result vector.
+
+    // e.g. <4 x float> has a mask of <0,5,2,7> i.e we need to extract from
+    // index 0 of first vector, index 1 of second vector,index 2 of first
+    // vector and finally index 3 of second vector and insert them at index
+    // <0,1,2,3> of result vector.
+    for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) {
+      Cost += static_cast<T *>(this)
+                  ->getVectorInstrCost(Instruction::InsertElement, Ty, i);
+      Cost += static_cast<T *>(this)
+                  ->getVectorInstrCost(Instruction::ExtractElement, Ty, i);
+    }
+    return Cost;
+  }
+
+  /// \brief Local query method delegates up to T which *must* implement this!
+  const TargetSubtargetInfo *getST() const {
+    return static_cast<const T *>(this)->getST();
+  }
+
+  /// \brief Local query method delegates up to T which *must* implement this!
+  const TargetLoweringBase *getTLI() const {
+    return static_cast<const T *>(this)->getTLI();
+  }
+
+protected:
+  explicit BasicTTIImplBase(const TargetMachine *TM)
+      : BaseT(TM->getDataLayout()) {}
+
+public:
+  // Provide value semantics. MSVC requires that we spell all of these out.
+  BasicTTIImplBase(const BasicTTIImplBase &Arg)
+      : BaseT(static_cast<const BaseT &>(Arg)) {}
+  BasicTTIImplBase(BasicTTIImplBase &&Arg)
+      : BaseT(std::move(static_cast<BaseT &>(Arg))) {}
+  BasicTTIImplBase &operator=(const BasicTTIImplBase &RHS) {
+    BaseT::operator=(static_cast<const BaseT &>(RHS));
+    return *this;
+  }
+  BasicTTIImplBase &operator=(BasicTTIImplBase &&RHS) {
+    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
+    return *this;
+  }
+
+  /// \name Scalar TTI Implementations
+  /// @{
+
+  bool hasBranchDivergence() { return false; }
+
+  bool isSourceOfDivergence(const Value *V) { return false; }
+
+  bool isLegalAddImmediate(int64_t imm) {
+    return getTLI()->isLegalAddImmediate(imm);
+  }
+
+  bool isLegalICmpImmediate(int64_t imm) {
+    return getTLI()->isLegalICmpImmediate(imm);
+  }
+
+  bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
+                             bool HasBaseReg, int64_t Scale) {
+    TargetLoweringBase::AddrMode AM;
+    AM.BaseGV = BaseGV;
+    AM.BaseOffs = BaseOffset;
+    AM.HasBaseReg = HasBaseReg;
+    AM.Scale = Scale;
+    return getTLI()->isLegalAddressingMode(AM, Ty);
+  }
+
+  int getScalingFactorCost(Type *Ty, GlobalValue *BaseGV, int64_t BaseOffset,
+                           bool HasBaseReg, int64_t Scale) {
+    TargetLoweringBase::AddrMode AM;
+    AM.BaseGV = BaseGV;
+    AM.BaseOffs = BaseOffset;
+    AM.HasBaseReg = HasBaseReg;
+    AM.Scale = Scale;
+    return getTLI()->getScalingFactorCost(AM, Ty);
+  }
+
+  bool isTruncateFree(Type *Ty1, Type *Ty2) {
+    return getTLI()->isTruncateFree(Ty1, Ty2);
+  }
+
+  bool isProfitableToHoist(Instruction *I) {
+    return getTLI()->isProfitableToHoist(I);
+  }
+
+  bool isTypeLegal(Type *Ty) {
+    EVT VT = getTLI()->getValueType(Ty);
+    return getTLI()->isTypeLegal(VT);
+  }
+
+  unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
+                            ArrayRef<const Value *> Arguments) {
+    return BaseT::getIntrinsicCost(IID, RetTy, Arguments);
+  }
+
+  unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy,
+                            ArrayRef<Type *> ParamTys) {
+    if (IID == Intrinsic::cttz) {
+      if (getTLI()->isCheapToSpeculateCttz())
+        return TargetTransformInfo::TCC_Basic;
+      return TargetTransformInfo::TCC_Expensive;
+    }
+
+    if (IID == Intrinsic::ctlz) {
+       if (getTLI()->isCheapToSpeculateCtlz())
+        return TargetTransformInfo::TCC_Basic;
+      return TargetTransformInfo::TCC_Expensive;
+    }
+
+    return BaseT::getIntrinsicCost(IID, RetTy, ParamTys);
+  }
+
+  unsigned getJumpBufAlignment() { return getTLI()->getJumpBufAlignment(); }
+
+  unsigned getJumpBufSize() { return getTLI()->getJumpBufSize(); }
+
+  bool shouldBuildLookupTables() {
+    const TargetLoweringBase *TLI = getTLI();
+    return TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+           TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
+  }
+
+  bool haveFastSqrt(Type *Ty) {
+    const TargetLoweringBase *TLI = getTLI();
+    EVT VT = TLI->getValueType(Ty);
+    return TLI->isTypeLegal(VT) &&
+           TLI->isOperationLegalOrCustom(ISD::FSQRT, VT);
+  }
+
+  unsigned getFPOpCost(Type *Ty) {
+    // By default, FP instructions are no more expensive since they are
+    // implemented in HW.  Target specific TTI can override this.
+    return TargetTransformInfo::TCC_Basic;
+  }
+
+  unsigned getOperationCost(unsigned Opcode, Type *Ty, Type *OpTy) {
+    const TargetLoweringBase *TLI = getTLI();
+    switch (Opcode) {
+    default: break;
+    case Instruction::Trunc: {
+      if (TLI->isTruncateFree(OpTy, Ty))
+        return TargetTransformInfo::TCC_Free;
+      return TargetTransformInfo::TCC_Basic;
+    }
+    case Instruction::ZExt: {
+      if (TLI->isZExtFree(OpTy, Ty))
+        return TargetTransformInfo::TCC_Free;
+      return TargetTransformInfo::TCC_Basic;
+    }
+    }
+
+    return BaseT::getOperationCost(Opcode, Ty, OpTy);
+  }
+
+  void getUnrollingPreferences(Loop *L, TTI::UnrollingPreferences &UP) {
+    // This unrolling functionality is target independent, but to provide some
+    // motivation for its intended use, for x86:
+
+    // According to the Intel 64 and IA-32 Architectures Optimization Reference
+    // Manual, Intel Core models and later have a loop stream detector (and
+    // associated uop queue) that can benefit from partial unrolling.
+    // The relevant requirements are:
+    //  - The loop must have no more than 4 (8 for Nehalem and later) branches
+    //    taken, and none of them may be calls.
+    //  - The loop can have no more than 18 (28 for Nehalem and later) uops.
+
+    // According to the Software Optimization Guide for AMD Family 15h
+    // Processors, models 30h-4fh (Steamroller and later) have a loop predictor
+    // and loop buffer which can benefit from partial unrolling.
+    // The relevant requirements are:
+    //  - The loop must have fewer than 16 branches
+    //  - The loop must have less than 40 uops in all executed loop branches
+
+    // The number of taken branches in a loop is hard to estimate here, and
+    // benchmarking has revealed that it is better not to be conservative when
+    // estimating the branch count. As a result, we'll ignore the branch limits
+    // until someone finds a case where it matters in practice.
+
+    unsigned MaxOps;
+    const TargetSubtargetInfo *ST = getST();
+    if (PartialUnrollingThreshold.getNumOccurrences() > 0)
+      MaxOps = PartialUnrollingThreshold;
+    else if (ST->getSchedModel().LoopMicroOpBufferSize > 0)
+      MaxOps = ST->getSchedModel().LoopMicroOpBufferSize;
+    else
+      return;
+
+    // Scan the loop: don't unroll loops with calls.
+    for (Loop::block_iterator I = L->block_begin(), E = L->block_end(); I != E;
+         ++I) {
+      BasicBlock *BB = *I;
+
+      for (BasicBlock::iterator J = BB->begin(), JE = BB->end(); J != JE; ++J)
+        if (isa<CallInst>(J) || isa<InvokeInst>(J)) {
+          ImmutableCallSite CS(J);
+          if (const Function *F = CS.getCalledFunction()) {
+            if (!static_cast<T *>(this)->isLoweredToCall(F))
+              continue;
+          }
+
+          return;
+        }
+    }
+
+    // Enable runtime and partial unrolling up to the specified size.
+    UP.Partial = UP.Runtime = true;
+    UP.PartialThreshold = UP.PartialOptSizeThreshold = MaxOps;
+  }
+
+  /// @}
+
+  /// \name Vector TTI Implementations
+  /// @{
+
+  unsigned getNumberOfRegisters(bool Vector) { return 1; }
+
+  unsigned getRegisterBitWidth(bool Vector) { return 32; }
+
+  unsigned getMaxInterleaveFactor(unsigned VF) { return 1; }
+
+  unsigned getArithmeticInstrCost(
+      unsigned Opcode, Type *Ty,
+      TTI::OperandValueKind Opd1Info = TTI::OK_AnyValue,
+      TTI::OperandValueKind Opd2Info = TTI::OK_AnyValue,
+      TTI::OperandValueProperties Opd1PropInfo = TTI::OP_None,
+      TTI::OperandValueProperties Opd2PropInfo = TTI::OP_None) {
+    // Check if any of the operands are vector operands.
+    const TargetLoweringBase *TLI = getTLI();
+    int ISD = TLI->InstructionOpcodeToISD(Opcode);
+    assert(ISD && "Invalid opcode");
+
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(Ty);
+
+    bool IsFloat = Ty->getScalarType()->isFloatingPointTy();
+    // Assume that floating point arithmetic operations cost twice as much as
+    // integer operations.
+    unsigned OpCost = (IsFloat ? 2 : 1);
+
+    if (TLI->isOperationLegalOrPromote(ISD, LT.second)) {
+      // The operation is legal. Assume it costs 1.
+      // If the type is split to multiple registers, assume that there is some
+      // overhead to this.
+      // TODO: Once we have extract/insert subvector cost we need to use them.
+      if (LT.first > 1)
+        return LT.first * 2 * OpCost;
+      return LT.first * 1 * OpCost;
+    }
+
+    if (!TLI->isOperationExpand(ISD, LT.second)) {
+      // If the operation is custom lowered then assume
+      // thare the code is twice as expensive.
+      return LT.first * 2 * OpCost;
+    }
+
+    // Else, assume that we need to scalarize this op.
+    if (Ty->isVectorTy()) {
+      unsigned Num = Ty->getVectorNumElements();
+      unsigned Cost = static_cast<T *>(this)
+                          ->getArithmeticInstrCost(Opcode, Ty->getScalarType());
+      // return the cost of multiple scalar invocation plus the cost of
+      // inserting
+      // and extracting the values.
+      return getScalarizationOverhead(Ty, true, true) + Num * Cost;
+    }
+
+    // We don't know anything about this scalar instruction.
+    return OpCost;
+  }
+
+  unsigned getShuffleCost(TTI::ShuffleKind Kind, Type *Tp, int Index,
+                          Type *SubTp) {
+    if (Kind == TTI::SK_Alternate) {
+      return getAltShuffleOverhead(Tp);
+    }
+    return 1;
+  }
+
+  unsigned getCastInstrCost(unsigned Opcode, Type *Dst, Type *Src) {
+    const TargetLoweringBase *TLI = getTLI();
+    int ISD = TLI->InstructionOpcodeToISD(Opcode);
+    assert(ISD && "Invalid opcode");
+
+    std::pair<unsigned, MVT> SrcLT = TLI->getTypeLegalizationCost(Src);
+    std::pair<unsigned, MVT> DstLT = TLI->getTypeLegalizationCost(Dst);
+
+    // Check for NOOP conversions.
+    if (SrcLT.first == DstLT.first &&
+        SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) {
+
+      // Bitcast between types that are legalized to the same type are free.
+      if (Opcode == Instruction::BitCast || Opcode == Instruction::Trunc)
+        return 0;
+    }
+
+    if (Opcode == Instruction::Trunc &&
+        TLI->isTruncateFree(SrcLT.second, DstLT.second))
+      return 0;
+
+    if (Opcode == Instruction::ZExt &&
+        TLI->isZExtFree(SrcLT.second, DstLT.second))
+      return 0;
+
+    // If the cast is marked as legal (or promote) then assume low cost.
+    if (SrcLT.first == DstLT.first &&
+        TLI->isOperationLegalOrPromote(ISD, DstLT.second))
+      return 1;
+
+    // Handle scalar conversions.
+    if (!Src->isVectorTy() && !Dst->isVectorTy()) {
+
+      // Scalar bitcasts are usually free.
+      if (Opcode == Instruction::BitCast)
+        return 0;
+
+      // Just check the op cost. If the operation is legal then assume it costs
+      // 1.
+      if (!TLI->isOperationExpand(ISD, DstLT.second))
+        return 1;
+
+      // Assume that illegal scalar instruction are expensive.
+      return 4;
+    }
+
+    // Check vector-to-vector casts.
+    if (Dst->isVectorTy() && Src->isVectorTy()) {
+
+      // If the cast is between same-sized registers, then the check is simple.
+      if (SrcLT.first == DstLT.first &&
+          SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) {
+
+        // Assume that Zext is done using AND.
+        if (Opcode == Instruction::ZExt)
+          return 1;
+
+        // Assume that sext is done using SHL and SRA.
+        if (Opcode == Instruction::SExt)
+          return 2;
+
+        // Just check the op cost. If the operation is legal then assume it
+        // costs
+        // 1 and multiply by the type-legalization overhead.
+        if (!TLI->isOperationExpand(ISD, DstLT.second))
+          return SrcLT.first * 1;
+      }
+
+      // If we are converting vectors and the operation is illegal, or
+      // if the vectors are legalized to different types, estimate the
+      // scalarization costs.
+      unsigned Num = Dst->getVectorNumElements();
+      unsigned Cost = static_cast<T *>(this)->getCastInstrCost(
+          Opcode, Dst->getScalarType(), Src->getScalarType());
+
+      // Return the cost of multiple scalar invocation plus the cost of
+      // inserting and extracting the values.
+      return getScalarizationOverhead(Dst, true, true) + Num * Cost;
+    }
+
+    // We already handled vector-to-vector and scalar-to-scalar conversions.
+    // This
+    // is where we handle bitcast between vectors and scalars. We need to assume
+    //  that the conversion is scalarized in one way or another.
+    if (Opcode == Instruction::BitCast)
+      // Illegal bitcasts are done by storing and loading from a stack slot.
+      return (Src->isVectorTy() ? getScalarizationOverhead(Src, false, true)
+                                : 0) +
+             (Dst->isVectorTy() ? getScalarizationOverhead(Dst, true, false)
+                                : 0);
+
+    llvm_unreachable("Unhandled cast");
+  }
+
+  unsigned getCFInstrCost(unsigned Opcode) {
+    // Branches are assumed to be predicted.
+    return 0;
+  }
+
+  unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy, Type *CondTy) {
+    const TargetLoweringBase *TLI = getTLI();
+    int ISD = TLI->InstructionOpcodeToISD(Opcode);
+    assert(ISD && "Invalid opcode");
+
+    // Selects on vectors are actually vector selects.
+    if (ISD == ISD::SELECT) {
+      assert(CondTy && "CondTy must exist");
+      if (CondTy->isVectorTy())
+        ISD = ISD::VSELECT;
+    }
+
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(ValTy);
+
+    if (!(ValTy->isVectorTy() && !LT.second.isVector()) &&
+        !TLI->isOperationExpand(ISD, LT.second)) {
+      // The operation is legal. Assume it costs 1. Multiply
+      // by the type-legalization overhead.
+      return LT.first * 1;
+    }
+
+    // Otherwise, assume that the cast is scalarized.
+    if (ValTy->isVectorTy()) {
+      unsigned Num = ValTy->getVectorNumElements();
+      if (CondTy)
+        CondTy = CondTy->getScalarType();
+      unsigned Cost = static_cast<T *>(this)->getCmpSelInstrCost(
+          Opcode, ValTy->getScalarType(), CondTy);
+
+      // Return the cost of multiple scalar invocation plus the cost of
+      // inserting
+      // and extracting the values.
+      return getScalarizationOverhead(ValTy, true, false) + Num * Cost;
+    }
+
+    // Unknown scalar opcode.
+    return 1;
+  }
+
+  unsigned getVectorInstrCost(unsigned Opcode, Type *Val, unsigned Index) {
+    std::pair<unsigned, MVT> LT =
+        getTLI()->getTypeLegalizationCost(Val->getScalarType());
+
+    return LT.first;
+  }
+
+  unsigned getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment,
+                           unsigned AddressSpace) {
+    assert(!Src->isVoidTy() && "Invalid type");
+    std::pair<unsigned, MVT> LT = getTLI()->getTypeLegalizationCost(Src);
+
+    // Assuming that all loads of legal types cost 1.
+    unsigned Cost = LT.first;
+
+    if (Src->isVectorTy() &&
+        Src->getPrimitiveSizeInBits() < LT.second.getSizeInBits()) {
+      // This is a vector load that legalizes to a larger type than the vector
+      // itself. Unless the corresponding extending load or truncating store is
+      // legal, then this will scalarize.
+      TargetLowering::LegalizeAction LA = TargetLowering::Expand;
+      EVT MemVT = getTLI()->getValueType(Src, true);
+      if (MemVT.isSimple() && MemVT != MVT::Other) {
+        if (Opcode == Instruction::Store)
+          LA = getTLI()->getTruncStoreAction(LT.second, MemVT.getSimpleVT());
+        else
+          LA = getTLI()->getLoadExtAction(ISD::EXTLOAD, LT.second, MemVT);
+      }
+
+      if (LA != TargetLowering::Legal && LA != TargetLowering::Custom) {
+        // This is a vector load/store for some illegal type that is scalarized.
+        // We must account for the cost of building or decomposing the vector.
+        Cost += getScalarizationOverhead(Src, Opcode != Instruction::Store,
+                                         Opcode == Instruction::Store);
+      }
+    }
+
+    return Cost;
+  }
+
+  unsigned getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
+                                 ArrayRef<Type *> Tys) {
+    unsigned ISD = 0;
+    switch (IID) {
+    default: {
+      // Assume that we need to scalarize this intrinsic.
+      unsigned ScalarizationCost = 0;
+      unsigned ScalarCalls = 1;
+      Type *ScalarRetTy = RetTy;
+      if (RetTy->isVectorTy()) {
+        ScalarizationCost = getScalarizationOverhead(RetTy, true, false);
+        ScalarCalls = std::max(ScalarCalls, RetTy->getVectorNumElements());
+        ScalarRetTy = RetTy->getScalarType();
+      }
+      SmallVector<Type *, 4> ScalarTys;
+      for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
+        Type *Ty = Tys[i];
+        if (Ty->isVectorTy()) {
+          ScalarizationCost += getScalarizationOverhead(Ty, false, true);
+          ScalarCalls = std::max(ScalarCalls, Ty->getVectorNumElements());
+          Ty = Ty->getScalarType();
+        }
+        ScalarTys.push_back(Ty);
+      }
+      if (ScalarCalls == 1)
+        return 1; // Return cost of a scalar intrinsic. Assume it to be cheap.
+
+      unsigned ScalarCost = static_cast<T *>(this)->getIntrinsicInstrCost(
+          IID, ScalarRetTy, ScalarTys);
+
+      return ScalarCalls * ScalarCost + ScalarizationCost;
+    }
+    // Look for intrinsics that can be lowered directly or turned into a scalar
+    // intrinsic call.
+    case Intrinsic::sqrt:
+      ISD = ISD::FSQRT;
+      break;
+    case Intrinsic::sin:
+      ISD = ISD::FSIN;
+      break;
+    case Intrinsic::cos:
+      ISD = ISD::FCOS;
+      break;
+    case Intrinsic::exp:
+      ISD = ISD::FEXP;
+      break;
+    case Intrinsic::exp2:
+      ISD = ISD::FEXP2;
+      break;
+    case Intrinsic::log:
+      ISD = ISD::FLOG;
+      break;
+    case Intrinsic::log10:
+      ISD = ISD::FLOG10;
+      break;
+    case Intrinsic::log2:
+      ISD = ISD::FLOG2;
+      break;
+    case Intrinsic::fabs:
+      ISD = ISD::FABS;
+      break;
+    case Intrinsic::minnum:
+      ISD = ISD::FMINNUM;
+      break;
+    case Intrinsic::maxnum:
+      ISD = ISD::FMAXNUM;
+      break;
+    case Intrinsic::copysign:
+      ISD = ISD::FCOPYSIGN;
+      break;
+    case Intrinsic::floor:
+      ISD = ISD::FFLOOR;
+      break;
+    case Intrinsic::ceil:
+      ISD = ISD::FCEIL;
+      break;
+    case Intrinsic::trunc:
+      ISD = ISD::FTRUNC;
+      break;
+    case Intrinsic::nearbyint:
+      ISD = ISD::FNEARBYINT;
+      break;
+    case Intrinsic::rint:
+      ISD = ISD::FRINT;
+      break;
+    case Intrinsic::round:
+      ISD = ISD::FROUND;
+      break;
+    case Intrinsic::pow:
+      ISD = ISD::FPOW;
+      break;
+    case Intrinsic::fma:
+      ISD = ISD::FMA;
+      break;
+    case Intrinsic::fmuladd:
+      ISD = ISD::FMA;
+      break;
+    // FIXME: We should return 0 whenever getIntrinsicCost == TCC_Free.
+    case Intrinsic::lifetime_start:
+    case Intrinsic::lifetime_end:
+      return 0;
+    case Intrinsic::masked_store:
+      return static_cast<T *>(this)
+          ->getMaskedMemoryOpCost(Instruction::Store, Tys[0], 0, 0);
+    case Intrinsic::masked_load:
+      return static_cast<T *>(this)
+          ->getMaskedMemoryOpCost(Instruction::Load, RetTy, 0, 0);
+    }
+
+    const TargetLoweringBase *TLI = getTLI();
+    std::pair<unsigned, MVT> LT = TLI->getTypeLegalizationCost(RetTy);
+
+    if (TLI->isOperationLegalOrPromote(ISD, LT.second)) {
+      // The operation is legal. Assume it costs 1.
+      // If the type is split to multiple registers, assume that there is some
+      // overhead to this.
+      // TODO: Once we have extract/insert subvector cost we need to use them.
+      if (LT.first > 1)
+        return LT.first * 2;
+      return LT.first * 1;
+    }
+
+    if (!TLI->isOperationExpand(ISD, LT.second)) {
+      // If the operation is custom lowered then assume
+      // thare the code is twice as expensive.
+      return LT.first * 2;
+    }
+
+    // If we can't lower fmuladd into an FMA estimate the cost as a floating
+    // point mul followed by an add.
+    if (IID == Intrinsic::fmuladd)
+      return static_cast<T *>(this)
+                 ->getArithmeticInstrCost(BinaryOperator::FMul, RetTy) +
+             static_cast<T *>(this)
+                 ->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy);
+
+    // Else, assume that we need to scalarize this intrinsic. For math builtins
+    // this will emit a costly libcall, adding call overhead and spills. Make it
+    // very expensive.
+    if (RetTy->isVectorTy()) {
+      unsigned ScalarizationCost = getScalarizationOverhead(RetTy, true, false);
+      unsigned ScalarCalls = RetTy->getVectorNumElements();
+      SmallVector<Type *, 4> ScalarTys;
+      for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
+        Type *Ty = Tys[i];
+        if (Ty->isVectorTy())
+          Ty = Ty->getScalarType();
+        ScalarTys.push_back(Ty);
+      }
+      unsigned ScalarCost = static_cast<T *>(this)->getIntrinsicInstrCost(
+          IID, RetTy->getScalarType(), ScalarTys);
+      for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) {
+        if (Tys[i]->isVectorTy()) {
+          ScalarizationCost += getScalarizationOverhead(Tys[i], false, true);
+          ScalarCalls = std::max(ScalarCalls, Tys[i]->getVectorNumElements());
+        }
+      }
+
+      return ScalarCalls * ScalarCost + ScalarizationCost;
+    }
+
+    // This is going to be turned into a library call, make it expensive.
+    return 10;
+  }
+
+  /// \brief Compute a cost of the given call instruction.
+  ///
+  /// Compute the cost of calling function F with return type RetTy and
+  /// argument types Tys. F might be nullptr, in this case the cost of an
+  /// arbitrary call with the specified signature will be returned.
+  /// This is used, for instance,  when we estimate call of a vector
+  /// counterpart of the given function.
+  /// \param F Called function, might be nullptr.
+  /// \param RetTy Return value types.
+  /// \param Tys Argument types.
+  /// \returns The cost of Call instruction.
+  unsigned getCallInstrCost(Function *F, Type *RetTy, ArrayRef<Type *> Tys) {
+    return 10;
+  }
+
+  unsigned getNumberOfParts(Type *Tp) {
+    std::pair<unsigned, MVT> LT = getTLI()->getTypeLegalizationCost(Tp);
+    return LT.first;
+  }
+
+  unsigned getAddressComputationCost(Type *Ty, bool IsComplex) { return 0; }
+
+  unsigned getReductionCost(unsigned Opcode, Type *Ty, bool IsPairwise) {
+    assert(Ty->isVectorTy() && "Expect a vector type");
+    unsigned NumVecElts = Ty->getVectorNumElements();
+    unsigned NumReduxLevels = Log2_32(NumVecElts);
+    unsigned ArithCost =
+        NumReduxLevels *
+        static_cast<T *>(this)->getArithmeticInstrCost(Opcode, Ty);
+    // Assume the pairwise shuffles add a cost.
+    unsigned ShuffleCost =
+        NumReduxLevels * (IsPairwise + 1) *
+        static_cast<T *>(this)
+            ->getShuffleCost(TTI::SK_ExtractSubvector, Ty, NumVecElts / 2, Ty);
+    return ShuffleCost + ArithCost + getScalarizationOverhead(Ty, false, true);
+  }
+
+  /// @}
+};
+
+/// \brief Concrete BasicTTIImpl that can be used if no further customization
+/// is needed.
+class BasicTTIImpl : public BasicTTIImplBase<BasicTTIImpl> {
+  typedef BasicTTIImplBase<BasicTTIImpl> BaseT;
+  friend class BasicTTIImplBase<BasicTTIImpl>;
+
+  const TargetSubtargetInfo *ST;
+  const TargetLoweringBase *TLI;
+
+  const TargetSubtargetInfo *getST() const { return ST; }
+  const TargetLoweringBase *getTLI() const { return TLI; }
+
+public:
+  explicit BasicTTIImpl(const TargetMachine *ST, Function &F);
+
+  // Provide value semantics. MSVC requires that we spell all of these out.
+  BasicTTIImpl(const BasicTTIImpl &Arg)
+      : BaseT(static_cast<const BaseT &>(Arg)), ST(Arg.ST), TLI(Arg.TLI) {}
+  BasicTTIImpl(BasicTTIImpl &&Arg)
+      : BaseT(std::move(static_cast<BaseT &>(Arg))), ST(std::move(Arg.ST)),
+        TLI(std::move(Arg.TLI)) {}
+  BasicTTIImpl &operator=(const BasicTTIImpl &RHS) {
+    BaseT::operator=(static_cast<const BaseT &>(RHS));
+    ST = RHS.ST;
+    TLI = RHS.TLI;
+    return *this;
+  }
+  BasicTTIImpl &operator=(BasicTTIImpl &&RHS) {
+    BaseT::operator=(std::move(static_cast<BaseT &>(RHS)));
+    ST = std::move(RHS.ST);
+    TLI = std::move(RHS.TLI);
+    return *this;
+  }
+};
+
+}
+
+#endif
diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h
index dd7703b1dbf75..1fd4eeb46b38a 100644
--- a/include/llvm/CodeGen/CallingConvLower.h
+++ b/include/llvm/CodeGen/CallingConvLower.h
@@ -19,6 +19,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/IR/CallingConv.h"
+#include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/Target/TargetCallingConv.h"
 
 namespace llvm {
@@ -122,8 +123,8 @@ public:
   // There is no need to differentiate between a pending CCValAssign and other
   // kinds, as they are stored in a different list.
   static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT,
-                                LocInfo HTP) {
-    return getReg(ValNo, ValVT, 0, LocVT, HTP);
+                                LocInfo HTP, unsigned ExtraInfo = 0) {
+    return getReg(ValNo, ValVT, ExtraInfo, LocVT, HTP);
   }
 
   void convertToReg(unsigned RegNo) {
@@ -146,6 +147,7 @@ public:
 
   unsigned getLocReg() const { assert(isRegLoc()); return Loc; }
   unsigned getLocMemOffset() const { assert(isMemLoc()); return Loc; }
+  unsigned getExtraInfo() const { return Loc; }
   MVT getLocVT() const { return LocVT; }
 
   LocInfo getLocInfo() const { return HTP; }
@@ -312,13 +314,13 @@ public:
   /// produce a single value.
   void AnalyzeCallResult(MVT VT, CCAssignFn Fn);
 
-  /// getFirstUnallocated - Return the first unallocated register in the set, or
-  /// NumRegs if they are all allocated.
-  unsigned getFirstUnallocated(const MCPhysReg *Regs, unsigned NumRegs) const {
-    for (unsigned i = 0; i != NumRegs; ++i)
+  /// getFirstUnallocated - Return the index of the first unallocated register
+  /// in the set, or Regs.size() if they are all allocated.
+  unsigned getFirstUnallocated(ArrayRef<MCPhysReg> Regs) const {
+    for (unsigned i = 0; i < Regs.size(); ++i)
       if (!isAllocated(Regs[i]))
         return i;
-    return NumRegs;
+    return Regs.size();
   }
 
   /// AllocateReg - Attempt to allocate one register.  If it is not available,
@@ -341,9 +343,9 @@ public:
   /// AllocateReg - Attempt to allocate one of the specified registers.  If none
   /// are available, return zero.  Otherwise, return the first one available,
   /// marking it and any aliases as allocated.
-  unsigned AllocateReg(const MCPhysReg *Regs, unsigned NumRegs) {
-    unsigned FirstUnalloc = getFirstUnallocated(Regs, NumRegs);
-    if (FirstUnalloc == NumRegs)
+  unsigned AllocateReg(ArrayRef<MCPhysReg> Regs) {
+    unsigned FirstUnalloc = getFirstUnallocated(Regs);
+    if (FirstUnalloc == Regs.size())
       return 0;    // Didn't find the reg.
 
     // Mark the register and any aliases as allocated.
@@ -382,10 +384,9 @@ public:
   }
 
   /// Version of AllocateReg with list of registers to be shadowed.
-  unsigned AllocateReg(const MCPhysReg *Regs, const MCPhysReg *ShadowRegs,
-                       unsigned NumRegs) {
-    unsigned FirstUnalloc = getFirstUnallocated(Regs, NumRegs);
-    if (FirstUnalloc == NumRegs)
+  unsigned AllocateReg(ArrayRef<MCPhysReg> Regs, const MCPhysReg *ShadowRegs) {
+    unsigned FirstUnalloc = getFirstUnallocated(Regs);
+    if (FirstUnalloc == Regs.size())
       return 0;    // Didn't find the reg.
 
     // Mark the register and any aliases as allocated.
@@ -415,8 +416,8 @@ public:
   /// Version of AllocateStack with list of extra registers to be shadowed.
   /// Note that, unlike AllocateReg, this shadows ALL of the shadow registers.
   unsigned AllocateStack(unsigned Size, unsigned Align,
-                         const MCPhysReg *ShadowRegs, unsigned NumShadowRegs) {
-    for (unsigned i = 0; i < NumShadowRegs; ++i)
+                         ArrayRef<MCPhysReg> ShadowRegs) {
+    for (unsigned i = 0; i < ShadowRegs.size(); ++i)
       MarkAllocated(ShadowRegs[i]);
     return AllocateStack(Size, Align);
   }
diff --git a/include/llvm/CodeGen/CommandFlags.h b/include/llvm/CodeGen/CommandFlags.h
index 973c5954f9ad3..a1b9b4e566a26 100644
--- a/include/llvm/CodeGen/CommandFlags.h
+++ b/include/llvm/CodeGen/CommandFlags.h
@@ -16,9 +16,12 @@
 #ifndef LLVM_CODEGEN_COMMANDFLAGS_H
 #define LLVM_CODEGEN_COMMANDFLAGS_H
 
+#include "llvm/IR/Module.h"
 #include "llvm/MC/MCTargetOptionsCommandFlags.h"
+#include "llvm//MC/SubtargetFeature.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Host.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include <string>
@@ -123,11 +126,6 @@ EnableHonorSignDependentRoundingFPMath("enable-sign-dependent-rounding-fp-math",
       cl::desc("Force codegen to assume rounding mode can change dynamically"),
       cl::init(false));
 
-cl::opt<bool>
-GenerateSoftFloatCalls("soft-float",
-                    cl::desc("Generate software floating point library calls"),
-                    cl::init(false));
-
 cl::opt<llvm::FloatABI::ABIType>
 FloatABIForCalls("float-abi",
                  cl::desc("Choose float ABI type"),
@@ -207,6 +205,10 @@ FunctionSections("function-sections",
                  cl::desc("Emit functions into separate sections"),
                  cl::init(false));
 
+cl::opt<bool> UniqueSectionNames("unique-section-names",
+                                 cl::desc("Give unique names to every section"),
+                                 cl::init(true));
+
 cl::opt<llvm::JumpTable::JumpTableType>
 JTableType("jump-table-type",
           cl::desc("Choose the type of Jump-Instruction Table for jumptable."),
@@ -222,57 +224,17 @@ JTableType("jump-table-type",
                          "Create one table per unique function type."),
               clEnumValEnd));
 
-cl::opt<bool>
-FCFI("fcfi",
-     cl::desc("Apply forward-edge control-flow integrity"),
-     cl::init(false));
-
-cl::opt<llvm::CFIntegrity>
-CFIType("cfi-type",
-        cl::desc("Choose the type of Control-Flow Integrity check to add"),
-        cl::init(CFIntegrity::Sub),
-        cl::values(
-            clEnumValN(CFIntegrity::Sub, "sub",
-                       "Subtract the pointer from the table base, then mask."),
-            clEnumValN(CFIntegrity::Ror, "ror",
-                       "Use rotate to check the offset from a table base."),
-            clEnumValN(CFIntegrity::Add, "add",
-                       "Mask out the high bits and add to an aligned base."),
-            clEnumValEnd));
-
-cl::opt<bool>
-CFIEnforcing("cfi-enforcing",
-             cl::desc("Enforce CFI or pass the violation to a function."),
-             cl::init(false));
-
-// Note that this option is linked to the cfi-enforcing option above: if
-// cfi-enforcing is set, then the cfi-func-name option is entirely ignored. If
-// cfi-enforcing is false and no cfi-func-name is set, then a default function
-// will be generated that ignores all CFI violations. The expected signature for
-// functions called with CFI violations is
-//
-// void (i8*, i8*)
-//
-// The first pointer is a C string containing the name of the function in which
-// the violation occurs, and the second pointer is the pointer that violated
-// CFI.
-cl::opt<std::string>
-CFIFuncName("cfi-func-name", cl::desc("The name of the CFI function to call"),
-            cl::init(""));
-
 // Common utility function tightly tied to the options listed here. Initializes
 // a TargetOptions object with CodeGen flags and returns it.
 static inline TargetOptions InitTargetOptionsFromCodeGenFlags() {
   TargetOptions Options;
   Options.LessPreciseFPMADOption = EnableFPMAD;
-  Options.NoFramePointerElim = DisableFPElim;
   Options.AllowFPOpFusion = FuseFPOps;
   Options.UnsafeFPMath = EnableUnsafeFPMath;
   Options.NoInfsFPMath = EnableNoInfsFPMath;
   Options.NoNaNsFPMath = EnableNoNaNsFPMath;
   Options.HonorSignDependentRoundingFPMathOption =
       EnableHonorSignDependentRoundingFPMath;
-  Options.UseSoftFloat = GenerateSoftFloatCalls;
   if (FloatABIForCalls != FloatABI::Default)
     Options.FloatABIType = FloatABIForCalls;
   Options.NoZerosInBSS = DontPlaceZerosInBSS;
@@ -284,17 +246,71 @@ static inline TargetOptions InitTargetOptionsFromCodeGenFlags() {
   Options.UseInitArray = !UseCtors;
   Options.DataSections = DataSections;
   Options.FunctionSections = FunctionSections;
+  Options.UniqueSectionNames = UniqueSectionNames;
 
   Options.MCOptions = InitMCTargetOptionsFromFlags();
   Options.JTType = JTableType;
-  Options.FCFI = FCFI;
-  Options.CFIType = CFIType;
-  Options.CFIEnforcing = CFIEnforcing;
-  Options.CFIFuncName = CFIFuncName;
 
   Options.ThreadModel = TMModel;
 
   return Options;
 }
 
+static inline std::string getCPUStr() {
+  // If user asked for the 'native' CPU, autodetect here. If autodection fails,
+  // this will set the CPU to an empty string which tells the target to
+  // pick a basic default.
+  if (MCPU == "native")
+    return sys::getHostCPUName();
+
+  return MCPU;
+}
+
+static inline std::string getFeaturesStr() {
+  SubtargetFeatures Features;
+
+  // If user asked for the 'native' CPU, we need to autodetect features.
+  // This is necessary for x86 where the CPU might not support all the
+  // features the autodetected CPU name lists in the target. For example,
+  // not all Sandybridge processors support AVX.
+  if (MCPU == "native") {
+    StringMap<bool> HostFeatures;
+    if (sys::getHostCPUFeatures(HostFeatures))
+      for (auto &F : HostFeatures)
+        Features.AddFeature(F.first(), F.second);
+  }
+
+  for (unsigned i = 0; i != MAttrs.size(); ++i)
+    Features.AddFeature(MAttrs[i]);
+
+  return Features.getString();
+}
+
+/// \brief Set function attributes of functions in Module M based on CPU,
+/// Features, and command line flags.
+static inline void setFunctionAttributes(StringRef CPU, StringRef Features,
+                                         Module &M) {
+  for (auto &F : M) {
+    auto &Ctx = F.getContext();
+    AttributeSet Attrs = F.getAttributes(), NewAttrs;
+
+    if (!CPU.empty())
+      NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex,
+                                       "target-cpu", CPU);
+
+    if (!Features.empty())
+      NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex,
+                                       "target-features", Features);
+
+    if (DisableFPElim.getNumOccurrences() > 0)
+      NewAttrs = NewAttrs.addAttribute(Ctx, AttributeSet::FunctionIndex,
+                                       "no-frame-pointer-elim",
+                                       DisableFPElim ? "true" : "false");
+
+    // Let NewAttrs override Attrs.
+    NewAttrs = Attrs.addAttributes(Ctx, AttributeSet::FunctionIndex, NewAttrs);
+    F.setAttributes(NewAttrs);
+  }
+}
+
 #endif
diff --git a/include/llvm/CodeGen/DIE.h b/include/llvm/CodeGen/DIE.h
index e310aef3dcbbb..8e40ef77d374f 100644
--- a/include/llvm/CodeGen/DIE.h
+++ b/include/llvm/CodeGen/DIE.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/CodeGen/DwarfStringPoolEntry.h"
 #include "llvm/Support/Dwarf.h"
 #include <vector>
 
@@ -95,7 +96,7 @@ public:
 
   /// Emit - Print the abbreviation using the specified asm printer.
   ///
-  void Emit(AsmPrinter *AP) const;
+  void Emit(const AsmPrinter *AP) const;
 
 #ifndef NDEBUG
   void print(raw_ostream &O);
@@ -200,8 +201,6 @@ public:
 /// to DWARF attribute classes.
 ///
 class DIEValue {
-  virtual void anchor();
-
 public:
   enum Type {
     isInteger,
@@ -216,13 +215,14 @@ public:
     isLocList,
   };
 
-protected:
+private:
   /// Ty - Type of data stored in the value.
   ///
   Type Ty;
 
+protected:
   explicit DIEValue(Type T) : Ty(T) {}
-  virtual ~DIEValue() {}
+  ~DIEValue() {}
 
 public:
   // Accessors
@@ -230,14 +230,14 @@ public:
 
   /// EmitValue - Emit value via the Dwarf writer.
   ///
-  virtual void EmitValue(AsmPrinter *AP, dwarf::Form Form) const = 0;
+  void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const;
 
   /// SizeOf - Return the size of a value in bytes.
   ///
-  virtual unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const = 0;
+  unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const;
 
 #ifndef NDEBUG
-  virtual void print(raw_ostream &O) const = 0;
+  void print(raw_ostream &O) const;
   void dump() const;
 #endif
 };
@@ -246,6 +246,8 @@ public:
 /// DIEInteger - An integer value DIE.
 ///
 class DIEInteger : public DIEValue {
+  friend DIEValue;
+
   uint64_t Integer;
 
 public:
@@ -273,21 +275,18 @@ public:
     return dwarf::DW_FORM_data8;
   }
 
-  /// EmitValue - Emit integer of appropriate size.
-  ///
-  void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override;
-
   uint64_t getValue() const { return Integer; }
-
-  /// SizeOf - Determine size of integer value in bytes.
-  ///
-  unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override;
+  void setValue(uint64_t Val) { Integer = Val; }
 
   // Implement isa/cast/dyncast.
   static bool classof(const DIEValue *I) { return I->getType() == isInteger; }
 
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
@@ -295,28 +294,26 @@ public:
 /// DIEExpr - An expression DIE.
 //
 class DIEExpr : public DIEValue {
+  friend class DIEValue;
+
   const MCExpr *Expr;
 
 public:
   explicit DIEExpr(const MCExpr *E) : DIEValue(isExpr), Expr(E) {}
 
-  /// EmitValue - Emit expression value.
-  ///
-  void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override;
-
   /// getValue - Get MCExpr.
   ///
   const MCExpr *getValue() const { return Expr; }
 
-  /// SizeOf - Determine size of expression value in bytes.
-  ///
-  unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override;
-
   // Implement isa/cast/dyncast.
   static bool classof(const DIEValue *E) { return E->getType() == isExpr; }
 
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
@@ -324,28 +321,26 @@ public:
 /// DIELabel - A label DIE.
 //
 class DIELabel : public DIEValue {
+  friend class DIEValue;
+
   const MCSymbol *Label;
 
 public:
   explicit DIELabel(const MCSymbol *L) : DIEValue(isLabel), Label(L) {}
 
-  /// EmitValue - Emit label value.
-  ///
-  void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override;
-
   /// getValue - Get MCSymbol.
   ///
   const MCSymbol *getValue() const { return Label; }
 
-  /// SizeOf - Determine size of label value in bytes.
-  ///
-  unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override;
-
   // Implement isa/cast/dyncast.
   static bool classof(const DIEValue *L) { return L->getType() == isLabel; }
 
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
@@ -353,6 +348,8 @@ public:
 /// DIEDelta - A simple label difference DIE.
 ///
 class DIEDelta : public DIEValue {
+  friend class DIEValue;
+
   const MCSymbol *LabelHi;
   const MCSymbol *LabelLo;
 
@@ -360,19 +357,15 @@ public:
   DIEDelta(const MCSymbol *Hi, const MCSymbol *Lo)
       : DIEValue(isDelta), LabelHi(Hi), LabelLo(Lo) {}
 
-  /// EmitValue - Emit delta value.
-  ///
-  void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of delta value in bytes.
-  ///
-  unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override;
-
   // Implement isa/cast/dyncast.
   static bool classof(const DIEValue *D) { return D->getType() == isDelta; }
 
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
@@ -380,29 +373,25 @@ public:
 /// DIEString - A container for string values.
 ///
 class DIEString : public DIEValue {
-  const DIEValue *Access;
-  StringRef Str;
+  friend class DIEValue;
+
+  DwarfStringPoolEntryRef S;
 
 public:
-  DIEString(const DIEValue *Acc, StringRef S)
-      : DIEValue(isString), Access(Acc), Str(S) {}
+  DIEString(DwarfStringPoolEntryRef S) : DIEValue(isString), S(S) {}
 
   /// getString - Grab the string out of the object.
-  StringRef getString() const { return Str; }
-
-  /// EmitValue - Emit delta value.
-  ///
-  void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of delta value in bytes.
-  ///
-  unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override;
+  StringRef getString() const { return S.getString(); }
 
   // Implement isa/cast/dyncast.
   static bool classof(const DIEValue *D) { return D->getType() == isString; }
 
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
@@ -411,6 +400,8 @@ public:
 /// this class can also be used as a proxy for a debug information entry not
 /// yet defined (ie. types.)
 class DIEEntry : public DIEValue {
+  friend class DIEValue;
+
   DIE &Entry;
 
 public:
@@ -419,53 +410,49 @@ public:
 
   DIE &getEntry() const { return Entry; }
 
-  /// EmitValue - Emit debug information entry offset.
-  ///
-  void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of debug information entry in bytes.
-  ///
-   unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override {
-    return Form == dwarf::DW_FORM_ref_addr ? getRefAddrSize(AP)
-                                           : sizeof(int32_t);
-  }
-
   /// Returns size of a ref_addr entry.
-  static unsigned getRefAddrSize(AsmPrinter *AP);
+  static unsigned getRefAddrSize(const AsmPrinter *AP);
 
   // Implement isa/cast/dyncast.
   static bool classof(const DIEValue *E) { return E->getType() == isEntry; }
 
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const {
+    return Form == dwarf::DW_FORM_ref_addr ? getRefAddrSize(AP)
+                                           : sizeof(int32_t);
+  }
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
 //===--------------------------------------------------------------------===//
 /// \brief A signature reference to a type unit.
 class DIETypeSignature : public DIEValue {
+  friend class DIEValue;
+
   const DwarfTypeUnit &Unit;
 
 public:
   explicit DIETypeSignature(const DwarfTypeUnit &Unit)
       : DIEValue(isTypeSignature), Unit(Unit) {}
 
-  /// \brief Emit type unit signature.
-  void EmitValue(AsmPrinter *Asm, dwarf::Form Form) const override;
+  // \brief Implement isa/cast/dyncast.
+  static bool classof(const DIEValue *E) {
+    return E->getType() == isTypeSignature;
+  }
 
-  /// Returns size of a ref_sig8 entry.
-  unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override {
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const {
     assert(Form == dwarf::DW_FORM_ref_sig8);
     return 8;
   }
 
-  // \brief Implement isa/cast/dyncast.
-  static bool classof(const DIEValue *E) {
-    return E->getType() == isTypeSignature;
-  }
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
-  void dump() const;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
@@ -473,13 +460,15 @@ public:
 /// DIELoc - Represents an expression location.
 //
 class DIELoc : public DIEValue, public DIE {
+  friend class DIEValue;
+
   mutable unsigned Size; // Size in bytes excluding size header.
 public:
   DIELoc() : DIEValue(isLoc), Size(0) {}
 
   /// ComputeSize - Calculate the size of the location expression.
   ///
-  unsigned ComputeSize(AsmPrinter *AP) const;
+  unsigned ComputeSize(const AsmPrinter *AP) const;
 
   /// BestForm - Choose the best form for data.
   ///
@@ -496,19 +485,15 @@ public:
     return dwarf::DW_FORM_block;
   }
 
-  /// EmitValue - Emit location data.
-  ///
-  void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of location data in bytes.
-  ///
-  unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override;
-
   // Implement isa/cast/dyncast.
   static bool classof(const DIEValue *E) { return E->getType() == isLoc; }
 
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
@@ -516,13 +501,15 @@ public:
 /// DIEBlock - Represents a block of values.
 //
 class DIEBlock : public DIEValue, public DIE {
+  friend class DIEValue;
+
   mutable unsigned Size; // Size in bytes excluding size header.
 public:
   DIEBlock() : DIEValue(isBlock), Size(0) {}
 
   /// ComputeSize - Calculate the size of the location expression.
   ///
-  unsigned ComputeSize(AsmPrinter *AP) const;
+  unsigned ComputeSize(const AsmPrinter *AP) const;
 
   /// BestForm - Choose the best form for data.
   ///
@@ -536,19 +523,15 @@ public:
     return dwarf::DW_FORM_block;
   }
 
-  /// EmitValue - Emit location data.
-  ///
-  void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of location data in bytes.
-  ///
-  unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override;
-
   // Implement isa/cast/dyncast.
   static bool classof(const DIEValue *E) { return E->getType() == isBlock; }
 
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
@@ -557,6 +540,8 @@ public:
 /// section.
 //
 class DIELocList : public DIEValue {
+  friend class DIEValue;
+
   // Index into the .debug_loc vector.
   size_t Index;
 
@@ -566,19 +551,15 @@ public:
   /// getValue - Grab the current index out.
   size_t getValue() const { return Index; }
 
-  /// EmitValue - Emit location data.
-  ///
-  void EmitValue(AsmPrinter *AP, dwarf::Form Form) const override;
-
-  /// SizeOf - Determine size of location data in bytes.
-  ///
-  unsigned SizeOf(AsmPrinter *AP, dwarf::Form Form) const override;
-
   // Implement isa/cast/dyncast.
   static bool classof(const DIEValue *E) { return E->getType() == isLocList; }
 
+private:
+  void EmitValueImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+  unsigned SizeOfImpl(const AsmPrinter *AP, dwarf::Form Form) const;
+
 #ifndef NDEBUG
-  void print(raw_ostream &O) const override;
+  void printImpl(raw_ostream &O) const;
 #endif
 };
 
diff --git a/include/llvm/CodeGen/DwarfStringPoolEntry.h b/include/llvm/CodeGen/DwarfStringPoolEntry.h
new file mode 100644
index 0000000000000..fc2b5ddd2d2cf
--- /dev/null
+++ b/include/llvm/CodeGen/DwarfStringPoolEntry.h
@@ -0,0 +1,51 @@
+//===- llvm/CodeGen/DwarfStringPoolEntry.h - String pool entry --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H
+#define LLVM_CODEGEN_DWARFSTRINGPOOLENTRY_H
+
+#include "llvm/ADT/StringMap.h"
+
+namespace llvm {
+
+class MCSymbol;
+
+/// Data for a string pool entry.
+struct DwarfStringPoolEntry {
+  MCSymbol *Symbol;
+  unsigned Offset;
+  unsigned Index;
+};
+
+/// String pool entry reference.
+struct DwarfStringPoolEntryRef {
+  const StringMapEntry<DwarfStringPoolEntry> *I = nullptr;
+
+public:
+  DwarfStringPoolEntryRef() = default;
+  explicit DwarfStringPoolEntryRef(
+      const StringMapEntry<DwarfStringPoolEntry> &I)
+      : I(&I) {}
+
+  explicit operator bool() const { return I; }
+  MCSymbol *getSymbol() const {
+    assert(I->second.Symbol && "No symbol available!");
+    return I->second.Symbol;
+  }
+  unsigned getOffset() const { return I->second.Offset; }
+  unsigned getIndex() const { return I->second.Index; }
+  StringRef getString() const { return I->first(); }
+
+  bool operator==(const DwarfStringPoolEntryRef &X) const { return I == X.I; }
+  bool operator!=(const DwarfStringPoolEntryRef &X) const { return I != X.I; }
+};
+
+} // end namespace llvm
+
+#endif
diff --git a/include/llvm/CodeGen/ForwardControlFlowIntegrity.h b/include/llvm/CodeGen/ForwardControlFlowIntegrity.h
deleted file mode 100644
index ec8e2ef243b7d..0000000000000
--- a/include/llvm/CodeGen/ForwardControlFlowIntegrity.h
+++ /dev/null
@@ -1,122 +0,0 @@
-//===-- ForwardControlFlowIntegrity.h: Forward-Edge CFI ---------*- C++ -*-===//
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This pass instruments indirect calls with checks to ensure that these calls
-// pass through the appropriate jump-instruction table generated by
-// JumpInstrTables.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_FORWARDCONTROLFLOWINTEGRITY_H
-#define LLVM_CODEGEN_FORWARDCONTROLFLOWINTEGRITY_H
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Pass.h"
-#include "llvm/Target/TargetOptions.h"
-#include <string>
-
-namespace llvm {
-
-class AnalysisUsage;
-class BasicBlock;
-class Constant;
-class Function;
-class Instruction;
-class Module;
-class Value;
-
-/// ForwardControlFlowIntegrity uses the information from JumpInstrTableInfo to
-/// prepend checks to indirect calls to make sure that these calls target valid
-/// locations.
-class ForwardControlFlowIntegrity : public ModulePass {
-public:
-  static char ID;
-
-  ForwardControlFlowIntegrity();
-  ForwardControlFlowIntegrity(JumpTable::JumpTableType JTT,
-                              CFIntegrity CFIType,
-                              bool CFIEnforcing, std::string CFIFuncName);
-  ~ForwardControlFlowIntegrity() override;
-
-  /// Runs the CFI pass on a given module. This works best if the module in
-  /// question is the result of link-time optimization (see lib/LTO).
-  bool runOnModule(Module &M) override;
-  const char *getPassName() const override {
-    return "Forward Control-Flow Integrity";
-  }
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-
-private:
-  typedef SmallVector<Instruction *, 64> CallSet;
-
-  /// A structure that is used to keep track of constant table information.
-  struct CFIConstants {
-    Constant *StartValue;
-    Constant *MaskValue;
-    Constant *Size;
-  };
-
-  /// A map from function type to the base of the table for this type and a mask
-  /// for the table
-  typedef DenseMap<FunctionType *, CFIConstants> CFITables;
-
-  CallSet IndirectCalls;
-
-  /// The type of jumptable implementation.
-  JumpTable::JumpTableType JTType;
-
-  /// The type of CFI check to add before each indirect call.
-  CFIntegrity CFIType;
-
-  /// A value that controls whether or not CFI violations cause a halt.
-  bool CFIEnforcing;
-
-  /// The name of the function to call in case of a CFI violation when
-  /// CFIEnforcing is false. There is a default function that ignores
-  /// violations.
-  std::string CFIFuncName;
-
-  /// The alignment of each entry in the table, from JumpInstrTableInfo. The
-  /// JumpInstrTableInfo class always makes this a power of two.
-  uint64_t ByteAlignment;
-
-  /// The base-2 logarithm of ByteAlignment, needed for some of the transforms
-  /// (like CFIntegrity::Ror)
-  unsigned LogByteAlignment;
-
-  /// Adds checks to each indirect call site to make sure that it is calling a
-  /// function in our jump table.
-  void updateIndirectCalls(Module &M, CFITables &CFIT);
-
-  /// Walks the instructions to find all the indirect calls.
-  void getIndirectCalls(Module &M);
-
-  /// Adds a function that handles violations in non-enforcing mode
-  /// (!CFIEnforcing). The default warning function simply returns, since the
-  /// exact details of how to handle CFI violations depend on the application.
-  void addWarningFunction(Module &M);
-
-  /// Rewrites a function pointer in a call/invoke instruction to force it into
-  /// a table.
-  void rewriteFunctionPointer(Module &M, Instruction *I, Value *FunPtr,
-                              Constant *JumpTableStart, Constant *JumpTableMask,
-                              Constant *JumpTableSize);
-
-  /// Inserts a check and a call to a warning function at a given instruction
-  /// that must be an indirect call.
-  void insertWarning(Module &M, BasicBlock *Block, Instruction *I,
-                     Value *FunPtr);
-};
-
-ModulePass *
-createForwardControlFlowIntegrityPass(JumpTable::JumpTableType JTT,
-                                      CFIntegrity CFIType,
-                                      bool CFIEnforcing, StringRef CFIFuncName);
-}
-
-#endif // LLVM_CODEGEN_FORWARDCONTROLFLOWINTEGRITY_H
diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h
index 7c574df4ba414..82c762ed850fe 100644
--- a/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -18,6 +18,7 @@
 #include "llvm/ADT/APInt.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/IndexedMap.h"
+#include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
@@ -73,6 +74,16 @@ public:
   /// cross-basic-block values.
   DenseMap<const Value*, unsigned> ValueMap;
 
+  // Keep track of frame indices allocated for statepoints as they could be used
+  // across basic block boundaries.
+  // Key of the map is statepoint instruction, value is a map from spilled
+  // llvm Value to the optional stack stack slot index.
+  // If optional is unspecified it means that we have visited this value
+  // but didn't spill it.
+  typedef DenseMap<const Value*, Optional<int>> StatepointSpilledValueMapTy;
+  DenseMap<const Instruction*, StatepointSpilledValueMapTy>
+    StatepointRelocatedValues;
+
   /// StaticAllocaMap - Keep track of frame indices for fixed sized allocas in
   /// the entry block.  This allows the allocas to be efficiently referenced
   /// anywhere in the function.
@@ -221,6 +232,8 @@ public:
   int getArgumentFrameIndex(const Argument *A);
 
 private:
+  void addSEHHandlersForLPads(ArrayRef<const LandingPadInst *> LPads);
+
   /// LiveOutRegInfo - Information about live out vregs.
   IndexedMap<LiveOutInfo, VirtReg2IndexFunctor> LiveOutRegInfo;
 };
@@ -232,11 +245,6 @@ private:
 /// floating-point support.
 void ComputeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo *MMI);
 
-/// AddCatchInfo - Extract the personality and type infos from an eh.selector
-/// call, and add them to the specified machine basic block.
-void AddCatchInfo(const CallInst &I,
-                  MachineModuleInfo *MMI, MachineBasicBlock *MBB);
-
 /// AddLandingPadInfo - Extract the exception handling information from the
 /// landingpad instruction and add them to the specified machine module info.
 void AddLandingPadInfo(const LandingPadInst &I, MachineModuleInfo &MMI,
diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h
index c7f1ab87fcb11..357b2d8a7ca7a 100644
--- a/include/llvm/CodeGen/GCMetadata.h
+++ b/include/llvm/CodeGen/GCMetadata.h
@@ -34,184 +34,173 @@
 #define LLVM_CODEGEN_GCMETADATA_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/CodeGen/GCStrategy.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/Pass.h"
 #include <memory>
 
 namespace llvm {
-  class AsmPrinter;
-  class GCStrategy;
-  class Constant;
-  class MCSymbol;
-
-  namespace GC {
-    /// PointKind - The type of a collector-safe point.
-    ///
-    enum PointKind {
-      Loop,    ///< Instr is a loop (backwards branch).
-      Return,  ///< Instr is a return instruction.
-      PreCall, ///< Instr is a call instruction.
-      PostCall ///< Instr is the return address of a call.
-    };
+class AsmPrinter;
+class Constant;
+class MCSymbol;
+
+/// GCPoint - Metadata for a collector-safe point in machine code.
+///
+struct GCPoint {
+  GC::PointKind Kind; ///< The kind of the safe point.
+  MCSymbol *Label;    ///< A label.
+  DebugLoc Loc;
+
+  GCPoint(GC::PointKind K, MCSymbol *L, DebugLoc DL)
+      : Kind(K), Label(L), Loc(DL) {}
+};
+
+/// GCRoot - Metadata for a pointer to an object managed by the garbage
+/// collector.
+struct GCRoot {
+  int Num;                  ///< Usually a frame index.
+  int StackOffset;          ///< Offset from the stack pointer.
+  const Constant *Metadata; ///< Metadata straight from the call
+                            ///< to llvm.gcroot.
+
+  GCRoot(int N, const Constant *MD) : Num(N), StackOffset(-1), Metadata(MD) {}
+};
+
+/// Garbage collection metadata for a single function.  Currently, this
+/// information only applies to GCStrategies which use GCRoot.
+class GCFunctionInfo {
+public:
+  typedef std::vector<GCPoint>::iterator iterator;
+  typedef std::vector<GCRoot>::iterator roots_iterator;
+  typedef std::vector<GCRoot>::const_iterator live_iterator;
+
+private:
+  const Function &F;
+  GCStrategy &S;
+  uint64_t FrameSize;
+  std::vector<GCRoot> Roots;
+  std::vector<GCPoint> SafePoints;
+
+  // FIXME: Liveness. A 2D BitVector, perhaps?
+  //
+  //   BitVector Liveness;
+  //
+  //   bool islive(int point, int root) =
+  //     Liveness[point * SafePoints.size() + root]
+  //
+  // The bit vector is the more compact representation where >3.2% of roots
+  // are live per safe point (1.5% on 64-bit hosts).
+
+public:
+  GCFunctionInfo(const Function &F, GCStrategy &S);
+  ~GCFunctionInfo();
+
+  /// getFunction - Return the function to which this metadata applies.
+  ///
+  const Function &getFunction() const { return F; }
+
+  /// getStrategy - Return the GC strategy for the function.
+  ///
+  GCStrategy &getStrategy() { return S; }
+
+  /// addStackRoot - Registers a root that lives on the stack. Num is the
+  ///                stack object ID for the alloca (if the code generator is
+  //                 using  MachineFrameInfo).
+  void addStackRoot(int Num, const Constant *Metadata) {
+    Roots.push_back(GCRoot(Num, Metadata));
+  }
+
+  /// removeStackRoot - Removes a root.
+  roots_iterator removeStackRoot(roots_iterator position) {
+    return Roots.erase(position);
+  }
+
+  /// addSafePoint - Notes the existence of a safe point. Num is the ID of the
+  /// label just prior to the safe point (if the code generator is using
+  /// MachineModuleInfo).
+  void addSafePoint(GC::PointKind Kind, MCSymbol *Label, DebugLoc DL) {
+    SafePoints.push_back(GCPoint(Kind, Label, DL));
   }
 
-  /// GCPoint - Metadata for a collector-safe point in machine code.
+  /// getFrameSize/setFrameSize - Records the function's frame size.
   ///
-  struct GCPoint {
-    GC::PointKind Kind; ///< The kind of the safe point.
-    MCSymbol *Label;    ///< A label.
-    DebugLoc Loc;
-
-    GCPoint(GC::PointKind K, MCSymbol *L, DebugLoc DL)
-        : Kind(K), Label(L), Loc(DL) {}
-  };
-
-  /// GCRoot - Metadata for a pointer to an object managed by the garbage
-  /// collector.
-  struct GCRoot {
-    int Num;            ///< Usually a frame index.
-    int StackOffset;    ///< Offset from the stack pointer.
-    const Constant *Metadata; ///< Metadata straight from the call
-                              ///< to llvm.gcroot.
-
-    GCRoot(int N, const Constant *MD) : Num(N), StackOffset(-1), Metadata(MD) {}
-  };
-
-
-  /// Garbage collection metadata for a single function.  Currently, this
-  /// information only applies to GCStrategies which use GCRoot.
-  class GCFunctionInfo {
-  public:
-    typedef std::vector<GCPoint>::iterator iterator;
-    typedef std::vector<GCRoot>::iterator roots_iterator;
-    typedef std::vector<GCRoot>::const_iterator live_iterator;
-
-  private:
-    const Function &F;
-    GCStrategy &S;
-    uint64_t FrameSize;
-    std::vector<GCRoot> Roots;
-    std::vector<GCPoint> SafePoints;
-
-    // FIXME: Liveness. A 2D BitVector, perhaps?
-    //
-    //   BitVector Liveness;
-    //
-    //   bool islive(int point, int root) =
-    //     Liveness[point * SafePoints.size() + root]
-    //
-    // The bit vector is the more compact representation where >3.2% of roots
-    // are live per safe point (1.5% on 64-bit hosts).
-
-  public:
-    GCFunctionInfo(const Function &F, GCStrategy &S);
-    ~GCFunctionInfo();
-
-    /// getFunction - Return the function to which this metadata applies.
-    ///
-    const Function &getFunction() const { return F; }
-
-    /// getStrategy - Return the GC strategy for the function.
-    ///
-    GCStrategy &getStrategy() { return S; }
-
-    /// addStackRoot - Registers a root that lives on the stack. Num is the
-    ///                stack object ID for the alloca (if the code generator is
-    //                 using  MachineFrameInfo).
-    void addStackRoot(int Num, const Constant *Metadata) {
-      Roots.push_back(GCRoot(Num, Metadata));
-    }
-
-    /// removeStackRoot - Removes a root.
-    roots_iterator removeStackRoot(roots_iterator position) {
-      return Roots.erase(position);
-    }
-
-    /// addSafePoint - Notes the existence of a safe point. Num is the ID of the
-    /// label just prior to the safe point (if the code generator is using
-    /// MachineModuleInfo).
-    void addSafePoint(GC::PointKind Kind, MCSymbol *Label, DebugLoc DL) {
-      SafePoints.push_back(GCPoint(Kind, Label, DL));
-    }
-
-    /// getFrameSize/setFrameSize - Records the function's frame size.
-    ///
-    uint64_t getFrameSize() const { return FrameSize; }
-    void setFrameSize(uint64_t S) { FrameSize = S; }
-
-    /// begin/end - Iterators for safe points.
-    ///
-    iterator begin() { return SafePoints.begin(); }
-    iterator end()   { return SafePoints.end();   }
-    size_t size() const { return SafePoints.size(); }
-
-    /// roots_begin/roots_end - Iterators for all roots in the function.
-    ///
-    roots_iterator roots_begin() { return Roots.begin(); }
-    roots_iterator roots_end  () { return Roots.end();   }
-    size_t roots_size() const { return Roots.size(); }
-
-    /// live_begin/live_end - Iterators for live roots at a given safe point.
-    ///
-    live_iterator live_begin(const iterator &p) { return roots_begin(); }
-    live_iterator live_end  (const iterator &p) { return roots_end();   }
-    size_t live_size(const iterator &p) const { return roots_size(); }
-  };
-
-  /// An analysis pass which caches information about the entire Module.
-  /// Records both the function level information used by GCRoots and a
-  /// cache of the 'active' gc strategy objects for the current Module.
-  class GCModuleInfo : public ImmutablePass {
-    typedef StringMap<GCStrategy*> strategy_map_type;
-    typedef std::vector<std::unique_ptr<GCStrategy>> list_type;
-
-    strategy_map_type StrategyMap;
-    list_type StrategyList;
-
-    GCStrategy *getOrCreateStrategy(const Module *M, const std::string &Name);
-
-  public:
-    /// List of per function info objects.  In theory, Each of these
-    /// may be associated with a different GC.
-    typedef std::vector<std::unique_ptr<GCFunctionInfo>> FuncInfoVec;
-
-    FuncInfoVec::iterator funcinfo_begin() { return Functions.begin(); }
-    FuncInfoVec::iterator funcinfo_end()   { return Functions.end();   }
-    
-
-  private:
-    /// Owning list of all GCFunctionInfos associated with this Module
-    FuncInfoVec Functions;
-
-    /// Non-owning map to bypass linear search when finding the GCFunctionInfo
-    /// associated with a particular Function.
-    typedef DenseMap<const Function*,GCFunctionInfo*> finfo_map_type;
-    finfo_map_type FInfoMap;
-  public:
-
-    typedef list_type::const_iterator iterator;
-
-    static char ID;
-
-    GCModuleInfo();
-
-    /// clear - Resets the pass. Any pass, which uses GCModuleInfo, should
-    /// call it in doFinalization().
-    ///
-    void clear();
-
-    /// begin/end - Iterators for used strategies.
-    ///
-    iterator begin() const { return StrategyList.begin(); }
-    iterator end()   const { return StrategyList.end();   }
-
-    /// get - Look up function metadata.  This is currently assumed
-    /// have the side effect of initializing the associated GCStrategy.  That
-    /// will soon change.
-    GCFunctionInfo &getFunctionInfo(const Function &F);
-  };
+  uint64_t getFrameSize() const { return FrameSize; }
+  void setFrameSize(uint64_t S) { FrameSize = S; }
 
+  /// begin/end - Iterators for safe points.
+  ///
+  iterator begin() { return SafePoints.begin(); }
+  iterator end() { return SafePoints.end(); }
+  size_t size() const { return SafePoints.size(); }
+
+  /// roots_begin/roots_end - Iterators for all roots in the function.
+  ///
+  roots_iterator roots_begin() { return Roots.begin(); }
+  roots_iterator roots_end() { return Roots.end(); }
+  size_t roots_size() const { return Roots.size(); }
+
+  /// live_begin/live_end - Iterators for live roots at a given safe point.
+  ///
+  live_iterator live_begin(const iterator &p) { return roots_begin(); }
+  live_iterator live_end(const iterator &p) { return roots_end(); }
+  size_t live_size(const iterator &p) const { return roots_size(); }
+};
+
+/// An analysis pass which caches information about the entire Module.
+/// Records both the function level information used by GCRoots and a
+/// cache of the 'active' gc strategy objects for the current Module.
+class GCModuleInfo : public ImmutablePass {
+  /// An owning list of all GCStrategies which have been created
+  SmallVector<std::unique_ptr<GCStrategy>, 1> GCStrategyList;
+  /// A helper map to speedup lookups into the above list
+  StringMap<GCStrategy*> GCStrategyMap;
+
+public:
+  /// Lookup the GCStrategy object associated with the given gc name.
+  /// Objects are owned internally; No caller should attempt to delete the
+  /// returned objects. 
+  GCStrategy *getGCStrategy(const StringRef Name);
+  
+  /// List of per function info objects.  In theory, Each of these
+  /// may be associated with a different GC.
+  typedef std::vector<std::unique_ptr<GCFunctionInfo>> FuncInfoVec;
+
+  FuncInfoVec::iterator funcinfo_begin() { return Functions.begin(); }
+  FuncInfoVec::iterator funcinfo_end() { return Functions.end(); }
+
+private:
+  /// Owning list of all GCFunctionInfos associated with this Module
+  FuncInfoVec Functions;
+
+  /// Non-owning map to bypass linear search when finding the GCFunctionInfo
+  /// associated with a particular Function.
+  typedef DenseMap<const Function *, GCFunctionInfo *> finfo_map_type;
+  finfo_map_type FInfoMap;
+
+public:
+  typedef SmallVector<std::unique_ptr<GCStrategy>,1>::const_iterator iterator;
+
+  static char ID;
+
+  GCModuleInfo();
+
+  /// clear - Resets the pass. Any pass, which uses GCModuleInfo, should
+  /// call it in doFinalization().
+  ///
+  void clear();
+
+  /// begin/end - Iterators for used strategies.
+  ///
+  iterator begin() const { return GCStrategyList.begin(); }
+  iterator end() const { return GCStrategyList.end(); }
+
+  /// get - Look up function metadata.  This is currently assumed
+  /// have the side effect of initializing the associated GCStrategy.  That
+  /// will soon change.
+  GCFunctionInfo &getFunctionInfo(const Function &F);
+};
 }
 
 #endif
diff --git a/include/llvm/CodeGen/GCMetadataPrinter.h b/include/llvm/CodeGen/GCMetadataPrinter.h
index 25fafba93f8b6..2208470291138 100644
--- a/include/llvm/CodeGen/GCMetadataPrinter.h
+++ b/include/llvm/CodeGen/GCMetadataPrinter.h
@@ -26,43 +26,39 @@
 
 namespace llvm {
 
-  class GCMetadataPrinter;
+class GCMetadataPrinter;
 
-  /// GCMetadataPrinterRegistry - The GC assembly printer registry uses all the
-  /// defaults from Registry.
-  typedef Registry<GCMetadataPrinter> GCMetadataPrinterRegistry;
+/// GCMetadataPrinterRegistry - The GC assembly printer registry uses all the
+/// defaults from Registry.
+typedef Registry<GCMetadataPrinter> GCMetadataPrinterRegistry;
 
-  /// GCMetadataPrinter - Emits GC metadata as assembly code.  Instances are
-  /// created, managed, and owned by the AsmPrinter.
-  class GCMetadataPrinter {
-  private:
-    GCStrategy *S;
-    friend class AsmPrinter;
+/// GCMetadataPrinter - Emits GC metadata as assembly code.  Instances are
+/// created, managed, and owned by the AsmPrinter.
+class GCMetadataPrinter {
+private:
+  GCStrategy *S;
+  friend class AsmPrinter;
 
-  protected:
-    // May only be subclassed.
-    GCMetadataPrinter();
+protected:
+  // May only be subclassed.
+  GCMetadataPrinter();
 
-  private:
-    GCMetadataPrinter(const GCMetadataPrinter &) LLVM_DELETED_FUNCTION;
-    GCMetadataPrinter &
-      operator=(const GCMetadataPrinter &) LLVM_DELETED_FUNCTION;
+private:
+  GCMetadataPrinter(const GCMetadataPrinter &) = delete;
+  GCMetadataPrinter &operator=(const GCMetadataPrinter &) = delete;
 
-  public:
-    GCStrategy &getStrategy() { return *S; }
+public:
+  GCStrategy &getStrategy() { return *S; }
 
-    /// Called before the assembly for the module is generated by
-    /// the AsmPrinter (but after target specific hooks.)
-    virtual void beginAssembly(Module &M, GCModuleInfo &Info,
-                               AsmPrinter &AP) {}
-    /// Called after the assembly for the module is generated by
-    /// the AsmPrinter (but before target specific hooks)
-    virtual void finishAssembly(Module &M, GCModuleInfo &Info,
-                                AsmPrinter &AP) {}
-
-    virtual ~GCMetadataPrinter();
-  };
+  /// Called before the assembly for the module is generated by
+  /// the AsmPrinter (but after target specific hooks.)
+  virtual void beginAssembly(Module &M, GCModuleInfo &Info, AsmPrinter &AP) {}
+  /// Called after the assembly for the module is generated by
+  /// the AsmPrinter (but before target specific hooks)
+  virtual void finishAssembly(Module &M, GCModuleInfo &Info, AsmPrinter &AP) {}
 
+  virtual ~GCMetadataPrinter();
+};
 }
 
 #endif
diff --git a/include/llvm/CodeGen/GCStrategy.h b/include/llvm/CodeGen/GCStrategy.h
index 0b0c3124c537c..a1b8e895898fe 100644
--- a/include/llvm/CodeGen/GCStrategy.h
+++ b/include/llvm/CodeGen/GCStrategy.h
@@ -16,11 +16,11 @@
 // GCStrategy is relevant for implementations using either gc.root or
 // gc.statepoint based lowering strategies, but is currently focused mostly on
 // options for gc.root.  This will change over time.
-// 
+//
 // When requested by a subclass of GCStrategy, the gc.root implementation will
 // populate GCModuleInfo and GCFunctionInfo with that about each Function in
 // the Module that opts in to garbage collection.  Specifically:
-// 
+//
 // - Safe points
 //   Garbage collection is generally only possible at certain points in code.
 //   GCStrategy can request that the collector insert such points:
@@ -28,7 +28,7 @@
 //     - At and after any call to a subroutine
 //     - Before returning from the current function
 //     - Before backwards branches (loops)
-// 
+//
 // - Roots
 //   When a reference to a GC-allocated object exists on the stack, it must be
 //   stored in an alloca registered with llvm.gcoot.
@@ -42,155 +42,136 @@
 // insertion support is planned.  gc.statepoint does not currently support
 // custom stack map formats; such can be generated by parsing the standard
 // stack map section if desired.
-// 
+//
 // The read and write barrier support can be used with either implementation.
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CODEGEN_GCSTRATEGY_H
-#define LLVM_CODEGEN_GCSTRATEGY_H
+#ifndef LLVM_IR_GCSTRATEGY_H
+#define LLVM_IR_GCSTRATEGY_H
 
 #include "llvm/ADT/Optional.h"
-#include "llvm/CodeGen/GCMetadata.h"
-#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Module.h"
+#include "llvm/IR/Value.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Registry.h"
 #include <string>
 
 namespace llvm {
-  /// GCStrategy describes a garbage collector algorithm's code generation
-  /// requirements, and provides overridable hooks for those needs which cannot
-  /// be abstractly described.  GCStrategy objects currently must be looked up
-  /// through the GCModuleInfo analysis pass.  They are owned by the analysis
-  /// pass and recreated every time that pass is invalidated.
-  class GCStrategy {
-  private:
-    std::string Name;
-    friend class GCModuleInfo;
-    
-  protected:
-    bool UseStatepoints;       /// Uses gc.statepoints as opposed to gc.roots,
-                               /// if set, none of the other options can be
-                               /// anything but their default values.
-
-    unsigned NeededSafePoints; ///< Bitmask of required safe points.
-    bool CustomReadBarriers;   ///< Default is to insert loads.
-    bool CustomWriteBarriers;  ///< Default is to insert stores.
-    bool CustomRoots;          ///< Default is to pass through to backend.
-    bool CustomSafePoints;     ///< Default is to use NeededSafePoints
-                               ///< to find safe points.
-    bool InitRoots;            ///< If set, roots are nulled during lowering.
-    bool UsesMetadata;         ///< If set, backend must emit metadata tables.
-    
-  public:
-    GCStrategy();
-    virtual ~GCStrategy() {}
-    
-    /// Return the name of the GC strategy.  This is the value of the collector
-    /// name string specified on functions which use this strategy.
-    const std::string &getName() const { return Name; }
-
-    /// By default, write barriers are replaced with simple store
-    /// instructions. If true, then performCustomLowering must instead lower
-    /// them. 
-    bool customWriteBarrier() const { return CustomWriteBarriers; }
-    
-    /// By default, read barriers are replaced with simple load
-    /// instructions. If true, then performCustomLowering must instead lower
-    /// them. 
-    bool customReadBarrier() const { return CustomReadBarriers; }
-
-    /// Returns true if this strategy is expecting the use of gc.statepoints,
-    /// and false otherwise.
-    bool useStatepoints() const { return UseStatepoints; }
-
-    /** @name Statepoint Specific Properties */
-    ///@{
-
-    /// If the value specified can be reliably distinguished, returns true for
-    /// pointers to GC managed locations and false for pointers to non-GC 
-    /// managed locations.  Note a GCStrategy can always return 'None' (i.e. an
-    /// empty optional indicating it can't reliably distinguish.   
-    virtual Optional<bool> isGCManagedPointer(const Value *V) const {
-      return None;
-    }
-    ///@}
-
-    /** @name GCRoot Specific Properties
-     * These properties and overrides only apply to collector strategies using
-     * GCRoot. 
-     */
-    ///@{
-
-    /// True if safe points of any kind are required. By default, none are
-    /// recorded. 
-    bool needsSafePoints() const {
-      return CustomSafePoints || NeededSafePoints != 0;
-    }
-    
-    /// True if the given kind of safe point is required. By default, none are
-    /// recorded. 
-    bool needsSafePoint(GC::PointKind Kind) const {
-      return (NeededSafePoints & 1 << Kind) != 0;
-    }
-        
-    /// By default, roots are left for the code generator so it can generate a
-    /// stack map. If true, then performCustomLowering must delete them.
-    bool customRoots() const { return CustomRoots; }
-
-    /// By default, the GC analysis will find safe points according to
-    /// NeededSafePoints. If true, then findCustomSafePoints must create them.
-    bool customSafePoints() const { return CustomSafePoints; }
-    
-    /// If set, gcroot intrinsics should initialize their allocas to null
-    /// before the first use. This is necessary for most GCs and is enabled by
-    /// default. 
-    bool initializeRoots() const { return InitRoots; }
-    
-    /// If set, appropriate metadata tables must be emitted by the back-end
-    /// (assembler, JIT, or otherwise). For statepoint, this method is
-    /// currently unsupported.  The stackmap information can be found in the
-    /// StackMap section as described in the documentation.
-    bool usesMetadata() const { return UsesMetadata; }
-
-    ///@}
-    
-    /// initializeCustomLowering/performCustomLowering - If any of the actions
-    /// are set to custom, performCustomLowering must be overriden to transform
-    /// the corresponding actions to LLVM IR. initializeCustomLowering is
-    /// optional to override. These are the only GCStrategy methods through
-    /// which the LLVM IR can be modified.  These methods apply mostly to
-    /// gc.root based implementations, but can be overriden to provide custom
-    /// barrier lowerings with gc.statepoint as well.
-    ///@{
-    virtual bool initializeCustomLowering(Module &F) {
-      // No changes made
-      return false;
-    }
-    virtual bool performCustomLowering(Function &F) {
-      llvm_unreachable("GCStrategy subclass specified a configuration which"
-                       "requires a custom lowering without providing one");
-    }
-    ///@}
-    /// Called if customSafepoints returns true, used only by gc.root
-    /// implementations. 
-    virtual bool findCustomSafePoints(GCFunctionInfo& FI, MachineFunction& MF) {
-      llvm_unreachable("GCStrategy subclass specified a configuration which"
-                       "requests custom safepoint identification without"
-                       "providing an implementation for such");
-    }
-  };
-
-  /// Subclasses of GCStrategy are made available for use during compilation by
-  /// adding them to the global GCRegistry.  This can done either within the
-  /// LLVM source tree or via a loadable plugin.  An example registeration
-  /// would be:
-  /// static GCRegistry::Add<CustomGC> X("custom-name",
-  ///        "my custom supper fancy gc strategy");
-  /// 
-  /// Note that to use a custom GCMetadataPrinter w/gc.roots, you must also
-  /// register your GCMetadataPrinter subclass with the
-  /// GCMetadataPrinterRegistery as well. 
-  typedef Registry<GCStrategy> GCRegistry;
+namespace GC {
+/// PointKind - Used to indicate whether the address of the call instruction
+/// or the address after the call instruction is listed in the stackmap.  For
+/// most runtimes, PostCall safepoints are appropriate.
+///
+enum PointKind {
+  PreCall, ///< Instr is a call instruction.
+  PostCall ///< Instr is the return address of a call.
+};
+}
+
+/// GCStrategy describes a garbage collector algorithm's code generation
+/// requirements, and provides overridable hooks for those needs which cannot
+/// be abstractly described.  GCStrategy objects must be looked up through
+/// the Function.  The objects themselves are owned by the Context and must
+/// be immutable.
+class GCStrategy {
+private:
+  std::string Name;
+  friend class GCModuleInfo;
+
+protected:
+  bool UseStatepoints; /// Uses gc.statepoints as opposed to gc.roots,
+                       /// if set, none of the other options can be
+                       /// anything but their default values.
+
+  unsigned NeededSafePoints; ///< Bitmask of required safe points.
+  bool CustomReadBarriers;   ///< Default is to insert loads.
+  bool CustomWriteBarriers;  ///< Default is to insert stores.
+  bool CustomRoots;          ///< Default is to pass through to backend.
+  bool InitRoots;            ///< If set, roots are nulled during lowering.
+  bool UsesMetadata;         ///< If set, backend must emit metadata tables.
+
+public:
+  GCStrategy();
+  virtual ~GCStrategy() {}
+
+  /// Return the name of the GC strategy.  This is the value of the collector
+  /// name string specified on functions which use this strategy.
+  const std::string &getName() const { return Name; }
+
+  /// By default, write barriers are replaced with simple store
+  /// instructions. If true, you must provide a custom pass to lower 
+  /// calls to @llvm.gcwrite.
+  bool customWriteBarrier() const { return CustomWriteBarriers; }
+
+  /// By default, read barriers are replaced with simple load
+  /// instructions. If true, you must provide a custom pass to lower 
+  /// calls to @llvm.gcread.
+  bool customReadBarrier() const { return CustomReadBarriers; }
+
+  /// Returns true if this strategy is expecting the use of gc.statepoints,
+  /// and false otherwise.
+  bool useStatepoints() const { return UseStatepoints; }
+
+  /** @name Statepoint Specific Properties */
+  ///@{
+
+  /// If the value specified can be reliably distinguished, returns true for
+  /// pointers to GC managed locations and false for pointers to non-GC
+  /// managed locations.  Note a GCStrategy can always return 'None' (i.e. an
+  /// empty optional indicating it can't reliably distinguish.
+  virtual Optional<bool> isGCManagedPointer(const Value *V) const {
+    return None;
+  }
+  ///@}
+
+  /** @name GCRoot Specific Properties
+   * These properties and overrides only apply to collector strategies using
+   * GCRoot.
+   */
+  ///@{
+
+  /// True if safe points of any kind are required. By default, none are
+  /// recorded.
+  bool needsSafePoints() const { return NeededSafePoints != 0; }
+
+  /// True if the given kind of safe point is required. By default, none are
+  /// recorded.
+  bool needsSafePoint(GC::PointKind Kind) const {
+    return (NeededSafePoints & 1 << Kind) != 0;
+  }
+
+  /// By default, roots are left for the code generator so it can generate a
+  /// stack map. If true, you must provide a custom pass to lower 
+  /// calls to @llvm.gcroot.
+  bool customRoots() const { return CustomRoots; }
+
+  /// If set, gcroot intrinsics should initialize their allocas to null
+  /// before the first use. This is necessary for most GCs and is enabled by
+  /// default.
+  bool initializeRoots() const { return InitRoots; }
+
+  /// If set, appropriate metadata tables must be emitted by the back-end
+  /// (assembler, JIT, or otherwise). For statepoint, this method is
+  /// currently unsupported.  The stackmap information can be found in the
+  /// StackMap section as described in the documentation.
+  bool usesMetadata() const { return UsesMetadata; }
+
+  ///@}
+};
+
+/// Subclasses of GCStrategy are made available for use during compilation by
+/// adding them to the global GCRegistry.  This can done either within the
+/// LLVM source tree or via a loadable plugin.  An example registeration
+/// would be:
+/// static GCRegistry::Add<CustomGC> X("custom-name",
+///        "my custom supper fancy gc strategy");
+///
+/// Note that to use a custom GCMetadataPrinter w/gc.roots, you must also
+/// register your GCMetadataPrinter subclass with the
+/// GCMetadataPrinterRegistery as well.
+typedef Registry<GCStrategy> GCRegistry;
 }
 
 #endif
diff --git a/include/llvm/CodeGen/GCs.h b/include/llvm/CodeGen/GCs.h
index 51a31842a5c00..5207f801c84ea 100644
--- a/include/llvm/CodeGen/GCs.h
+++ b/include/llvm/CodeGen/GCs.h
@@ -15,29 +15,32 @@
 #define LLVM_CODEGEN_GCS_H
 
 namespace llvm {
-  class GCStrategy;
-  class GCMetadataPrinter;
+class GCStrategy;
+class GCMetadataPrinter;
 
-  /// FIXME: Collector instances are not useful on their own. These no longer
-  ///        serve any purpose except to link in the plugins.
+/// FIXME: Collector instances are not useful on their own. These no longer
+///        serve any purpose except to link in the plugins.
 
-  /// Creates an ocaml-compatible garbage collector.
-  void linkOcamlGC();
+/// Creates a CoreCLR-compatible garbage collector.
+void linkCoreCLRGC();
 
-  /// Creates an ocaml-compatible metadata printer.
-  void linkOcamlGCPrinter();
+/// Creates an ocaml-compatible garbage collector.
+void linkOcamlGC();
 
-  /// Creates an erlang-compatible garbage collector.
-  void linkErlangGC();
+/// Creates an ocaml-compatible metadata printer.
+void linkOcamlGCPrinter();
 
-  /// Creates an erlang-compatible metadata printer.
-  void linkErlangGCPrinter();
+/// Creates an erlang-compatible garbage collector.
+void linkErlangGC();
 
-  /// Creates a shadow stack garbage collector. This collector requires no code
-  /// generator support.
-  void linkShadowStackGC();
+/// Creates an erlang-compatible metadata printer.
+void linkErlangGCPrinter();
 
-  void linkStatepointExampleGC();
+/// Creates a shadow stack garbage collector. This collector requires no code
+/// generator support.
+void linkShadowStackGC();
+
+void linkStatepointExampleGC();
 }
 
 #endif
diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h
index 952362ed6ce3b..c2071fe5dd2d7 100644
--- a/include/llvm/CodeGen/ISDOpcodes.h
+++ b/include/llvm/CodeGen/ISDOpcodes.h
@@ -229,11 +229,18 @@ namespace ISD {
     SMULO, UMULO,
 
     /// Simple binary floating point operators.
-    FADD, FSUB, FMUL, FMA, FDIV, FREM,
+    FADD, FSUB, FMUL, FDIV, FREM,
+
+    /// FMA - Perform a * b + c with no intermediate rounding step.
+    FMA,
+
+    /// FMAD - Perform a * b + c, while getting the same result as the
+    /// separately rounded operations.
+    FMAD,
 
     /// FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.  NOTE: This
-    /// DAG node does not require that X and Y have the same type, just that the
-    /// are both floating point.  X and the result must have the same type.
+    /// DAG node does not require that X and Y have the same type, just that
+    /// they are both floating point.  X and the result must have the same type.
     /// FCOPYSIGN(f32, f64) is allowed.
     FCOPYSIGN,
 
@@ -301,6 +308,10 @@ namespace ISD {
     /// part.
     MULHU, MULHS,
 
+    /// [US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned
+    /// integers.
+    SMIN, SMAX, UMIN, UMAX,
+
     /// Bitwise operators - logical and, logical or, logical xor.
     AND, OR, XOR,
 
@@ -680,13 +691,29 @@ namespace ISD {
     ATOMIC_LOAD_UMIN,
     ATOMIC_LOAD_UMAX,
 
-    // Masked load and store
+    // Masked load and store - consecutive vector load and store operations
+    // with additional mask operand that prevents memory accesses to the
+    // masked-off lanes.
     MLOAD, MSTORE,
 
+    // Masked gather and scatter - load and store operations for a vector of
+    // random addresses with additional mask operand that prevents memory
+    // accesses to the masked-off lanes.
+    MGATHER, MSCATTER,
+
     /// This corresponds to the llvm.lifetime.* intrinsics. The first operand
     /// is the chain and the second operand is the alloca pointer.
     LIFETIME_START, LIFETIME_END,
 
+    /// GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the
+    /// beginning and end of GC transition  sequence, and carry arbitrary
+    /// information that target might need for lowering.  The first operand is
+    /// a chain, the rest are specified by the target and not touched by the DAG
+    /// optimizers. GC_TRANSITION_START..GC_TRANSITION_END pairs may not be
+    /// nested.
+    GC_TRANSITION_START,
+    GC_TRANSITION_END,
+
     /// BUILTIN_OP_END - This must be the last enum value in this list.
     /// The target-specific pre-isel opcode values start here.
     BUILTIN_OP_END
@@ -696,7 +723,7 @@ namespace ISD {
   /// which do not reference a specific memory location should be less than
   /// this value. Those that do must not be less than this value, and can
   /// be used with SelectionDAG::getMemIntrinsicNode.
-  static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+180;
+  static const int FIRST_TARGET_MEMORY_OPCODE = BUILTIN_OP_END+200;
 
   //===--------------------------------------------------------------------===//
   /// MemIndexedMode enum - This enum defines the load / store indexed
diff --git a/include/llvm/CodeGen/JumpInstrTables.h b/include/llvm/CodeGen/JumpInstrTables.h
deleted file mode 100644
index 005bc1eb2b2de..0000000000000
--- a/include/llvm/CodeGen/JumpInstrTables.h
+++ /dev/null
@@ -1,105 +0,0 @@
-//===-- JumpInstrTables.h: Jump-Instruction Tables --------------*- C++ -*-===//
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-///
-/// \file
-/// \brief An implementation of tables consisting of jump instructions
-///
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_JUMPINSTRTABLES_H
-#define LLVM_CODEGEN_JUMPINSTRTABLES_H
-
-#include "llvm/ADT/DenseMap.h"
-#include "llvm/Pass.h"
-#include "llvm/Target/TargetOptions.h"
-
-namespace llvm {
-class Constant;
-class Function;
-class FunctionType;
-class JumpInstrTableInfo;
-class Module;
-
-/// A class to manage a set of jump tables indexed on function type. It looks at
-/// each function in the module to find all the functions that have the
-/// jumptable attribute set. For each such function, it creates a new
-/// jump-instruction-table function and stores the mapping in the ImmutablePass
-/// JumpInstrTableInfo.
-///
-/// These special functions get lowered in AsmPrinter to assembly of the form:
-/// \verbatim
-///   .globl f
-///   .type f,@function
-///   .align 8,0x90
-/// f:
-///   jmp f_orig@PLT
-/// \endverbatim
-///
-/// Support for an architecture depends on three functions in TargetInstrInfo:
-/// getUnconditionalBranch, getTrap, and getJumpInstrTableEntryBound. AsmPrinter
-/// uses these to generate the appropriate instructions for the jump statement
-/// (an unconditional branch) and for padding to make the table have a size that
-/// is a power of two. This padding uses a trap instruction to ensure that calls
-/// to this area halt the program. The default implementations of these
-/// functions call llvm_unreachable, except for getJumpInstrTableEntryBound,
-/// which returns 0 by default.
-class JumpInstrTables : public ModulePass {
-public:
-  static char ID;
-
-  JumpInstrTables();
-  JumpInstrTables(JumpTable::JumpTableType JTT);
-  virtual ~JumpInstrTables();
-  bool runOnModule(Module &M) override;
-  const char *getPassName() const override { return "Jump-Instruction Tables"; }
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-
-  /// Creates a jump-instruction table function for the Target and adds it to
-  /// the tables.
-  Function *insertEntry(Module &M, Function *Target);
-
-  /// Checks to see if there is already a table for the given FunctionType.
-  bool hasTable(FunctionType *FunTy);
-
-  /// Maps the function into a subset of function types, depending on the
-  /// jump-instruction table style selected from JumpTableTypes in
-  /// JumpInstrTables.cpp. The choice of mapping determines the number of
-  /// jump-instruction tables generated by this pass. E.g., the simplest mapping
-  /// converts every function type into void f(); so, all functions end up in a
-  /// single table.
-  static FunctionType *transformType(JumpTable::JumpTableType JTT,
-                                     FunctionType *FunTy);
-private:
-  /// The metadata used while a jump table is being built
-  struct TableMeta {
-    /// The number of this table
-    unsigned TableNum;
-
-    /// The current number of jump entries in the table.
-    unsigned Count;
-  };
-
-  typedef DenseMap<FunctionType *, struct TableMeta> JumpMap;
-
-  /// The current state of functions and jump entries in the table(s).
-  JumpMap Metadata;
-
-  /// The ImmutablePass that stores information about the generated tables.
-  JumpInstrTableInfo *JITI;
-
-  /// The total number of tables.
-  unsigned TableCount;
-
-  /// The type of tables to build.
-  JumpTable::JumpTableType JTType;
-};
-
-/// Creates a JumpInstrTables pass for the given type of jump table.
-ModulePass *createJumpInstrTablesPass(JumpTable::JumpTableType JTT);
-}
-
-#endif /* LLVM_CODEGEN_JUMPINSTRTABLES_H */
diff --git a/include/llvm/CodeGen/LatencyPriorityQueue.h b/include/llvm/CodeGen/LatencyPriorityQueue.h
index cf601ae5384c3..f347f66e0981c 100644
--- a/include/llvm/CodeGen/LatencyPriorityQueue.h
+++ b/include/llvm/CodeGen/LatencyPriorityQueue.h
@@ -83,8 +83,6 @@ namespace llvm {
 
     void remove(SUnit *SU) override;
 
-    void dump(ScheduleDAG* DAG) const override;
-
     // scheduledNode - As nodes are scheduled, we look to see if there are any
     // successor nodes that have a single unscheduled predecessor.  If so, that
     // single predecessor has a higher priority, since scheduling it will make
diff --git a/include/llvm/CodeGen/LexicalScopes.h b/include/llvm/CodeGen/LexicalScopes.h
index 11a360a491a73..7d7e48af2a0ff 100644
--- a/include/llvm/CodeGen/LexicalScopes.h
+++ b/include/llvm/CodeGen/LexicalScopes.h
@@ -23,7 +23,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/IR/DebugLoc.h"
-#include "llvm/IR/Metadata.h"
+#include "llvm/IR/DebugInfoMetadata.h"
 #include "llvm/IR/ValueHandle.h"
 #include <unordered_map>
 #include <utility>
@@ -45,7 +45,8 @@ typedef std::pair<const MachineInstr *, const MachineInstr *> InsnRange;
 class LexicalScope {
 
 public:
-  LexicalScope(LexicalScope *P, const MDNode *D, const MDNode *I, bool A)
+  LexicalScope(LexicalScope *P, const DILocalScope *D, const DILocation *I,
+               bool A)
       : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A),
         LastInsn(nullptr), FirstInsn(nullptr), DFSIn(0), DFSOut(0) {
     assert((!D || D->isResolved()) && "Expected resolved node");
@@ -57,8 +58,8 @@ public:
   // Accessors.
   LexicalScope *getParent() const { return Parent; }
   const MDNode *getDesc() const { return Desc; }
-  const MDNode *getInlinedAt() const { return InlinedAtLocation; }
-  const MDNode *getScopeNode() const { return Desc; }
+  const DILocation *getInlinedAt() const { return InlinedAtLocation; }
+  const DILocalScope *getScopeNode() const { return Desc; }
   bool isAbstractScope() const { return AbstractScope; }
   SmallVectorImpl<LexicalScope *> &getChildren() { return Children; }
   SmallVectorImpl<InsnRange> &getRanges() { return Ranges; }
@@ -118,8 +119,8 @@ public:
 
 private:
   LexicalScope *Parent;                        // Parent to this scope.
-  const MDNode *Desc;                          // Debug info descriptor.
-  const MDNode *InlinedAtLocation;             // Location at which this
+  const DILocalScope *Desc;                    // Debug info descriptor.
+  const DILocation *InlinedAtLocation;         // Location at which this
                                                // scope is inlined.
   bool AbstractScope;                          // Abstract Scope
   SmallVector<LexicalScope *, 4> Children;     // Scopes defined in scope.
@@ -158,16 +159,16 @@ public:
   /// getMachineBasicBlocks - Populate given set using machine basic blocks
   /// which have machine instructions that belong to lexical scope identified by
   /// DebugLoc.
-  void getMachineBasicBlocks(DebugLoc DL,
+  void getMachineBasicBlocks(const DILocation *DL,
                              SmallPtrSetImpl<const MachineBasicBlock *> &MBBs);
 
   /// dominates - Return true if DebugLoc's lexical scope dominates at least one
   /// machine instruction's lexical scope in a given machine basic block.
-  bool dominates(DebugLoc DL, MachineBasicBlock *MBB);
+  bool dominates(const DILocation *DL, MachineBasicBlock *MBB);
 
   /// findLexicalScope - Find lexical scope, either regular or inlined, for the
   /// given DebugLoc. Return NULL if not found.
-  LexicalScope *findLexicalScope(DebugLoc DL);
+  LexicalScope *findLexicalScope(const DILocation *DL);
 
   /// getAbstractScopesList - Return a reference to list of abstract scopes.
   ArrayRef<LexicalScope *> getAbstractScopesList() const {
@@ -175,17 +176,19 @@ public:
   }
 
   /// findAbstractScope - Find an abstract scope or return null.
-  LexicalScope *findAbstractScope(const MDNode *N) {
+  LexicalScope *findAbstractScope(const DILocalScope *N) {
     auto I = AbstractScopeMap.find(N);
     return I != AbstractScopeMap.end() ? &I->second : nullptr;
   }
 
-  /// findInlinedScope - Find an inlined scope for the given DebugLoc or return
-  /// NULL.
-  LexicalScope *findInlinedScope(DebugLoc DL);
+  /// findInlinedScope - Find an inlined scope for the given scope/inlined-at.
+  LexicalScope *findInlinedScope(const DILocalScope *N, const DILocation *IA) {
+    auto I = InlinedLexicalScopeMap.find(std::make_pair(N, IA));
+    return I != InlinedLexicalScopeMap.end() ? &I->second : nullptr;
+  }
 
   /// findLexicalScope - Find regular lexical scope or return null.
-  LexicalScope *findLexicalScope(const MDNode *N) {
+  LexicalScope *findLexicalScope(const DILocalScope *N) {
     auto I = LexicalScopeMap.find(N);
     return I != LexicalScopeMap.end() ? &I->second : nullptr;
   }
@@ -194,18 +197,24 @@ public:
   void dump();
 
   /// getOrCreateAbstractScope - Find or create an abstract lexical scope.
-  LexicalScope *getOrCreateAbstractScope(const MDNode *N);
+  LexicalScope *getOrCreateAbstractScope(const DILocalScope *Scope);
 
 private:
-  /// getOrCreateLexicalScope - Find lexical scope for the given DebugLoc. If
+  /// getOrCreateLexicalScope - Find lexical scope for the given Scope/IA. If
   /// not available then create new lexical scope.
-  LexicalScope *getOrCreateLexicalScope(DebugLoc DL);
+  LexicalScope *getOrCreateLexicalScope(const DILocalScope *Scope,
+                                        const DILocation *IA = nullptr);
+  LexicalScope *getOrCreateLexicalScope(const DILocation *DL) {
+    return DL ? getOrCreateLexicalScope(DL->getScope(), DL->getInlinedAt())
+              : nullptr;
+  }
 
   /// getOrCreateRegularScope - Find or create a regular lexical scope.
-  LexicalScope *getOrCreateRegularScope(MDNode *Scope);
+  LexicalScope *getOrCreateRegularScope(const DILocalScope *Scope);
 
   /// getOrCreateInlinedScope - Find or create an inlined lexical scope.
-  LexicalScope *getOrCreateInlinedScope(MDNode *Scope, MDNode *InlinedAt);
+  LexicalScope *getOrCreateInlinedScope(const DILocalScope *Scope,
+                                        const DILocation *InlinedAt);
 
   /// extractLexicalScopes - Extract instruction ranges for each lexical scopes
   /// for the given machine function.
@@ -221,17 +230,18 @@ private:
 
   /// LexicalScopeMap - Tracks the scopes in the current function.
   // Use an unordered_map to ensure value pointer validity over insertion.
-  std::unordered_map<const MDNode *, LexicalScope> LexicalScopeMap;
+  std::unordered_map<const DILocalScope *, LexicalScope> LexicalScopeMap;
 
   /// InlinedLexicalScopeMap - Tracks inlined function scopes in current
   /// function.
-  std::unordered_map<std::pair<const MDNode *, const MDNode *>, LexicalScope,
-                     pair_hash<const MDNode *, const MDNode *>>
-  InlinedLexicalScopeMap;
+  std::unordered_map<std::pair<const DILocalScope *, const DILocation *>,
+                     LexicalScope,
+                     pair_hash<const DILocalScope *, const DILocation *>>
+      InlinedLexicalScopeMap;
 
   /// AbstractScopeMap - These scopes are  not included LexicalScopeMap.
   // Use an unordered_map to ensure value pointer validity over insertion.
-  std::unordered_map<const MDNode *, LexicalScope> AbstractScopeMap;
+  std::unordered_map<const DILocalScope *, LexicalScope> AbstractScopeMap;
 
   /// AbstractScopesList - Tracks abstract scopes constructed while processing
   /// a function.
diff --git a/include/llvm/CodeGen/LinkAllCodegenComponents.h b/include/llvm/CodeGen/LinkAllCodegenComponents.h
index e7ccbfa617e52..fee131e4a3c6b 100644
--- a/include/llvm/CodeGen/LinkAllCodegenComponents.h
+++ b/include/llvm/CodeGen/LinkAllCodegenComponents.h
@@ -36,6 +36,7 @@ namespace {
       (void) llvm::createGreedyRegisterAllocator();
       (void) llvm::createDefaultPBQPRegisterAllocator();
 
+      llvm::linkCoreCLRGC();
       llvm::linkOcamlGC();
       llvm::linkErlangGC();
       llvm::linkShadowStackGC();
diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h
index ce9845ee1673a..9b8b91c9b80e2 100644
--- a/include/llvm/CodeGen/LiveInterval.h
+++ b/include/llvm/CodeGen/LiveInterval.h
@@ -27,6 +27,7 @@
 #include "llvm/Support/Allocator.h"
 #include <cassert>
 #include <climits>
+#include <set>
 
 namespace llvm {
   class CoalescerPair;
@@ -194,6 +195,12 @@ namespace llvm {
     Segments segments;   // the liveness segments
     VNInfoList valnos;   // value#'s
 
+    // The segment set is used temporarily to accelerate initial computation
+    // of live ranges of physical registers in computeRegUnitRange.
+    // After that the set is flushed to the segment vector and deleted.
+    typedef std::set<Segment> SegmentSet;
+    std::unique_ptr<SegmentSet> segmentSet;
+
     typedef Segments::iterator iterator;
     iterator begin() { return segments.begin(); }
     iterator end()   { return segments.end(); }
@@ -211,12 +218,16 @@ namespace llvm {
     const_vni_iterator vni_end() const   { return valnos.end(); }
 
     /// Constructs a new LiveRange object.
-    LiveRange() {
-    }
+    LiveRange(bool UseSegmentSet = false)
+        : segmentSet(UseSegmentSet ? llvm::make_unique<SegmentSet>()
+                                   : nullptr) {}
 
     /// Constructs a new LiveRange object by copying segments and valnos from
     /// another LiveRange.
     LiveRange(const LiveRange &Other, BumpPtrAllocator &Allocator) {
+      assert(Other.segmentSet == nullptr &&
+             "Copying of LiveRanges with active SegmentSets is not supported");
+
       // Duplicate valnos.
       for (const VNInfo *VNI : Other.valnos) {
         createValueCopy(VNI, Allocator);
@@ -437,14 +448,12 @@ namespace llvm {
     /// Add the specified Segment to this range, merging segments as
     /// appropriate.  This returns an iterator to the inserted segment (which
     /// may have grown since it was inserted).
-    iterator addSegment(Segment S) {
-      return addSegmentFrom(S, segments.begin());
-    }
+    iterator addSegment(Segment S);
 
-    /// extendInBlock - If this range is live before Kill in the basic block
-    /// that starts at StartIdx, extend it to be live up to Kill, and return
-    /// the value. If there is no segment before Kill, return NULL.
-    VNInfo *extendInBlock(SlotIndex StartIdx, SlotIndex Kill);
+    /// If this range is live before @p Use in the basic block that starts at
+    /// @p StartIdx, extend it to be live up to @p Use, and return the value. If
+    /// there is no segment before @p Use, return nullptr.
+    VNInfo *extendInBlock(SlotIndex StartIdx, SlotIndex Use);
 
     /// join - Join two live ranges (this, and other) together.  This applies
     /// mappings to the value numbers in the LHS/RHS ranges as specified.  If
@@ -540,6 +549,12 @@ namespace llvm {
       return thisIndex < otherIndex;
     }
 
+    /// Flush segment set into the regular segment vector.
+    /// The method is to be called after the live range
+    /// has been created, if use of the segment set was
+    /// activated in the constructor of the live range.
+    void flushSegmentSet();
+
     void print(raw_ostream &OS) const;
     void dump() const;
 
@@ -557,10 +572,8 @@ namespace llvm {
     void append(const LiveRange::Segment S);
 
   private:
-
-    iterator addSegmentFrom(Segment S, iterator From);
-    void extendSegmentEndTo(iterator I, SlotIndex NewEnd);
-    iterator extendSegmentStartTo(iterator I, SlotIndex NewStr);
+    friend class LiveRangeUpdater;
+    void addSegmentToSet(Segment S);
     void markValNoForDeletion(VNInfo *V);
 
   };
@@ -606,6 +619,10 @@ namespace llvm {
     LiveInterval(unsigned Reg, float Weight)
       : SubRanges(nullptr), reg(Reg), weight(Weight) {}
 
+    ~LiveInterval() {
+      clearSubRanges();
+    }
+
     template<typename T>
     class SingleLinkedListIterator {
       T *P;
@@ -681,9 +698,7 @@ namespace llvm {
     }
 
     /// Removes all subregister liveness information.
-    void clearSubRanges() {
-      SubRanges = nullptr;
-    }
+    void clearSubRanges();
 
     /// Removes all subranges without any segments (subranges without segments
     /// are not considered valid and should only exist temporarily).
@@ -726,13 +741,14 @@ namespace llvm {
 #endif
 
   private:
-    LiveInterval& operator=(const LiveInterval& rhs) LLVM_DELETED_FUNCTION;
-
     /// Appends @p Range to SubRanges list.
     void appendSubRange(SubRange *Range) {
       Range->Next = SubRanges;
       SubRanges = Range;
     }
+
+    /// Free memory held by SubRange.
+    void freeSubRange(SubRange *S);
   };
 
   inline raw_ostream &operator<<(raw_ostream &OS, const LiveInterval &LI) {
diff --git a/include/llvm/CodeGen/LiveIntervalAnalysis.h b/include/llvm/CodeGen/LiveIntervalAnalysis.h
index d8c921fce3137..9673f80e08566 100644
--- a/include/llvm/CodeGen/LiveIntervalAnalysis.h
+++ b/include/llvm/CodeGen/LiveIntervalAnalysis.h
@@ -27,12 +27,15 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/SlotIndexes.h"
 #include "llvm/Support/Allocator.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include <cmath>
 #include <iterator>
 
 namespace llvm {
 
+extern cl::opt<bool> UseSegmentSetForPhysRegs;
+
   class AliasAnalysis;
   class BitVector;
   class BlockFrequency;
@@ -97,7 +100,7 @@ namespace llvm {
   public:
     static char ID; // Pass identification, replacement for typeid
     LiveIntervals();
-    virtual ~LiveIntervals();
+    ~LiveIntervals() override;
 
     // Calculate the spill weight to assign to a single instruction.
     static float getSpillWeight(bool isDef, bool isUse,
@@ -181,12 +184,6 @@ namespace llvm {
     void pruneValue(LiveRange &LR, SlotIndex Kill,
                     SmallVectorImpl<SlotIndex> *EndPoints);
 
-    /// Subregister aware variant of pruneValue(LiveRange &LR, SlotIndex Kill,
-    /// SmallVectorImpl<SlotIndex> &EndPoints). Prunes the value in the main
-    /// range and all sub ranges.
-    void pruneValue(LiveInterval &LI, SlotIndex Kill,
-                    SmallVectorImpl<SlotIndex> *EndPoints);
-
     SlotIndexes *getSlotIndexes() const {
       return Indexes;
     }
@@ -383,7 +380,8 @@ namespace llvm {
       LiveRange *LR = RegUnitRanges[Unit];
       if (!LR) {
         // Compute missing ranges on demand.
-        RegUnitRanges[Unit] = LR = new LiveRange();
+        // Use segment set to speed-up initial computation of the live range.
+        RegUnitRanges[Unit] = LR = new LiveRange(UseSegmentSetForPhysRegs);
         computeRegUnitRange(*LR, Unit);
       }
       return *LR;
@@ -399,6 +397,15 @@ namespace llvm {
       return RegUnitRanges[Unit];
     }
 
+    /// Remove value numbers and related live segments starting at position
+    /// @p Pos that are part of any liverange of physical register @p Reg or one
+    /// of its subregisters.
+    void removePhysRegDefAt(unsigned Reg, SlotIndex Pos);
+
+    /// Remove value number and related live segments of @p LI and its subranges
+    /// that start at position @p Pos.
+    void removeVRegDefAt(LiveInterval &LI, SlotIndex Pos);
+
   private:
     /// Compute live intervals for all virtual registers.
     void computeVirtRegs();
diff --git a/include/llvm/CodeGen/LiveIntervalUnion.h b/include/llvm/CodeGen/LiveIntervalUnion.h
index 1381c46a2750e..967f0cbac7194 100644
--- a/include/llvm/CodeGen/LiveIntervalUnion.h
+++ b/include/llvm/CodeGen/LiveIntervalUnion.h
@@ -179,8 +179,8 @@ public:
     }
 
   private:
-    Query(const Query&) LLVM_DELETED_FUNCTION;
-    void operator=(const Query&) LLVM_DELETED_FUNCTION;
+    Query(const Query&) = delete;
+    void operator=(const Query&) = delete;
   };
 
   // Array of LiveIntervalUnions.
diff --git a/include/llvm/CodeGen/LivePhysRegs.h b/include/llvm/CodeGen/LivePhysRegs.h
index 91e4ddcde1706..6ffd3eeea351f 100644
--- a/include/llvm/CodeGen/LivePhysRegs.h
+++ b/include/llvm/CodeGen/LivePhysRegs.h
@@ -44,8 +44,8 @@ class LivePhysRegs {
   const TargetRegisterInfo *TRI;
   SparseSet<unsigned> LiveRegs;
 
-  LivePhysRegs(const LivePhysRegs&) LLVM_DELETED_FUNCTION;
-  LivePhysRegs &operator=(const LivePhysRegs&) LLVM_DELETED_FUNCTION;
+  LivePhysRegs(const LivePhysRegs&) = delete;
+  LivePhysRegs &operator=(const LivePhysRegs&) = delete;
 public:
   /// \brief Constructs a new empty LivePhysRegs set.
   LivePhysRegs() : TRI(nullptr), LiveRegs() {}
@@ -57,9 +57,9 @@ public:
   }
 
   /// \brief Clear and initialize the LivePhysRegs set.
-  void init(const TargetRegisterInfo *_TRI) {
-    assert(_TRI && "Invalid TargetRegisterInfo pointer.");
-    TRI = _TRI;
+  void init(const TargetRegisterInfo *TRI) {
+    assert(TRI && "Invalid TargetRegisterInfo pointer.");
+    this->TRI = TRI;
     LiveRegs.clear();
     LiveRegs.setUniverse(TRI->getNumRegs());
   }
@@ -93,7 +93,8 @@ public:
   }
 
   /// \brief Removes physical registers clobbered by the regmask operand @p MO.
-  void removeRegsInMask(const MachineOperand &MO);
+  void removeRegsInMask(const MachineOperand &MO,
+        SmallVectorImpl<std::pair<unsigned, const MachineOperand*>> *Clobbers);
 
   /// \brief Returns true if register @p Reg is contained in the set. This also
   /// works if only the super register of @p Reg has been defined, because we
@@ -109,7 +110,11 @@ public:
   /// instruction(bundle): Remove killed-uses, add defs. This is the not
   /// recommended way, because it depends on accurate kill flags. If possible
   /// use stepBackwards() instead of this function.
-  void stepForward(const MachineInstr &MI);
+  /// The clobbers set will be the list of registers either defined or clobbered
+  /// by a regmask.  The operand will identify whether this is a regmask or
+  /// register operand.
+  void stepForward(const MachineInstr &MI,
+        SmallVectorImpl<std::pair<unsigned, const MachineOperand*>> &Clobbers);
 
   /// \brief Adds all live-in registers of basic block @p MBB.
   void addLiveIns(const MachineBasicBlock *MBB) {
diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h
index 44c3c4eaf7b17..de855f2fe7a0f 100644
--- a/include/llvm/CodeGen/LiveRangeEdit.h
+++ b/include/llvm/CodeGen/LiveRangeEdit.h
@@ -122,7 +122,7 @@ public:
     MRI.setDelegate(this);
   }
 
-  ~LiveRangeEdit() { MRI.resetDelegate(this); }
+  ~LiveRangeEdit() override { MRI.resetDelegate(this); }
 
   LiveInterval &getParent() const {
    assert(Parent && "No parent LiveInterval");
diff --git a/include/llvm/CodeGen/LiveStackAnalysis.h b/include/llvm/CodeGen/LiveStackAnalysis.h
index df683983fa53e..f495507c66ecd 100644
--- a/include/llvm/CodeGen/LiveStackAnalysis.h
+++ b/include/llvm/CodeGen/LiveStackAnalysis.h
@@ -21,6 +21,7 @@
 #include "llvm/Support/Allocator.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include <map>
+#include <unordered_map>
 
 namespace llvm {
 
@@ -33,7 +34,7 @@ namespace llvm {
 
     /// S2IMap - Stack slot indices to live interval mapping.
     ///
-    typedef std::map<int, LiveInterval> SS2IntervalMap;
+    typedef std::unordered_map<int, LiveInterval> SS2IntervalMap;
     SS2IntervalMap S2IMap;
 
     /// S2RCMap - Stack slot indices to register class mapping.
diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h
index 1440b967aea21..357aef0ee60da 100644
--- a/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/include/llvm/CodeGen/MachineBasicBlock.h
@@ -315,9 +315,18 @@ public:
 
   // LiveIn management methods.
 
-  /// addLiveIn - Add the specified register as a live in.  Note that it
-  /// is an error to add the same register to the same set more than once.
-  void addLiveIn(unsigned Reg)  { LiveIns.push_back(Reg); }
+  /// Adds the specified register as a live in. Note that it is an error to add
+  /// the same register to the same set more than once unless the intention is
+  /// to call sortUniqueLiveIns after all registers are added.
+  void addLiveIn(unsigned Reg) { LiveIns.push_back(Reg); }
+
+  /// Sorts and uniques the LiveIns vector. It can be significantly faster to do
+  /// this than repeatedly calling isLiveIn before calling addLiveIn for every
+  /// LiveIn insertion.
+  void sortUniqueLiveIns() {
+    std::sort(LiveIns.begin(), LiveIns.end());
+    LiveIns.erase(std::unique(LiveIns.begin(), LiveIns.end()), LiveIns.end());
+  }
 
   /// Add PhysReg as live in to this block, and ensure that there is a copy of
   /// PhysReg to a virtual register of class RC. Return the virtual register
@@ -624,17 +633,18 @@ public:
                          ///< neighborhood.
   };
 
-  /// computeRegisterLiveness - Return whether (physical) register \c Reg
-  /// has been <def>ined and not <kill>ed as of just before \c MI.
+  /// Return whether (physical) register \p Reg has been <def>ined and not
+  /// <kill>ed as of just before \p Before.
   ///
-  /// Search is localised to a neighborhood of
-  /// \c Neighborhood instructions before (searching for defs or kills) and
-  /// Neighborhood instructions after (searching just for defs) MI.
+  /// Search is localised to a neighborhood of \p Neighborhood instructions
+  /// before (searching for defs or kills) and \p Neighborhood instructions
+  /// after (searching just for defs) \p Before.
   ///
-  /// \c Reg must be a physical register.
+  /// \p Reg must be a physical register.
   LivenessQueryResult computeRegisterLiveness(const TargetRegisterInfo *TRI,
-                                              unsigned Reg, MachineInstr *MI,
-                                              unsigned Neighborhood=10);
+                                              unsigned Reg,
+                                              const_iterator Before,
+                                              unsigned Neighborhood=10) const;
 
   // Debugging methods.
   void dump() const;
diff --git a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index 1aef689eb7a44..feb394e7a69e0 100644
--- a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -35,7 +35,7 @@ public:
 
   MachineBlockFrequencyInfo();
 
-  ~MachineBlockFrequencyInfo();
+  ~MachineBlockFrequencyInfo() override;
 
   void getAnalysisUsage(AnalysisUsage &AU) const override;
 
diff --git a/include/llvm/CodeGen/MachineDominanceFrontier.h b/include/llvm/CodeGen/MachineDominanceFrontier.h
index e099e716c63da..4131194a0c0fc 100644
--- a/include/llvm/CodeGen/MachineDominanceFrontier.h
+++ b/include/llvm/CodeGen/MachineDominanceFrontier.h
@@ -26,8 +26,8 @@ public:
   typedef DominanceFrontierBase<MachineBasicBlock>::iterator iterator;
   typedef DominanceFrontierBase<MachineBasicBlock>::const_iterator const_iterator;
 
-  void operator=(const MachineDominanceFrontier &) LLVM_DELETED_FUNCTION;
-  MachineDominanceFrontier(const MachineDominanceFrontier &) LLVM_DELETED_FUNCTION;
+  void operator=(const MachineDominanceFrontier &) = delete;
+  MachineDominanceFrontier(const MachineDominanceFrontier &) = delete;
 
   static char ID;
 
diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h
index a6980a6daeacd..4428fa618fb01 100644
--- a/include/llvm/CodeGen/MachineDominators.h
+++ b/include/llvm/CodeGen/MachineDominators.h
@@ -45,9 +45,6 @@ class MachineDominatorTree : public MachineFunctionPass {
     MachineBasicBlock *FromBB;
     MachineBasicBlock *ToBB;
     MachineBasicBlock *NewBB;
-    CriticalEdge(MachineBasicBlock *FromBB, MachineBasicBlock *ToBB,
-                 MachineBasicBlock *NewBB)
-        : FromBB(FromBB), ToBB(ToBB), NewBB(NewBB) {}
   };
 
   /// \brief Pile up all the critical edges to be split.
@@ -67,74 +64,7 @@ class MachineDominatorTree : public MachineFunctionPass {
   /// the fast query path of DT as much as possible.
   ///
   /// \post CriticalEdgesToSplit.empty().
-  void applySplitCriticalEdges() const {
-    // Bail out early if there is nothing to do.
-    if (CriticalEdgesToSplit.empty())
-      return;
-
-    // For each element in CriticalEdgesToSplit, remember whether or
-    // not element is the new immediate domminator of its successor.
-    // The mapping is done by index, i.e., the information for the ith
-    // element of CriticalEdgesToSplit is the ith element of IsNewIDom.
-    SmallVector<bool, 32> IsNewIDom;
-    IsNewIDom.resize(CriticalEdgesToSplit.size());
-    size_t Idx = 0;
-
-    // Collect all the dominance properties info, before invalidating
-    // the underlying DT.
-    for (CriticalEdge &Edge : CriticalEdgesToSplit) {
-      // Update dominator information.
-      MachineBasicBlock *Succ = Edge.ToBB;
-      MachineDomTreeNode *SucccDTNode = DT->getNode(Succ);
-
-      IsNewIDom[Idx] = true;
-      for (MachineBasicBlock *PredBB : Succ->predecessors()) {
-        if (PredBB == Edge.NewBB)
-          continue;
-        // If we are in this situation:
-        // FromBB1        FromBB2
-        //    +              +
-        //   + +            + +
-        //  +   +          +   +
-        // ...  Split1  Split2 ...
-        //           +   +
-        //            + +
-        //             +
-        //            Succ
-        // Instead of checking the domiance property with Split2, we
-        // check it with FromBB2 since Split2 is still unknown of the
-        // underlying DT structure.
-        if (NewBBs.count(PredBB)) {
-          assert(PredBB->pred_size() == 1 && "A basic block resulting from a "
-                                             "critical edge split has more "
-                                             "than one predecessor!");
-          PredBB = *PredBB->pred_begin();
-        }
-        if (!DT->dominates(SucccDTNode, DT->getNode(PredBB))) {
-          IsNewIDom[Idx] = false;
-          break;
-        }
-      }
-      ++Idx;
-    }
-
-    // Now, update DT with the collected dominance properties info.
-    Idx = 0;
-    for (CriticalEdge &Edge : CriticalEdgesToSplit) {
-      // We know FromBB dominates NewBB.
-      MachineDomTreeNode *NewDTNode = DT->addNewBlock(Edge.NewBB, Edge.FromBB);
-      MachineDomTreeNode *SucccDTNode = DT->getNode(Edge.ToBB);
-
-      // If all the other predecessors of "Succ" are dominated by "Succ" itself
-      // then the new block is the new immediate dominator of "Succ". Otherwise,
-      // the new block doesn't dominate anything.
-      if (IsNewIDom[Idx])
-        DT->changeImmediateDominator(SucccDTNode, NewDTNode);
-      ++Idx;
-    }
-    NewBBs.clear();
-    CriticalEdgesToSplit.clear();
-  }
+  void applySplitCriticalEdges() const;
 
 public:
   static char ID; // Pass ID, replacement for typeid
@@ -142,7 +72,7 @@ public:
 
   MachineDominatorTree();
 
-  ~MachineDominatorTree();
+  ~MachineDominatorTree() override;
 
   DominatorTreeBase<MachineBasicBlock> &getBase() {
     applySplitCriticalEdges();
@@ -307,7 +237,7 @@ public:
     (void)Inserted;
     assert(Inserted &&
            "A basic block inserted via edge splitting cannot appear twice");
-    CriticalEdgesToSplit.push_back(CriticalEdge(FromBB, ToBB, NewBB));
+    CriticalEdgesToSplit.push_back({FromBB, ToBB, NewBB});
   }
 };
 
diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h
index 667736021f927..40f3b4944cc1e 100644
--- a/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/include/llvm/CodeGen/MachineFrameInfo.h
@@ -246,6 +246,21 @@ class MachineFrameInfo {
   /// True if this is a varargs function that contains a musttail call.
   bool HasMustTailInVarArgFunc;
 
+  /// True if this function contains a tail call. If so immutable objects like
+  /// function arguments are no longer so. A tail call *can* override fixed
+  /// stack objects like arguments so we can't treat them as immutable.
+  bool HasTailCall;
+
+  /// Not null, if shrink-wrapping found a better place for the prologue.
+  MachineBasicBlock *Save;
+  /// Not null, if shrink-wrapping found a better place for the epilogue.
+  MachineBasicBlock *Restore;
+
+  /// Check if it exists a path from \p MBB leading to the basic
+  /// block with a SavePoint (a.k.a. prologue).
+  bool isBeforeSavePoint(const MachineFunction &MF,
+                         const MachineBasicBlock &MBB) const;
+
 public:
   explicit MachineFrameInfo(unsigned StackAlign, bool isStackRealign,
                             bool RealignOpt)
@@ -269,6 +284,9 @@ public:
     HasInlineAsmWithSPAdjust = false;
     HasVAStart = false;
     HasMustTailInVarArgFunc = false;
+    Save = nullptr;
+    Restore = nullptr;
+    HasTailCall = false;
   }
 
   /// hasStackObjects - Return true if there are any stack objects in this
@@ -496,6 +514,10 @@ public:
   bool hasMustTailInVarArgFunc() const { return HasMustTailInVarArgFunc; }
   void setHasMustTailInVarArgFunc(bool B) { HasMustTailInVarArgFunc = B; }
 
+  /// Returns true if the function contains a tail call.
+  bool hasTailCall() const { return HasTailCall; }
+  void setHasTailCall() { HasTailCall = true; }
+
   /// getMaxCallFrameSize - Return the maximum size of a call frame that must be
   /// allocated for an outgoing function call.  This is only available if
   /// CallFrameSetup/Destroy pseudo instructions are used by the target, and
@@ -516,10 +538,6 @@ public:
   /// on the stack.  Returns an index with a negative value.
   int CreateFixedSpillStackObject(uint64_t Size, int64_t SPOffset);
 
-  /// Allocates memory at a fixed, target-specific offset from the frame
-  /// pointer. Marks the function as having its frame address taken.
-  int CreateFrameAllocation(uint64_t Size);
-
   /// isFixedObjectIndex - Returns true if the specified index corresponds to a
   /// fixed stack object.
   bool isFixedObjectIndex(int ObjectIdx) const {
@@ -537,6 +555,9 @@ public:
   /// isImmutableObjectIndex - Returns true if the specified index corresponds
   /// to an immutable object.
   bool isImmutableObjectIndex(int ObjectIdx) const {
+    // Tail calling functions can clobber their function arguments.
+    if (HasTailCall)
+      return false;
     assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
            "Invalid Object Idx!");
     return Objects[ObjectIdx+NumFixedObjects].isImmutable;
@@ -601,6 +622,11 @@ public:
 
   void setCalleeSavedInfoValid(bool v) { CSIValid = v; }
 
+  MachineBasicBlock *getSavePoint() const { return Save; }
+  void setSavePoint(MachineBasicBlock *NewSave) { Save = NewSave; }
+  MachineBasicBlock *getRestorePoint() const { return Restore; }
+  void setRestorePoint(MachineBasicBlock *NewRestore) { Restore = NewRestore; }
+
   /// getPristineRegs - Return a set of physical registers that are pristine on
   /// entry to the MBB.
   ///
diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h
index 4e9ff9ebb4fea..94610cabf566d 100644
--- a/include/llvm/CodeGen/MachineFunction.h
+++ b/include/llvm/CodeGen/MachineFunction.h
@@ -145,8 +145,8 @@ class MachineFunction {
   /// True if the function includes any inline assembly.
   bool HasInlineAsm;
 
-  MachineFunction(const MachineFunction &) LLVM_DELETED_FUNCTION;
-  void operator=(const MachineFunction&) LLVM_DELETED_FUNCTION;
+  MachineFunction(const MachineFunction &) = delete;
+  void operator=(const MachineFunction&) = delete;
 public:
   MachineFunction(const Function *Fn, const TargetMachine &TM,
                   unsigned FunctionNum, MachineModuleInfo &MMI);
@@ -176,6 +176,13 @@ public:
   const TargetSubtargetInfo &getSubtarget() const { return *STI; }
   void setSubtarget(const TargetSubtargetInfo *ST) { STI = ST; }
 
+  /// getSubtarget - This method returns a pointer to the specified type of
+  /// TargetSubtargetInfo.  In debug builds, it verifies that the object being
+  /// returned is of the correct type.
+  template<typename STC> const STC &getSubtarget() const {
+    return *static_cast<const STC *>(STI);
+  }
+
   /// getRegInfo - Return information about the registers currently in use.
   ///
   MachineRegisterInfo &getRegInfo() { return *RegInfo; }
diff --git a/include/llvm/CodeGen/MachineFunctionAnalysis.h b/include/llvm/CodeGen/MachineFunctionAnalysis.h
index 36f1c6627096c..023eeb1b4d0ea 100644
--- a/include/llvm/CodeGen/MachineFunctionAnalysis.h
+++ b/include/llvm/CodeGen/MachineFunctionAnalysis.h
@@ -31,7 +31,7 @@ private:
 public:
   static char ID;
   explicit MachineFunctionAnalysis(const TargetMachine &tm);
-  ~MachineFunctionAnalysis();
+  ~MachineFunctionAnalysis() override;
 
   MachineFunction &getMF() const { return *MF; }
 
diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h
index bcf1f5caaa8c6..e57257c76bc70 100644
--- a/include/llvm/CodeGen/MachineInstr.h
+++ b/include/llvm/CodeGen/MachineInstr.h
@@ -42,7 +42,7 @@ class MachineFunction;
 class MachineMemOperand;
 
 //===----------------------------------------------------------------------===//
-/// MachineInstr - Representation of each machine instruction.
+/// Representation of each machine instruction.
 ///
 /// This class isn't a POD type, but it must have a trivial destructor. When a
 /// MachineFunction is deleted, all the contained MachineInstrs are deallocated
@@ -93,25 +93,25 @@ private:
 
   DebugLoc debugLoc;                    // Source line information.
 
-  MachineInstr(const MachineInstr&) LLVM_DELETED_FUNCTION;
-  void operator=(const MachineInstr&) LLVM_DELETED_FUNCTION;
+  MachineInstr(const MachineInstr&) = delete;
+  void operator=(const MachineInstr&) = delete;
   // Use MachineFunction::DeleteMachineInstr() instead.
-  ~MachineInstr() LLVM_DELETED_FUNCTION;
+  ~MachineInstr() = delete;
 
   // Intrusive list support
   friend struct ilist_traits<MachineInstr>;
   friend struct ilist_traits<MachineBasicBlock>;
   void setParent(MachineBasicBlock *P) { Parent = P; }
 
-  /// MachineInstr ctor - This constructor creates a copy of the given
+  /// This constructor creates a copy of the given
   /// MachineInstr in the given MachineFunction.
   MachineInstr(MachineFunction &, const MachineInstr &);
 
-  /// MachineInstr ctor - This constructor create a MachineInstr and add the
-  /// implicit operands.  It reserves space for number of operands specified by
+  /// This constructor create a MachineInstr and add the implicit operands.
+  /// It reserves space for number of operands specified by
   /// MCInstrDesc.  An explicit DebugLoc is supplied.
-  MachineInstr(MachineFunction&, const MCInstrDesc &MCID,
-               const DebugLoc dl, bool NoImp = false);
+  MachineInstr(MachineFunction &, const MCInstrDesc &MCID, DebugLoc dl,
+               bool NoImp = false);
 
   // MachineInstrs are pool-allocated and owned by MachineFunction.
   friend class MachineFunction;
@@ -120,43 +120,38 @@ public:
   const MachineBasicBlock* getParent() const { return Parent; }
   MachineBasicBlock* getParent() { return Parent; }
 
-  /// getAsmPrinterFlags - Return the asm printer flags bitvector.
-  ///
+  /// Return the asm printer flags bitvector.
   uint8_t getAsmPrinterFlags() const { return AsmPrinterFlags; }
 
-  /// clearAsmPrinterFlags - clear the AsmPrinter bitvector
-  ///
+  /// Clear the AsmPrinter bitvector.
   void clearAsmPrinterFlags() { AsmPrinterFlags = 0; }
 
-  /// getAsmPrinterFlag - Return whether an AsmPrinter flag is set.
-  ///
+  /// Return whether an AsmPrinter flag is set.
   bool getAsmPrinterFlag(CommentFlag Flag) const {
     return AsmPrinterFlags & Flag;
   }
 
-  /// setAsmPrinterFlag - Set a flag for the AsmPrinter.
-  ///
+  /// Set a flag for the AsmPrinter.
   void setAsmPrinterFlag(CommentFlag Flag) {
     AsmPrinterFlags |= (uint8_t)Flag;
   }
 
-  /// clearAsmPrinterFlag - clear specific AsmPrinter flags
-  ///
+  /// Clear specific AsmPrinter flags.
   void clearAsmPrinterFlag(CommentFlag Flag) {
     AsmPrinterFlags &= ~Flag;
   }
 
-  /// getFlags - Return the MI flags bitvector.
+  /// Return the MI flags bitvector.
   uint8_t getFlags() const {
     return Flags;
   }
 
-  /// getFlag - Return whether an MI flag is set.
+  /// Return whether an MI flag is set.
   bool getFlag(MIFlag Flag) const {
     return Flags & Flag;
   }
 
-  /// setFlag - Set a MI flag.
+  /// Set a MI flag.
   void setFlag(MIFlag Flag) {
     Flags |= (uint8_t)Flag;
   }
@@ -172,8 +167,7 @@ public:
     Flags &= ~((uint8_t)Flag);
   }
 
-  /// isInsideBundle - Return true if MI is in a bundle (but not the first MI
-  /// in a bundle).
+  /// Return true if MI is in a bundle (but not the first MI in a bundle).
   ///
   /// A bundle looks like this before it's finalized:
   ///   ----------------
@@ -212,7 +206,7 @@ public:
     return getFlag(BundledPred);
   }
 
-  /// isBundled - Return true if this instruction part of a bundle. This is true
+  /// Return true if this instruction part of a bundle. This is true
   /// if either itself or its following instruction is marked "InsideBundle".
   bool isBundled() const {
     return isBundledWithPred() || isBundledWithSucc();
@@ -240,30 +234,25 @@ public:
   /// Break bundle below this instruction.
   void unbundleFromSucc();
 
-  /// getDebugLoc - Returns the debug location id of this MachineInstr.
-  ///
-  DebugLoc getDebugLoc() const { return debugLoc; }
+  /// Returns the debug location id of this MachineInstr.
+  const DebugLoc &getDebugLoc() const { return debugLoc; }
 
-  /// \brief Return the debug variable referenced by
+  /// Return the debug variable referenced by
   /// this DBG_VALUE instruction.
-  DIVariable getDebugVariable() const {
+  const DILocalVariable *getDebugVariable() const {
     assert(isDebugValue() && "not a DBG_VALUE");
-    DIVariable Var(getOperand(2).getMetadata());
-    assert(Var.Verify() && "not a DIVariable");
-    return Var;
+    return cast<DILocalVariable>(getOperand(2).getMetadata());
   }
 
-  /// \brief Return the complex address expression referenced by
+  /// Return the complex address expression referenced by
   /// this DBG_VALUE instruction.
-  DIExpression getDebugExpression() const {
+  const DIExpression *getDebugExpression() const {
     assert(isDebugValue() && "not a DBG_VALUE");
-    DIExpression Expr(getOperand(3).getMetadata());
-    assert(Expr.Verify() && "not a DIExpression");
-    return Expr;
+    return cast<DIExpression>(getOperand(3).getMetadata());
   }
 
-  /// emitError - Emit an error referring to the source location of this
-  /// instruction. This should only be used for inline assembly that is somehow
+  /// Emit an error referring to the source location of this instruction.
+  /// This should only be used for inline assembly that is somehow
   /// impossible to compile. Other errors should have been handled much
   /// earlier.
   ///
@@ -271,13 +260,11 @@ public:
   ///
   void emitError(StringRef Msg) const;
 
-  /// getDesc - Returns the target instruction descriptor of this
-  /// MachineInstr.
+  /// Returns the target instruction descriptor of this MachineInstr.
   const MCInstrDesc &getDesc() const { return *MCID; }
 
-  /// getOpcode - Returns the opcode of this MachineInstr.
-  ///
-  int getOpcode() const { return MCID->Opcode; }
+  /// Returns the opcode of this MachineInstr.
+  unsigned getOpcode() const { return MCID->Opcode; }
 
   /// Access to explicit operands of the instruction.
   ///
@@ -292,8 +279,7 @@ public:
     return Operands[i];
   }
 
-  /// getNumExplicitOperands - Returns the number of non-implicit operands.
-  ///
+  /// Returns the number of non-implicit operands.
   unsigned getNumExplicitOperands() const;
 
   /// iterator/begin/end - Iterate over all operands of a machine instruction.
@@ -357,8 +343,7 @@ public:
     return iterator_range<mmo_iterator>(memoperands_begin(), memoperands_end());
   }
 
-  /// hasOneMemOperand - Return true if this instruction has exactly one
-  /// MachineMemOperand.
+  /// Return true if this instruction has exactly one MachineMemOperand.
   bool hasOneMemOperand() const {
     return NumMemRefs == 1;
   }
@@ -372,7 +357,7 @@ public:
     AllInBundle      // Return true if all instructions in bundle have property
   };
 
-  /// hasProperty - Return true if the instruction (or in the case of a bundle,
+  /// Return true if the instruction (or in the case of a bundle,
   /// the instructions inside the bundle) has the specified property.
   /// The first argument is the property being queried.
   /// The second argument indicates whether the query should look inside
@@ -386,23 +371,22 @@ public:
     return hasPropertyInBundle(1 << MCFlag, Type);
   }
 
-  /// isVariadic - Return true if this instruction can have a variable number of
-  /// operands.  In this case, the variable operands will be after the normal
+  /// Return true if this instruction can have a variable number of operands.
+  /// In this case, the variable operands will be after the normal
   /// operands but before the implicit definitions and uses (if any are
   /// present).
   bool isVariadic(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Variadic, Type);
   }
 
-  /// hasOptionalDef - Set if this instruction has an optional definition, e.g.
+  /// Set if this instruction has an optional definition, e.g.
   /// ARM instructions which can set condition code if 's' bit is set.
   bool hasOptionalDef(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::HasOptionalDef, Type);
   }
 
-  /// isPseudo - Return true if this is a pseudo instruction that doesn't
+  /// Return true if this is a pseudo instruction that doesn't
   /// correspond to a real machine instruction.
-  ///
   bool isPseudo(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Pseudo, Type);
   }
@@ -415,16 +399,15 @@ public:
     return hasProperty(MCID::Call, Type);
   }
 
-  /// isBarrier - Returns true if the specified instruction stops control flow
+  /// Returns true if the specified instruction stops control flow
   /// from executing the instruction immediately following it.  Examples include
   /// unconditional branches and return instructions.
   bool isBarrier(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::Barrier, Type);
   }
 
-  /// isTerminator - Returns true if this instruction part of the terminator for
-  /// a basic block.  Typically this is things like return and branch
-  /// instructions.
+  /// Returns true if this instruction part of the terminator for a basic block.
+  /// Typically this is things like return and branch instructions.
   ///
   /// Various passes use this to insert code into the bottom of a basic block,
   /// but before control flow occurs.
@@ -432,21 +415,21 @@ public:
     return hasProperty(MCID::Terminator, Type);
   }
 
-  /// isBranch - Returns true if this is a conditional, unconditional, or
-  /// indirect branch.  Predicates below can be used to discriminate between
+  /// Returns true if this is a conditional, unconditional, or indirect branch.
+  /// Predicates below can be used to discriminate between
   /// these cases, and the TargetInstrInfo::AnalyzeBranch method can be used to
   /// get more information.
   bool isBranch(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::Branch, Type);
   }
 
-  /// isIndirectBranch - Return true if this is an indirect branch, such as a
+  /// Return true if this is an indirect branch, such as a
   /// branch through a register.
   bool isIndirectBranch(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::IndirectBranch, Type);
   }
 
-  /// isConditionalBranch - Return true if this is a branch which may fall
+  /// Return true if this is a branch which may fall
   /// through to the next instruction or may transfer control flow to some other
   /// block.  The TargetInstrInfo::AnalyzeBranch method can be used to get more
   /// information about this branch.
@@ -454,7 +437,7 @@ public:
     return isBranch(Type) & !isBarrier(Type) & !isIndirectBranch(Type);
   }
 
-  /// isUnconditionalBranch - Return true if this is a branch which always
+  /// Return true if this is a branch which always
   /// transfers control flow to some other block.  The
   /// TargetInstrInfo::AnalyzeBranch method can be used to get more information
   /// about this branch.
@@ -472,43 +455,41 @@ public:
     return hasProperty(MCID::Predicable, Type);
   }
 
-  /// isCompare - Return true if this instruction is a comparison.
+  /// Return true if this instruction is a comparison.
   bool isCompare(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Compare, Type);
   }
 
-  /// isMoveImmediate - Return true if this instruction is a move immediate
+  /// Return true if this instruction is a move immediate
   /// (including conditional moves) instruction.
   bool isMoveImmediate(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::MoveImm, Type);
   }
 
-  /// isBitcast - Return true if this instruction is a bitcast instruction.
-  ///
+  /// Return true if this instruction is a bitcast instruction.
   bool isBitcast(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Bitcast, Type);
   }
 
-  /// isSelect - Return true if this instruction is a select instruction.
-  ///
+  /// Return true if this instruction is a select instruction.
   bool isSelect(QueryType Type = IgnoreBundle) const {
     return hasProperty(MCID::Select, Type);
   }
 
-  /// isNotDuplicable - Return true if this instruction cannot be safely
-  /// duplicated.  For example, if the instruction has a unique labels attached
+  /// Return true if this instruction cannot be safely duplicated.
+  /// For example, if the instruction has a unique labels attached
   /// to it, duplicating it would cause multiple definition errors.
   bool isNotDuplicable(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::NotDuplicable, Type);
   }
 
-  /// hasDelaySlot - Returns true if the specified instruction has a delay slot
+  /// Returns true if the specified instruction has a delay slot
   /// which must be filled by the code generator.
   bool hasDelaySlot(QueryType Type = AnyInBundle) const {
     return hasProperty(MCID::DelaySlot, Type);
   }
 
-  /// canFoldAsLoad - Return true for instructions that can be folded as
+  /// Return true for instructions that can be folded as
   /// memory operands in other instructions. The most common use for this
   /// is instructions that are simple loads from memory that don't modify
   /// the loaded value in any way, but it can also be used for instructions
@@ -567,7 +548,7 @@ public:
   // Side Effect Analysis
   //===--------------------------------------------------------------------===//
 
-  /// mayLoad - Return true if this instruction could possibly read memory.
+  /// Return true if this instruction could possibly read memory.
   /// Instructions with this flag set are not necessarily simple load
   /// instructions, they may load a value and modify it, for example.
   bool mayLoad(QueryType Type = AnyInBundle) const {
@@ -579,8 +560,7 @@ public:
     return hasProperty(MCID::MayLoad, Type);
   }
 
-
-  /// mayStore - Return true if this instruction could possibly modify memory.
+  /// Return true if this instruction could possibly modify memory.
   /// Instructions with this flag set are not necessarily simple store
   /// instructions, they may store a modified value based on their operands, or
   /// may not actually modify anything, for example.
@@ -593,11 +573,16 @@ public:
     return hasProperty(MCID::MayStore, Type);
   }
 
+  /// Return true if this instruction could possibly read or modify memory.
+  bool mayLoadOrStore(QueryType Type = AnyInBundle) const {
+    return mayLoad(Type) || mayStore(Type);
+  }
+
   //===--------------------------------------------------------------------===//
   // Flags that indicate whether an instruction can be modified by a method.
   //===--------------------------------------------------------------------===//
 
-  /// isCommutable - Return true if this may be a 2- or 3-address
+  /// Return true if this may be a 2- or 3-address
   /// instruction (of the form "X = op Y, Z, ..."), which produces the same
   /// result if Y and Z are exchanged.  If this flag is set, then the
   /// TargetInstrInfo::commuteInstruction method may be used to hack on the
@@ -611,7 +596,7 @@ public:
     return hasProperty(MCID::Commutable, Type);
   }
 
-  /// isConvertibleTo3Addr - Return true if this is a 2-address instruction
+  /// Return true if this is a 2-address instruction
   /// which can be changed into a 3-address instruction if needed.  Doing this
   /// transformation can be profitable in the register allocator, because it
   /// means that the instruction can use a 2-address form if possible, but
@@ -629,7 +614,7 @@ public:
     return hasProperty(MCID::ConvertibleTo3Addr, Type);
   }
 
-  /// usesCustomInsertionHook - Return true if this instruction requires
+  /// Return true if this instruction requires
   /// custom insertion support when the DAG scheduler is inserting it into a
   /// machine basic block.  If this is true for the instruction, it basically
   /// means that it is a pseudo instruction used at SelectionDAG time that is
@@ -641,7 +626,7 @@ public:
     return hasProperty(MCID::UsesCustomInserter, Type);
   }
 
-  /// hasPostISelHook - Return true if this instruction requires *adjustment*
+  /// Return true if this instruction requires *adjustment*
   /// after instruction selection by calling a target hook. For example, this
   /// can be used to fill in ARM 's' optional operand depending on whether
   /// the conditional flag register is used.
@@ -649,8 +634,8 @@ public:
     return hasProperty(MCID::HasPostISelHook, Type);
   }
 
-  /// isRematerializable - Returns true if this instruction is a candidate for
-  /// remat.  This flag is deprecated, please don't use it anymore.  If this
+  /// Returns true if this instruction is a candidate for remat.
+  /// This flag is deprecated, please don't use it anymore.  If this
   /// flag is set, the isReallyTriviallyReMaterializable() method is called to
   /// verify the instruction is really rematable.
   bool isRematerializable(QueryType Type = AllInBundle) const {
@@ -659,9 +644,9 @@ public:
     return hasProperty(MCID::Rematerializable, Type);
   }
 
-  /// isAsCheapAsAMove - Returns true if this instruction has the same cost (or
-  /// less) than a move instruction. This is useful during certain types of
-  /// optimizations (e.g., remat during two-address conversion or machine licm)
+  /// Returns true if this instruction has the same cost (or less) than a move
+  /// instruction. This is useful during certain types of optimizations
+  /// (e.g., remat during two-address conversion or machine licm)
   /// where we would like to remat or hoist the instruction, but not if it costs
   /// more than moving the instruction into the appropriate register. Note, we
   /// are not marking copies from and to the same register class with this flag.
@@ -670,7 +655,7 @@ public:
     return hasProperty(MCID::CheapAsAMove, Type);
   }
 
-  /// hasExtraSrcRegAllocReq - Returns true if this instruction source operands
+  /// Returns true if this instruction source operands
   /// have special register allocation requirements that are not captured by the
   /// operand register classes. e.g. ARM::STRD's two source registers must be an
   /// even / odd pair, ARM::STM registers have to be in ascending order.
@@ -680,7 +665,7 @@ public:
     return hasProperty(MCID::ExtraSrcRegAllocReq, Type);
   }
 
-  /// hasExtraDefRegAllocReq - Returns true if this instruction def operands
+  /// Returns true if this instruction def operands
   /// have special register allocation requirements that are not captured by the
   /// operand register classes. e.g. ARM::LDRD's two def registers must be an
   /// even / odd pair, ARM::LDM registers have to be in ascending order.
@@ -698,7 +683,7 @@ public:
     IgnoreVRegDefs  // Ignore virtual register definitions
   };
 
-  /// isIdenticalTo - Return true if this instruction is identical to (same
+  /// Return true if this instruction is identical to (same
   /// opcode and same operands as) the specified instruction.
   bool isIdenticalTo(const MachineInstr *Other,
                      MICheckType Check = CheckDefs) const;
@@ -739,8 +724,7 @@ public:
   bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; }
   bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; }
 
-  /// isLabel - Returns true if the MachineInstr represents a label.
-  ///
+  /// Returns true if the MachineInstr represents a label.
   bool isLabel() const { return isEHLabel() || isGCLabel(); }
   bool isCFIInstruction() const {
     return getOpcode() == TargetOpcode::CFI_INSTRUCTION;
@@ -789,19 +773,19 @@ public:
     return getOpcode() == TargetOpcode::EXTRACT_SUBREG;
   }
 
-  /// isCopyLike - Return true if the instruction behaves like a copy.
+  /// Return true if the instruction behaves like a copy.
   /// This does not include native copy instructions.
   bool isCopyLike() const {
     return isCopy() || isSubregToReg();
   }
 
-  /// isIdentityCopy - Return true is the instruction is an identity copy.
+  /// Return true is the instruction is an identity copy.
   bool isIdentityCopy() const {
     return isCopy() && getOperand(0).getReg() == getOperand(1).getReg() &&
       getOperand(0).getSubReg() == getOperand(1).getSubReg();
   }
 
-  /// isTransient - Return true if this is a transient instruction that is
+  /// Return true if this is a transient instruction that is
   /// either very likely to be eliminated during register allocation (such as
   /// copy-like instructions), or if this instruction doesn't have an
   /// execution-time cost.
@@ -832,8 +816,8 @@ public:
   /// skips, 0 for unbundled instructions.
   unsigned getBundleSize() const;
 
-  /// readsRegister - Return true if the MachineInstr reads the specified
-  /// register. If TargetRegisterInfo is passed, then it also checks if there
+  /// Return true if the MachineInstr reads the specified register.
+  /// If TargetRegisterInfo is passed, then it also checks if there
   /// is a read of a super-register.
   /// This does not count partial redefines of virtual registers as reads:
   ///   %reg1024:6 = OP.
@@ -842,30 +826,29 @@ public:
     return findRegisterUseOperandIdx(Reg, false, TRI) != -1;
   }
 
-  /// readsVirtualRegister - Return true if the MachineInstr reads the specified
-  /// virtual register. Take into account that a partial define is a
+  /// Return true if the MachineInstr reads the specified virtual register.
+  /// Take into account that a partial define is a
   /// read-modify-write operation.
   bool readsVirtualRegister(unsigned Reg) const {
     return readsWritesVirtualRegister(Reg).first;
   }
 
-  /// readsWritesVirtualRegister - Return a pair of bools (reads, writes)
-  /// indicating if this instruction reads or writes Reg. This also considers
-  /// partial defines.
+  /// Return a pair of bools (reads, writes) indicating if this instruction
+  /// reads or writes Reg. This also considers partial defines.
   /// If Ops is not null, all operand indices for Reg are added.
   std::pair<bool,bool> readsWritesVirtualRegister(unsigned Reg,
                                 SmallVectorImpl<unsigned> *Ops = nullptr) const;
 
-  /// killsRegister - Return true if the MachineInstr kills the specified
-  /// register. If TargetRegisterInfo is passed, then it also checks if there is
+  /// Return true if the MachineInstr kills the specified register.
+  /// If TargetRegisterInfo is passed, then it also checks if there is
   /// a kill of a super-register.
   bool killsRegister(unsigned Reg,
                      const TargetRegisterInfo *TRI = nullptr) const {
     return findRegisterUseOperandIdx(Reg, true, TRI) != -1;
   }
 
-  /// definesRegister - Return true if the MachineInstr fully defines the
-  /// specified register. If TargetRegisterInfo is passed, then it also checks
+  /// Return true if the MachineInstr fully defines the specified register.
+  /// If TargetRegisterInfo is passed, then it also checks
   /// if there is a def of a super-register.
   /// NOTE: It's ignoring subreg indices on virtual registers.
   bool definesRegister(unsigned Reg,
@@ -873,28 +856,28 @@ public:
     return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1;
   }
 
-  /// modifiesRegister - Return true if the MachineInstr modifies (fully define
-  /// or partially define) the specified register.
+  /// Return true if the MachineInstr modifies (fully define or partially
+  /// define) the specified register.
   /// NOTE: It's ignoring subreg indices on virtual registers.
   bool modifiesRegister(unsigned Reg, const TargetRegisterInfo *TRI) const {
     return findRegisterDefOperandIdx(Reg, false, true, TRI) != -1;
   }
 
-  /// registerDefIsDead - Returns true if the register is dead in this machine
-  /// instruction. If TargetRegisterInfo is passed, then it also checks
+  /// Returns true if the register is dead in this machine instruction.
+  /// If TargetRegisterInfo is passed, then it also checks
   /// if there is a dead def of a super-register.
   bool registerDefIsDead(unsigned Reg,
                          const TargetRegisterInfo *TRI = nullptr) const {
     return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1;
   }
 
-  /// findRegisterUseOperandIdx() - Returns the operand index that is a use of
-  /// the specific register or -1 if it is not found. It further tightens
-  /// the search criteria to a use that kills the register if isKill is true.
+  /// Returns the operand index that is a use of the specific register or -1
+  /// if it is not found. It further tightens the search criteria to a use
+  /// that kills the register if isKill is true.
   int findRegisterUseOperandIdx(unsigned Reg, bool isKill = false,
                                 const TargetRegisterInfo *TRI = nullptr) const;
 
-  /// findRegisterUseOperand - Wrapper for findRegisterUseOperandIdx, it returns
+  /// Wrapper for findRegisterUseOperandIdx, it returns
   /// a pointer to the MachineOperand rather than an index.
   MachineOperand *findRegisterUseOperand(unsigned Reg, bool isKill = false,
                                       const TargetRegisterInfo *TRI = nullptr) {
@@ -902,17 +885,17 @@ public:
     return (Idx == -1) ? nullptr : &getOperand(Idx);
   }
 
-  /// findRegisterDefOperandIdx() - Returns the operand index that is a def of
-  /// the specified register or -1 if it is not found. If isDead is true, defs
-  /// that are not dead are skipped. If Overlap is true, then it also looks for
-  /// defs that merely overlap the specified register. If TargetRegisterInfo is
-  /// non-null, then it also checks if there is a def of a super-register.
+  /// Returns the operand index that is a def of the specified register or
+  /// -1 if it is not found. If isDead is true, defs that are not dead are
+  /// skipped. If Overlap is true, then it also looks for defs that merely
+  /// overlap the specified register. If TargetRegisterInfo is non-null,
+  /// then it also checks if there is a def of a super-register.
   /// This may also return a register mask operand when Overlap is true.
   int findRegisterDefOperandIdx(unsigned Reg,
                                 bool isDead = false, bool Overlap = false,
                                 const TargetRegisterInfo *TRI = nullptr) const;
 
-  /// findRegisterDefOperand - Wrapper for findRegisterDefOperandIdx, it returns
+  /// Wrapper for findRegisterDefOperandIdx, it returns
   /// a pointer to the MachineOperand rather than an index.
   MachineOperand *findRegisterDefOperand(unsigned Reg, bool isDead = false,
                                       const TargetRegisterInfo *TRI = nullptr) {
@@ -920,12 +903,12 @@ public:
     return (Idx == -1) ? nullptr : &getOperand(Idx);
   }
 
-  /// findFirstPredOperandIdx() - Find the index of the first operand in the
+  /// Find the index of the first operand in the
   /// operand list that is used to represent the predicate. It returns -1 if
   /// none is found.
   int findFirstPredOperandIdx() const;
 
-  /// findInlineAsmFlagIdx() - Find the index of the flag word operand that
+  /// Find the index of the flag word operand that
   /// corresponds to operand OpIdx on an inline asm instruction.  Returns -1 if
   /// getOperand(OpIdx) does not belong to an inline asm operand group.
   ///
@@ -937,9 +920,9 @@ public:
   ///
   int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const;
 
-  /// getRegClassConstraint - Compute the static register class constraint for
-  /// operand OpIdx.  For normal instructions, this is derived from the
-  /// MCInstrDesc.  For inline assembly it is derived from the flag words.
+  /// Compute the static register class constraint for operand OpIdx.
+  /// For normal instructions, this is derived from the MCInstrDesc.
+  /// For inline assembly it is derived from the flag words.
   ///
   /// Returns NULL if the static register classs constraint cannot be
   /// determined.
@@ -980,20 +963,20 @@ public:
                               const TargetInstrInfo *TII,
                               const TargetRegisterInfo *TRI) const;
 
-  /// tieOperands - Add a tie between the register operands at DefIdx and
-  /// UseIdx. The tie will cause the register allocator to ensure that the two
+  /// Add a tie between the register operands at DefIdx and UseIdx.
+  /// The tie will cause the register allocator to ensure that the two
   /// operands are assigned the same physical register.
   ///
   /// Tied operands are managed automatically for explicit operands in the
   /// MCInstrDesc. This method is for exceptional cases like inline asm.
   void tieOperands(unsigned DefIdx, unsigned UseIdx);
 
-  /// findTiedOperandIdx - Given the index of a tied register operand, find the
+  /// Given the index of a tied register operand, find the
   /// operand it is tied to. Defs are tied to uses and vice versa. Returns the
   /// index of the tied operand which must exist.
   unsigned findTiedOperandIdx(unsigned OpIdx) const;
 
-  /// isRegTiedToUseOperand - Given the index of a register def operand,
+  /// Given the index of a register def operand,
   /// check if the register def is tied to a source operand, due to either
   /// two-address elimination or inline assembly constraints. Returns the
   /// first tied use operand index by reference if UseOpIdx is not null.
@@ -1007,9 +990,9 @@ public:
     return true;
   }
 
-  /// isRegTiedToDefOperand - Return true if the use operand of the specified
-  /// index is tied to a def operand. It also returns the def operand index by
-  /// reference if DefOpIdx is not null.
+  /// Return true if the use operand of the specified index is tied to a def
+  /// operand. It also returns the def operand index by reference if DefOpIdx
+  /// is not null.
   bool isRegTiedToDefOperand(unsigned UseOpIdx,
                              unsigned *DefOpIdx = nullptr) const {
     const MachineOperand &MO = getOperand(UseOpIdx);
@@ -1020,16 +1003,15 @@ public:
     return true;
   }
 
-  /// clearKillInfo - Clears kill flags on all operands.
-  ///
+  /// Clears kill flags on all operands.
   void clearKillInfo();
 
-  /// substituteRegister - Replace all occurrences of FromReg with ToReg:SubIdx,
+  /// Replace all occurrences of FromReg with ToReg:SubIdx,
   /// properly composing subreg indices where necessary.
   void substituteRegister(unsigned FromReg, unsigned ToReg, unsigned SubIdx,
                           const TargetRegisterInfo &RegInfo);
 
-  /// addRegisterKilled - We have determined MI kills a register. Look for the
+  /// We have determined MI kills a register. Look for the
   /// operand that uses it and mark it as IsKill. If AddIfNotFound is true,
   /// add a implicit operand if it's not found. Returns true if the operand
   /// exists / is added.
@@ -1037,23 +1019,31 @@ public:
                          const TargetRegisterInfo *RegInfo,
                          bool AddIfNotFound = false);
 
-  /// clearRegisterKills - Clear all kill flags affecting Reg.  If RegInfo is
+  /// Clear all kill flags affecting Reg.  If RegInfo is
   /// provided, this includes super-register kills.
   void clearRegisterKills(unsigned Reg, const TargetRegisterInfo *RegInfo);
 
-  /// addRegisterDead - We have determined MI defined a register without a use.
+  /// We have determined MI defined a register without a use.
   /// Look for the operand that defines it and mark it as IsDead. If
   /// AddIfNotFound is true, add a implicit operand if it's not found. Returns
   /// true if the operand exists / is added.
   bool addRegisterDead(unsigned Reg, const TargetRegisterInfo *RegInfo,
                        bool AddIfNotFound = false);
 
-  /// addRegisterDefined - We have determined MI defines a register. Make sure
-  /// there is an operand defining Reg.
+  /// Clear all dead flags on operands defining register @p Reg.
+  void clearRegisterDeads(unsigned Reg);
+
+  /// Mark all subregister defs of register @p Reg with the undef flag.
+  /// This function is used when we determined to have a subregister def in an
+  /// otherwise undefined super register.
+  void addRegisterDefReadUndef(unsigned Reg);
+
+  /// We have determined MI defines a register. Make sure there is an operand
+  /// defining Reg.
   void addRegisterDefined(unsigned Reg,
                           const TargetRegisterInfo *RegInfo = nullptr);
 
-  /// setPhysRegsDeadExcept - Mark every physreg used by this instruction as
+  /// Mark every physreg used by this instruction as
   /// dead except those in the UsedRegs list.
   ///
   /// On instructions with register mask operands, also add implicit-def
@@ -1061,32 +1051,30 @@ public:
   void setPhysRegsDeadExcept(ArrayRef<unsigned> UsedRegs,
                              const TargetRegisterInfo &TRI);
 
-  /// isSafeToMove - Return true if it is safe to move this instruction. If
+  /// Return true if it is safe to move this instruction. If
   /// SawStore is set to true, it means that there is a store (or call) between
   /// the instruction's location and its intended destination.
-  bool isSafeToMove(const TargetInstrInfo *TII, AliasAnalysis *AA,
-                    bool &SawStore) const;
+  bool isSafeToMove(AliasAnalysis *AA, bool &SawStore) const;
 
-  /// hasOrderedMemoryRef - Return true if this instruction may have an ordered
+  /// Return true if this instruction may have an ordered
   /// or volatile memory reference, or if the information describing the memory
   /// reference is not available. Return false if it is known to have no
   /// ordered or volatile memory references.
   bool hasOrderedMemoryRef() const;
 
-  /// isInvariantLoad - Return true if this instruction is loading from a
+  /// Return true if this instruction is loading from a
   /// location whose value is invariant across the function.  For example,
   /// loading a value from the constant pool or from the argument area of
   /// a function if it does not change.  This should only return true of *all*
   /// loads the instruction does are invariant (if it does multiple loads).
   bool isInvariantLoad(AliasAnalysis *AA) const;
 
-  /// isConstantValuePHI - If the specified instruction is a PHI that always
-  /// merges together the same virtual register, return the register, otherwise
-  /// return 0.
+  /// If the specified instruction is a PHI that always merges together the
+  /// same virtual register, return the register, otherwise return 0.
   unsigned isConstantValuePHI() const;
 
-  /// hasUnmodeledSideEffects - Return true if this instruction has side
-  /// effects that are not modeled by mayLoad / mayStore, etc.
+  /// Return true if this instruction has side effects that are not modeled
+  /// by mayLoad / mayStore, etc.
   /// For all instructions, the property is encoded in MCInstrDesc::Flags
   /// (see MCInstrDesc::hasUnmodeledSideEffects(). The only exception is
   /// INLINEASM instruction, in which case the side effect property is encoded
@@ -1094,19 +1082,17 @@ public:
   ///
   bool hasUnmodeledSideEffects() const;
 
-  /// allDefsAreDead - Return true if all the defs of this instruction are dead.
-  ///
+  /// Return true if all the defs of this instruction are dead.
   bool allDefsAreDead() const;
 
-  /// copyImplicitOps - Copy implicit register operands from specified
+  /// Copy implicit register operands from specified
   /// instruction to this instruction.
   void copyImplicitOps(MachineFunction &MF, const MachineInstr *MI);
 
   //
   // Debugging support
   //
-  void print(raw_ostream &OS, const TargetMachine *TM = nullptr,
-             bool SkipOpers = false) const;
+  void print(raw_ostream &OS, bool SkipOpers = false) const;
   void dump() const;
 
   //===--------------------------------------------------------------------===//
@@ -1131,44 +1117,41 @@ public:
   /// preferred.
   void addOperand(const MachineOperand &Op);
 
-  /// setDesc - Replace the instruction descriptor (thus opcode) of
+  /// Replace the instruction descriptor (thus opcode) of
   /// the current instruction with a new one.
-  ///
   void setDesc(const MCInstrDesc &tid) { MCID = &tid; }
 
-  /// setDebugLoc - Replace current source information with new such.
+  /// Replace current source information with new such.
   /// Avoid using this, the constructor argument is preferable.
-  ///
-  void setDebugLoc(const DebugLoc dl) {
-    debugLoc = dl;
+  void setDebugLoc(DebugLoc dl) {
+    debugLoc = std::move(dl);
     assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
   }
 
-  /// RemoveOperand - Erase an operand  from an instruction, leaving it with one
+  /// Erase an operand  from an instruction, leaving it with one
   /// fewer operand than it started with.
-  ///
   void RemoveOperand(unsigned i);
 
-  /// addMemOperand - Add a MachineMemOperand to the machine instruction.
+  /// Add a MachineMemOperand to the machine instruction.
   /// This function should be used only occasionally. The setMemRefs function
   /// is the primary method for setting up a MachineInstr's MemRefs list.
   void addMemOperand(MachineFunction &MF, MachineMemOperand *MO);
 
-  /// setMemRefs - Assign this MachineInstr's memory reference descriptor
-  /// list. This does not transfer ownership.
+  /// Assign this MachineInstr's memory reference descriptor list.
+  /// This does not transfer ownership.
   void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) {
     MemRefs = NewMemRefs;
     NumMemRefs = uint8_t(NewMemRefsEnd - NewMemRefs);
     assert(NumMemRefs == NewMemRefsEnd - NewMemRefs && "Too many memrefs");
   }
 
-private:
-  /// getRegInfo - If this instruction is embedded into a MachineFunction,
-  /// return the MachineRegisterInfo object for the current function, otherwise
-  /// return null.
-  MachineRegisterInfo *getRegInfo();
+  /// Clear this MachineInstr's memory reference descriptor list.
+  void clearMemRefs() {
+    MemRefs = nullptr;
+    NumMemRefs = 0;
+  }
 
-  /// untieRegOperand - Break any tie involving OpIdx.
+  /// Break any tie involving OpIdx.
   void untieRegOperand(unsigned OpIdx) {
     MachineOperand &MO = getOperand(OpIdx);
     if (MO.isReg() && MO.isTied()) {
@@ -1177,22 +1160,27 @@ private:
     }
   }
 
-  /// addImplicitDefUseOperands - Add all implicit def and use operands to
-  /// this instruction.
+
+private:
+  /// If this instruction is embedded into a MachineFunction, return the
+  /// MachineRegisterInfo object for the current function, otherwise
+  /// return null.
+  MachineRegisterInfo *getRegInfo();
+
+  /// Add all implicit def and use operands to this instruction.
   void addImplicitDefUseOperands(MachineFunction &MF);
 
-  /// RemoveRegOperandsFromUseLists - Unlink all of the register operands in
-  /// this instruction from their respective use lists.  This requires that the
-  /// operands already be on their use lists.
+  /// Unlink all of the register operands in this instruction from their
+  /// respective use lists.  This requires that the operands already be on their
+  /// use lists.
   void RemoveRegOperandsFromUseLists(MachineRegisterInfo&);
 
-  /// AddRegOperandsToUseLists - Add all of the register operands in
-  /// this instruction from their respective use lists.  This requires that the
-  /// operands not be on their use lists yet.
+  /// Add all of the register operands in this instruction from their
+  /// respective use lists.  This requires that the operands not be on their
+  /// use lists yet.
   void AddRegOperandsToUseLists(MachineRegisterInfo&);
 
-  /// hasPropertyInBundle - Slow path for hasProperty when we're dealing with a
-  /// bundle.
+  /// Slow path for hasProperty when we're dealing with a bundle.
   bool hasPropertyInBundle(unsigned Mask, QueryType Type) const;
 
   /// \brief Implements the logic of getRegClassConstraintEffectForVReg for the
@@ -1203,8 +1191,8 @@ private:
       const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const;
 };
 
-/// MachineInstrExpressionTrait - Special DenseMapInfo traits to compare
-/// MachineInstr* by *value* of the instruction rather than by pointer value.
+/// Special DenseMapInfo traits to compare MachineInstr* by *value* of the
+/// instruction rather than by pointer value.
 /// The hashing and equality testing functions ignore definitions so this is
 /// useful for CSE, etc.
 struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> {
diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h
index 8859b6a019ea4..47397c6d67434 100644
--- a/include/llvm/CodeGen/MachineInstrBuilder.h
+++ b/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -174,8 +174,9 @@ public:
 
   const MachineInstrBuilder &addMetadata(const MDNode *MD) const {
     MI->addOperand(*MF, MachineOperand::CreateMetadata(MD));
-    assert((MI->isDebugValue() ? MI->getDebugVariable().Verify() : true) &&
-           "first MDNode argument of a DBG_VALUE not a DIVariable");
+    assert((MI->isDebugValue() ? static_cast<bool>(MI->getDebugVariable())
+                               : true) &&
+           "first MDNode argument of a DBG_VALUE not a variable");
     return *this;
   }
 
@@ -355,8 +356,10 @@ inline MachineInstrBuilder BuildMI(MachineFunction &MF, DebugLoc DL,
                                    const MCInstrDesc &MCID, bool IsIndirect,
                                    unsigned Reg, unsigned Offset,
                                    const MDNode *Variable, const MDNode *Expr) {
-  assert(DIVariable(Variable).Verify() && "not a DIVariable");
-  assert(DIExpression(Expr).Verify() && "not a DIExpression");
+  assert(isa<DILocalVariable>(Variable) && "not a variable");
+  assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
+  assert(cast<DILocalVariable>(Variable)->isValidLocationForIntrinsic(DL) &&
+         "Expected inlined-at fields to agree");
   if (IsIndirect)
     return BuildMI(MF, DL, MCID)
         .addReg(Reg, RegState::Debug)
@@ -382,8 +385,8 @@ inline MachineInstrBuilder BuildMI(MachineBasicBlock &BB,
                                    const MCInstrDesc &MCID, bool IsIndirect,
                                    unsigned Reg, unsigned Offset,
                                    const MDNode *Variable, const MDNode *Expr) {
-  assert(DIVariable(Variable).Verify() && "not a DIVariable");
-  assert(DIExpression(Expr).Verify() && "not a DIExpression");
+  assert(isa<DILocalVariable>(Variable) && "not a variable");
+  assert(cast<DIExpression>(Expr)->isValid() && "not an expression");
   MachineFunction &MF = *BB.getParent();
   MachineInstr *MI =
       BuildMI(MF, DL, MCID, IsIndirect, Reg, Offset, Variable, Expr);
diff --git a/include/llvm/CodeGen/MachineLoopInfo.h b/include/llvm/CodeGen/MachineLoopInfo.h
index 4fbd46b6fb575..f7bcf45d93c56 100644
--- a/include/llvm/CodeGen/MachineLoopInfo.h
+++ b/include/llvm/CodeGen/MachineLoopInfo.h
@@ -74,8 +74,8 @@ class MachineLoopInfo : public MachineFunctionPass {
   LoopInfoBase<MachineBasicBlock, MachineLoop> LI;
   friend class LoopBase<MachineBasicBlock, MachineLoop>;
 
-  void operator=(const MachineLoopInfo &) LLVM_DELETED_FUNCTION;
-  MachineLoopInfo(const MachineLoopInfo &) LLVM_DELETED_FUNCTION;
+  void operator=(const MachineLoopInfo &) = delete;
+  MachineLoopInfo(const MachineLoopInfo &) = delete;
 
 public:
   static char ID; // Pass identification, replacement for typeid
diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h
index eb5086cbe5a5f..a16c294a07491 100644
--- a/include/llvm/CodeGen/MachineMemOperand.h
+++ b/include/llvm/CodeGen/MachineMemOperand.h
@@ -199,6 +199,24 @@ public:
   /// Profile - Gather unique data for the object.
   ///
   void Profile(FoldingSetNodeID &ID) const;
+
+  friend bool operator==(const MachineMemOperand &LHS,
+                         const MachineMemOperand &RHS) {
+    return LHS.getValue() == RHS.getValue() &&
+           LHS.getPseudoValue() == RHS.getPseudoValue() &&
+           LHS.getSize() == RHS.getSize() &&
+           LHS.getOffset() == RHS.getOffset() &&
+           LHS.getFlags() == RHS.getFlags() &&
+           LHS.getAAInfo() == RHS.getAAInfo() &&
+           LHS.getRanges() == RHS.getRanges() &&
+           LHS.getAlignment() == RHS.getAlignment() &&
+           LHS.getAddrSpace() == RHS.getAddrSpace();
+  }
+
+  friend bool operator!=(const MachineMemOperand &LHS,
+                         const MachineMemOperand &RHS) {
+    return !(LHS == RHS);
+  }
 };
 
 raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MRO);
diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h
index 584ce65c9d175..ce45c16591533 100644
--- a/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/include/llvm/CodeGen/MachineModuleInfo.h
@@ -35,6 +35,7 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/Analysis/LibCallSemantics.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Metadata.h"
 #include "llvm/IR/ValueHandle.h"
@@ -50,6 +51,7 @@ namespace llvm {
 // Forward declarations.
 class Constant;
 class GlobalVariable;
+class BlockAddress;
 class MDNode;
 class MMIAddrLabelMap;
 class MachineBasicBlock;
@@ -57,21 +59,33 @@ class MachineFunction;
 class Module;
 class PointerType;
 class StructType;
+struct WinEHFuncInfo;
+
+struct SEHHandler {
+  // Filter or finally function. Null indicates a catch-all.
+  const Function *FilterOrFinally;
+
+  // Address of block to recover at. Null for a finally handler.
+  const BlockAddress *RecoverBA;
+};
 
 //===----------------------------------------------------------------------===//
 /// LandingPadInfo - This structure is used to retain landing pad info for
 /// the current function.
 ///
 struct LandingPadInfo {
-  MachineBasicBlock *LandingPadBlock;    // Landing pad block.
-  SmallVector<MCSymbol*, 1> BeginLabels; // Labels prior to invoke.
-  SmallVector<MCSymbol*, 1> EndLabels;   // Labels after invoke.
-  MCSymbol *LandingPadLabel;             // Label at beginning of landing pad.
-  const Function *Personality;           // Personality function.
-  std::vector<int> TypeIds;              // List of type ids (filters negative)
+  MachineBasicBlock *LandingPadBlock;      // Landing pad block.
+  SmallVector<MCSymbol *, 1> BeginLabels;  // Labels prior to invoke.
+  SmallVector<MCSymbol *, 1> EndLabels;    // Labels after invoke.
+  SmallVector<SEHHandler, 1> SEHHandlers;  // SEH handlers active at this lpad.
+  MCSymbol *LandingPadLabel;               // Label at beginning of landing pad.
+  const Function *Personality;             // Personality function.
+  std::vector<int> TypeIds;               // List of type ids (filters negative).
+  int WinEHState;                         // WinEH specific state number.
 
   explicit LandingPadInfo(MachineBasicBlock *MBB)
-    : LandingPadBlock(MBB), LandingPadLabel(nullptr), Personality(nullptr) {}
+      : LandingPadBlock(MBB), LandingPadLabel(nullptr), Personality(nullptr),
+        WinEHState(-1) {}
 };
 
 //===----------------------------------------------------------------------===//
@@ -86,7 +100,10 @@ public:
   virtual ~MachineModuleInfoImpl();
   typedef std::vector<std::pair<MCSymbol*, StubValueTy> > SymbolListTy;
 protected:
-  static SymbolListTy GetSortedStubs(const DenseMap<MCSymbol*, StubValueTy>&);
+
+  /// Return the entries from a DenseMap in a deterministic sorted orer.
+  /// Clears the map.
+  static SymbolListTy getSortedStubs(DenseMap<MCSymbol*, StubValueTy>&);
 };
 
 //===----------------------------------------------------------------------===//
@@ -168,16 +185,21 @@ class MachineModuleInfo : public ImmutablePass {
   /// details.
   bool UsesMorestackAddr;
 
+  EHPersonality PersonalityTypeCache;
+
+  DenseMap<const Function *, std::unique_ptr<WinEHFuncInfo>> FuncInfoMap;
+
 public:
   static char ID; // Pass identification, replacement for typeid
 
   struct VariableDbgInfo {
-    TrackingMDNodeRef Var;
-    TrackingMDNodeRef Expr;
+    const DILocalVariable *Var;
+    const DIExpression *Expr;
     unsigned Slot;
-    DebugLoc Loc;
+    const DILocation *Loc;
 
-    VariableDbgInfo(MDNode *Var, MDNode *Expr, unsigned Slot, DebugLoc Loc)
+    VariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr,
+                    unsigned Slot, const DILocation *Loc)
         : Var(Var), Expr(Expr), Slot(Slot), Loc(Loc) {}
   };
   typedef SmallVector<VariableDbgInfo, 4> VariableDbgInfoMapTy;
@@ -187,7 +209,7 @@ public:
   // Real constructor.
   MachineModuleInfo(const MCAsmInfo &MAI, const MCRegisterInfo &MRI,
                     const MCObjectFileInfo *MOFI);
-  ~MachineModuleInfo();
+  ~MachineModuleInfo() override;
 
   // Initialization and Finalization
   bool doInitialization(Module &) override;
@@ -203,6 +225,12 @@ public:
   void setModule(const Module *M) { TheModule = M; }
   const Module *getModule() const { return TheModule; }
 
+  const Function *getWinEHParent(const Function *F) const;
+  WinEHFuncInfo &getWinEHFuncInfo(const Function *F);
+  bool hasWinEHFuncInfo(const Function *F) const {
+    return FuncInfoMap.count(getWinEHParent(F)) > 0;
+  }
+
   /// getInfo - Keep track of various per-function pieces of information for
   /// backends that would like to do so.
   ///
@@ -300,6 +328,8 @@ public:
   void addPersonality(MachineBasicBlock *LandingPad,
                       const Function *Personality);
 
+  void addWinEHState(MachineBasicBlock *LandingPad, int State);
+
   /// getPersonalityIndex - Get index of the current personality function inside
   /// Personalitites array
   unsigned getPersonalityIndex() const;
@@ -330,6 +360,12 @@ public:
   ///
   void addCleanup(MachineBasicBlock *LandingPad);
 
+  void addSEHCatchHandler(MachineBasicBlock *LandingPad, const Function *Filter,
+                          const BlockAddress *RecoverLabel);
+
+  void addSEHCleanupHandler(MachineBasicBlock *LandingPad,
+                            const Function *Cleanup);
+
   /// getTypeIDFor - Return the type id for the specified typeinfo.  This is
   /// function wide.
   unsigned getTypeIDFor(const GlobalValue *TI);
@@ -407,10 +443,13 @@ public:
   /// of one is required to emit exception handling info.
   const Function *getPersonality() const;
 
+  /// Classify the personality function amongst known EH styles.
+  EHPersonality getPersonalityType();
+
   /// setVariableDbgInfo - Collect information used to emit debugging
   /// information of a variable.
-  void setVariableDbgInfo(MDNode *Var, MDNode *Expr, unsigned Slot,
-                          DebugLoc Loc) {
+  void setVariableDbgInfo(const DILocalVariable *Var, const DIExpression *Expr,
+                          unsigned Slot, const DILocation *Loc) {
     VariableDbgInfos.emplace_back(Var, Expr, Slot, Loc);
   }
 
diff --git a/include/llvm/CodeGen/MachineModuleInfoImpls.h b/include/llvm/CodeGen/MachineModuleInfoImpls.h
index 7afc7eb6b3577..a67f9b5666b17 100644
--- a/include/llvm/CodeGen/MachineModuleInfoImpls.h
+++ b/include/llvm/CodeGen/MachineModuleInfoImpls.h
@@ -58,14 +58,14 @@ namespace llvm {
     }
 
     /// Accessor methods to return the set of stubs in sorted order.
-    SymbolListTy GetFnStubList() const {
-      return GetSortedStubs(FnStubs);
+    SymbolListTy GetFnStubList() {
+      return getSortedStubs(FnStubs);
     }
-    SymbolListTy GetGVStubList() const {
-      return GetSortedStubs(GVStubs);
+    SymbolListTy GetGVStubList() {
+      return getSortedStubs(GVStubs);
     }
-    SymbolListTy GetHiddenGVStubList() const {
-      return GetSortedStubs(HiddenGVStubs);
+    SymbolListTy GetHiddenGVStubList() {
+      return getSortedStubs(HiddenGVStubs);
     }
   };
 
@@ -87,8 +87,8 @@ namespace llvm {
 
     /// Accessor methods to return the set of stubs in sorted order.
 
-    SymbolListTy GetGVStubList() const {
-      return GetSortedStubs(GVStubs);
+    SymbolListTy GetGVStubList() {
+      return getSortedStubs(GVStubs);
     }
   };
 
diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h
index eed1e575f93b6..ddffdcaf1e4db 100644
--- a/include/llvm/CodeGen/MachineOperand.h
+++ b/include/llvm/CodeGen/MachineOperand.h
@@ -65,7 +65,7 @@ public:
 private:
   /// OpKind - Specify what kind of operand this is.  This discriminates the
   /// union.
-  MachineOperandType OpKind;
+  MachineOperandType OpKind : 8;
 
   /// Subregister number for MO_Register.  A value of 0 indicates the
   /// MO_Register has no subReg.
@@ -217,7 +217,7 @@ public:
   ///
   void clearParent() { ParentMI = nullptr; }
 
-  void print(raw_ostream &os, const TargetMachine *TM = nullptr) const;
+  void print(raw_ostream &os, const TargetRegisterInfo *TRI = nullptr) const;
 
   //===--------------------------------------------------------------------===//
   // Accessors that tell you what kind of MachineOperand you're looking at.
@@ -554,6 +554,12 @@ public:
   /// already, the setFPImm method should be used.
   void ChangeToFPImmediate(const ConstantFP *FPImm);
 
+  /// ChangeToES - Replace this operand with a new external symbol operand.
+  void ChangeToES(const char *SymName, unsigned char TargetFlags = 0);
+
+  /// ChangeToMCSymbol - Replace this operand with a new MC symbol operand.
+  void ChangeToMCSymbol(MCSymbol *Sym);
+
   /// ChangeToRegister - Replace this operand with a new register operand of
   /// the specified value.  If an operand is known to be an register already,
   /// the setReg method should be used.
diff --git a/include/llvm/CodeGen/MachinePassRegistry.h b/include/llvm/CodeGen/MachinePassRegistry.h
index c962e687243e7..6731983c58746 100644
--- a/include/llvm/CodeGen/MachinePassRegistry.h
+++ b/include/llvm/CodeGen/MachinePassRegistry.h
@@ -122,11 +122,12 @@ template<class RegistryClass>
 class RegisterPassParser : public MachinePassRegistryListener,
                    public cl::parser<typename RegistryClass::FunctionPassCtor> {
 public:
-  RegisterPassParser() {}
-  ~RegisterPassParser() { RegistryClass::setListener(nullptr); }
+  RegisterPassParser(cl::Option &O)
+      : cl::parser<typename RegistryClass::FunctionPassCtor>(O) {}
+  ~RegisterPassParser() override { RegistryClass::setListener(nullptr); }
 
-  void initialize(cl::Option &O) {
-    cl::parser<typename RegistryClass::FunctionPassCtor>::initialize(O);
+  void initialize() {
+    cl::parser<typename RegistryClass::FunctionPassCtor>::initialize();
 
     // Add existing passes to option.
     for (RegistryClass *Node = RegistryClass::getList();
diff --git a/include/llvm/CodeGen/MachinePostDominators.h b/include/llvm/CodeGen/MachinePostDominators.h
index aab5c407629f7..70bdb191ad343 100644
--- a/include/llvm/CodeGen/MachinePostDominators.h
+++ b/include/llvm/CodeGen/MachinePostDominators.h
@@ -33,7 +33,7 @@ public:
 
   MachinePostDominatorTree();
 
-  ~MachinePostDominatorTree();
+  ~MachinePostDominatorTree() override;
 
   FunctionPass *createMachinePostDominatorTreePass();
 
diff --git a/include/llvm/CodeGen/MachineRegionInfo.h b/include/llvm/CodeGen/MachineRegionInfo.h
index 43499dba71c02..cf49c297c2880 100644
--- a/include/llvm/CodeGen/MachineRegionInfo.h
+++ b/include/llvm/CodeGen/MachineRegionInfo.h
@@ -57,8 +57,6 @@ public:
 
   }
 
-  ~MachineRegionNode() { }
-
   bool operator==(const MachineRegion &RN) const {
     return this == reinterpret_cast<const MachineRegionNode*>(&RN);
   }
@@ -80,7 +78,7 @@ class MachineRegionInfo : public RegionInfoBase<RegionTraits<MachineFunction>> {
 public:
   explicit MachineRegionInfo();
 
-  virtual ~MachineRegionInfo();
+  ~MachineRegionInfo() override;
 
   // updateStatistics - Update statistic about created regions.
   void updateStatistics(MachineRegion *R) final;
@@ -98,7 +96,7 @@ public:
   static char ID;
   explicit MachineRegionInfoPass();
 
-  ~MachineRegionInfoPass();
+  ~MachineRegionInfoPass() override;
 
   MachineRegionInfo &getRegionInfo() {
     return RI;
diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h
index caa48a5cf0cfc..e5b837aeea286 100644
--- a/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -123,8 +123,8 @@ private:
   /// second element.
   std::vector<std::pair<unsigned, unsigned> > LiveIns;
 
-  MachineRegisterInfo(const MachineRegisterInfo&) LLVM_DELETED_FUNCTION;
-  void operator=(const MachineRegisterInfo&) LLVM_DELETED_FUNCTION;
+  MachineRegisterInfo(const MachineRegisterInfo&) = delete;
+  void operator=(const MachineRegisterInfo&) = delete;
 public:
   explicit MachineRegisterInfo(const MachineFunction *MF);
 
@@ -182,7 +182,18 @@ public:
   /// information.
   void invalidateLiveness() { TracksLiveness = false; }
 
-  bool tracksSubRegLiveness() const { return TracksSubRegLiveness; }
+  /// Returns true if liveness for register class @p RC should be tracked at
+  /// the subregister level.
+  bool shouldTrackSubRegLiveness(const TargetRegisterClass &RC) const {
+    return subRegLivenessEnabled() && RC.HasDisjunctSubRegs;
+  }
+  bool shouldTrackSubRegLiveness(unsigned VReg) const {
+    assert(TargetRegisterInfo::isVirtualRegister(VReg) && "Must pass a VReg");
+    return shouldTrackSubRegLiveness(*getRegClass(VReg));
+  }
+  bool subRegLivenessEnabled() const {
+    return TracksSubRegLiveness;
+  }
 
   void enableSubRegLiveness(bool Enable = true) {
     TracksSubRegLiveness = Enable;
@@ -593,7 +604,7 @@ public:
   /// virtual register, for example after removing instructions or splitting
   /// the live range.
   ///
-  bool recomputeRegClass(unsigned Reg, const TargetMachine&);
+  bool recomputeRegClass(unsigned Reg);
 
   /// createVirtualRegister - Create and return a new virtual register in the
   /// function with the specified register class.
@@ -609,22 +620,25 @@ public:
 
   /// setRegAllocationHint - Specify a register allocation hint for the
   /// specified virtual register.
-  void setRegAllocationHint(unsigned Reg, unsigned Type, unsigned PrefReg) {
-    RegAllocHints[Reg].first  = Type;
-    RegAllocHints[Reg].second = PrefReg;
+  void setRegAllocationHint(unsigned VReg, unsigned Type, unsigned PrefReg) {
+    assert(TargetRegisterInfo::isVirtualRegister(VReg));
+    RegAllocHints[VReg].first  = Type;
+    RegAllocHints[VReg].second = PrefReg;
   }
 
   /// getRegAllocationHint - Return the register allocation hint for the
   /// specified virtual register.
   std::pair<unsigned, unsigned>
-  getRegAllocationHint(unsigned Reg) const {
-    return RegAllocHints[Reg];
+  getRegAllocationHint(unsigned VReg) const {
+    assert(TargetRegisterInfo::isVirtualRegister(VReg));
+    return RegAllocHints[VReg];
   }
 
   /// getSimpleHint - Return the preferred register allocation hint, or 0 if a
   /// standard simple hint (Type == 0) is not set.
-  unsigned getSimpleHint(unsigned Reg) const {
-    std::pair<unsigned, unsigned> Hint = getRegAllocationHint(Reg);
+  unsigned getSimpleHint(unsigned VReg) const {
+    assert(TargetRegisterInfo::isVirtualRegister(VReg));
+    std::pair<unsigned, unsigned> Hint = getRegAllocationHint(VReg);
     return Hint.first ? 0 : Hint.second;
   }
 
@@ -829,7 +843,6 @@ public:
     typedef std::iterator<std::forward_iterator_tag,
                           MachineInstr, ptrdiff_t>::pointer pointer;
 
-    defusechain_iterator(const defusechain_iterator &I) : Op(I.Op) {}
     defusechain_iterator() : Op(nullptr) {}
 
     bool operator==(const defusechain_iterator &x) const {
@@ -932,7 +945,6 @@ public:
     typedef std::iterator<std::forward_iterator_tag,
                           MachineInstr, ptrdiff_t>::pointer pointer;
 
-    defusechain_instr_iterator(const defusechain_instr_iterator &I) : Op(I.Op){}
     defusechain_instr_iterator() : Op(nullptr) {}
 
     bool operator==(const defusechain_instr_iterator &x) const {
diff --git a/include/llvm/CodeGen/MachineSSAUpdater.h b/include/llvm/CodeGen/MachineSSAUpdater.h
index 486a26e927148..5f988ad86320f 100644
--- a/include/llvm/CodeGen/MachineSSAUpdater.h
+++ b/include/llvm/CodeGen/MachineSSAUpdater.h
@@ -107,8 +107,8 @@ public:
 private:
   unsigned GetValueAtEndOfBlockInternal(MachineBasicBlock *BB);
 
-  void operator=(const MachineSSAUpdater&) LLVM_DELETED_FUNCTION;
-  MachineSSAUpdater(const MachineSSAUpdater&) LLVM_DELETED_FUNCTION;
+  void operator=(const MachineSSAUpdater&) = delete;
+  MachineSSAUpdater(const MachineSSAUpdater&) = delete;
 };
 
 } // End llvm namespace
diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h
index a31940161ca52..e80e14e5ccf75 100644
--- a/include/llvm/CodeGen/MachineScheduler.h
+++ b/include/llvm/CodeGen/MachineScheduler.h
@@ -385,7 +385,7 @@ public:
         ShouldTrackPressure(false), RPTracker(RegPressure),
         TopRPTracker(TopPressure), BotRPTracker(BotPressure) {}
 
-  virtual ~ScheduleDAGMILive();
+  ~ScheduleDAGMILive() override;
 
   /// Return true if this DAG supports VReg liveness and RegPressure.
   bool hasVRegLiveness() const override { return true; }
@@ -909,7 +909,7 @@ public:
   PostGenericScheduler(const MachineSchedContext *C):
     GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ") {}
 
-  virtual ~PostGenericScheduler() {}
+  ~PostGenericScheduler() override {}
 
   void initPolicy(MachineBasicBlock::iterator Begin,
                   MachineBasicBlock::iterator End,
diff --git a/include/llvm/CodeGen/MachineValueType.h b/include/llvm/CodeGen/MachineValueType.h
index e3fbfe89c203e..a728df3546778 100644
--- a/include/llvm/CodeGen/MachineValueType.h
+++ b/include/llvm/CodeGen/MachineValueType.h
@@ -26,7 +26,7 @@ namespace llvm {
   /// MVT - Machine Value Type. Every type that is supported natively by some
   /// processor targeted by LLVM occurs here. This means that any legal value
   /// type can be represented by an MVT.
-  class MVT {
+class MVT {
   public:
     enum SimpleValueType {
       // INVALID_SIMPLE_VALUE_TYPE - Simple value types less than zero are
@@ -86,22 +86,23 @@ namespace llvm {
       v4i64          =  39,   //  4 x i64
       v8i64          =  40,   //  8 x i64
       v16i64         =  41,   // 16 x i64
-
+      v1i128         =  42,   //  1 x i128
+      
       FIRST_INTEGER_VECTOR_VALUETYPE = v2i1,
-      LAST_INTEGER_VECTOR_VALUETYPE = v16i64,
-
-      v2f16          =  42,   //  2 x f16
-      v4f16          =  43,   //  4 x f16
-      v8f16          =  44,   //  8 x f16
-      v1f32          =  45,   //  1 x f32
-      v2f32          =  46,   //  2 x f32
-      v4f32          =  47,   //  4 x f32
-      v8f32          =  48,   //  8 x f32
-      v16f32         =  49,   // 16 x f32
-      v1f64          =  50,   //  1 x f64
-      v2f64          =  51,   //  2 x f64
-      v4f64          =  52,   //  4 x f64
-      v8f64          =  53,   //  8 x f64
+      LAST_INTEGER_VECTOR_VALUETYPE = v1i128,
+
+      v2f16          =  43,   //  2 x f16
+      v4f16          =  44,   //  4 x f16
+      v8f16          =  45,   //  8 x f16
+      v1f32          =  46,   //  1 x f32
+      v2f32          =  47,   //  2 x f32
+      v4f32          =  48,   //  4 x f32
+      v8f32          =  49,   //  8 x f32
+      v16f32         =  50,   // 16 x f32
+      v1f64          =  51,   //  1 x f64
+      v2f64          =  52,   //  2 x f64
+      v4f64          =  53,   //  4 x f64
+      v8f64          =  54,   //  8 x f64
 
       FIRST_FP_VECTOR_VALUETYPE = v2f16,
       LAST_FP_VECTOR_VALUETYPE = v8f64,
@@ -109,18 +110,18 @@ namespace llvm {
       FIRST_VECTOR_VALUETYPE = v2i1,
       LAST_VECTOR_VALUETYPE  = v8f64,
 
-      x86mmx         =  54,   // This is an X86 MMX value
+      x86mmx         =  55,   // This is an X86 MMX value
 
-      Glue           =  55,   // This glues nodes together during pre-RA sched
+      Glue           =  56,   // This glues nodes together during pre-RA sched
 
-      isVoid         =  56,   // This has no value
+      isVoid         =  57,   // This has no value
 
-      Untyped        =  57,   // This value takes a register, but has
+      Untyped        =  58,   // This value takes a register, but has
                               // unspecified type.  The register class
                               // will be determined by the opcode.
 
       FIRST_VALUETYPE = 0,    // This is always the beginning of the list.
-      LAST_VALUETYPE =  58,   // This always remains at the end of the list.
+      LAST_VALUETYPE =  59,   // This always remains at the end of the list.
 
       // This is the current maximum for LAST_VALUETYPE.
       // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
@@ -152,13 +153,17 @@ namespace llvm {
 
       // iPTR - An int value the size of the pointer of the current
       // target.  This should only be used internal to tblgen!
-      iPTR           = 255
+      iPTR           = 255,
+
+      // Any - Any type. This is used for intrinsics that have overloadings.
+      // This is only for tblgen's consumption!
+      Any            = 256
     };
 
     SimpleValueType SimpleTy;
 
-    MVT() : SimpleTy((SimpleValueType)(INVALID_SIMPLE_VALUE_TYPE)) {}
-    MVT(SimpleValueType SVT) : SimpleTy(SVT) { }
+    LLVM_CONSTEXPR MVT() : SimpleTy(INVALID_SIMPLE_VALUE_TYPE) {}
+    LLVM_CONSTEXPR MVT(SimpleValueType SVT) : SimpleTy(SVT) { }
 
     bool operator>(const MVT& S)  const { return SimpleTy >  S.SimpleTy; }
     bool operator<(const MVT& S)  const { return SimpleTy <  S.SimpleTy; }
@@ -218,10 +223,10 @@ namespace llvm {
 
     /// is128BitVector - Return true if this is a 128-bit vector type.
     bool is128BitVector() const {
-      return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 ||
-              SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 ||
-              SimpleTy == MVT::v8f16 || SimpleTy == MVT::v4f32 ||
-              SimpleTy == MVT::v2f64);
+      return (SimpleTy == MVT::v16i8  || SimpleTy == MVT::v8i16 ||
+              SimpleTy == MVT::v4i32  || SimpleTy == MVT::v2i64 ||
+              SimpleTy == MVT::v1i128 || SimpleTy == MVT::v8f16 ||
+              SimpleTy == MVT::v4f32  || SimpleTy == MVT::v2f64);
     }
 
     /// is256BitVector - Return true if this is a 256-bit vector type.
@@ -245,7 +250,8 @@ namespace llvm {
 
     /// isOverloaded - Return true if this is an overloaded type for TableGen.
     bool isOverloaded() const {
-      return (SimpleTy==MVT::iAny || SimpleTy==MVT::fAny ||
+      return (SimpleTy==MVT::Any  ||
+              SimpleTy==MVT::iAny || SimpleTy==MVT::fAny ||
               SimpleTy==MVT::vAny || SimpleTy==MVT::iPTRAny);
     }
 
@@ -305,6 +311,7 @@ namespace llvm {
       case v4i64:
       case v8i64:
       case v16i64: return i64;
+      case v1i128: return i128;
       case v2f16:
       case v4f16:
       case v8f16: return f16;
@@ -363,6 +370,7 @@ namespace llvm {
       case v1i16:
       case v1i32:
       case v1i64:
+      case v1i128:
       case v1f32:
       case v1f64: return 1;
       }
@@ -380,6 +388,7 @@ namespace llvm {
       case iAny:
       case fAny:
       case vAny:
+      case Any:
         llvm_unreachable("Value type is overloaded.");
       case Metadata:
         llvm_unreachable("Value type is metadata.");
@@ -421,6 +430,7 @@ namespace llvm {
       case v8i16:
       case v4i32:
       case v2i64:
+      case v1i128:
       case v8f16:
       case v4f32:
       case v2f64: return 128;
@@ -556,6 +566,9 @@ namespace llvm {
         if (NumElements == 8)  return MVT::v8i64;
         if (NumElements == 16) return MVT::v16i64;
         break;
+      case MVT::i128:
+        if (NumElements == 1)  return MVT::v1i128;
+        break;
       case MVT::f16:
         if (NumElements == 2)  return MVT::v2f16;
         if (NumElements == 4)  return MVT::v4f16;
diff --git a/include/llvm/CodeGen/PBQP/Graph.h b/include/llvm/CodeGen/PBQP/Graph.h
index 4dc5674ae134c..f73383ed10007 100644
--- a/include/llvm/CodeGen/PBQP/Graph.h
+++ b/include/llvm/CodeGen/PBQP/Graph.h
@@ -21,6 +21,7 @@
 #include <list>
 #include <map>
 #include <set>
+#include <vector>
 
 namespace llvm {
 namespace PBQP {
@@ -507,14 +508,14 @@ namespace PBQP {
       return getNode(NId).getAdjEdgeIds().size();
     }
 
-    /// @brief Set an edge's cost matrix.
+    /// @brief Update an edge's cost matrix.
     /// @param EId Edge id.
     /// @param Costs New cost matrix.
     template <typename OtherMatrixT>
-    void setEdgeCosts(EdgeId EId, OtherMatrixT Costs) {
+    void updateEdgeCosts(EdgeId EId, OtherMatrixT Costs) {
       MatrixPtr AllocatedCosts = CostAlloc.getMatrix(std::move(Costs));
       if (Solver)
-        Solver->handleSetEdgeCosts(EId, *AllocatedCosts);
+        Solver->handleUpdateCosts(EId, *AllocatedCosts);
       getEdge(EId).Costs = AllocatedCosts;
     }
 
@@ -548,14 +549,14 @@ namespace PBQP {
     /// @brief Get the first node connected to this edge.
     /// @param EId Edge id.
     /// @return The first node connected to the given edge.
-    NodeId getEdgeNode1Id(EdgeId EId) {
+    NodeId getEdgeNode1Id(EdgeId EId) const {
       return getEdge(EId).getN1Id();
     }
 
     /// @brief Get the second node connected to this edge.
     /// @param EId Edge id.
     /// @return The second node connected to the given edge.
-    NodeId getEdgeNode2Id(EdgeId EId) {
+    NodeId getEdgeNode2Id(EdgeId EId) const {
       return getEdge(EId).getN2Id();
     }
 
@@ -672,69 +673,6 @@ namespace PBQP {
       Edges.clear();
       FreeEdgeIds.clear();
     }
-
-    /// @brief Dump a graph to an output stream.
-    template <typename OStream>
-    void dumpToStream(OStream &OS) {
-      OS << nodeIds().size() << " " << edgeIds().size() << "\n";
-
-      for (auto NId : nodeIds()) {
-        const Vector& V = getNodeCosts(NId);
-        OS << "\n" << V.getLength() << "\n";
-        assert(V.getLength() != 0 && "Empty vector in graph.");
-        OS << V[0];
-        for (unsigned i = 1; i < V.getLength(); ++i) {
-          OS << " " << V[i];
-        }
-        OS << "\n";
-      }
-
-      for (auto EId : edgeIds()) {
-        NodeId N1Id = getEdgeNode1Id(EId);
-        NodeId N2Id = getEdgeNode2Id(EId);
-        assert(N1Id != N2Id && "PBQP graphs shound not have self-edges.");
-        const Matrix& M = getEdgeCosts(EId);
-        OS << "\n" << N1Id << " " << N2Id << "\n"
-           << M.getRows() << " " << M.getCols() << "\n";
-        assert(M.getRows() != 0 && "No rows in matrix.");
-        assert(M.getCols() != 0 && "No cols in matrix.");
-        for (unsigned i = 0; i < M.getRows(); ++i) {
-          OS << M[i][0];
-          for (unsigned j = 1; j < M.getCols(); ++j) {
-            OS << " " << M[i][j];
-          }
-          OS << "\n";
-        }
-      }
-    }
-
-    /// @brief Dump this graph to dbgs().
-    void dump() {
-      dumpToStream(dbgs());
-    }
-
-    /// @brief Print a representation of this graph in DOT format.
-    /// @param OS Output stream to print on.
-    template <typename OStream>
-    void printDot(OStream &OS) {
-      OS << "graph {\n";
-      for (auto NId : nodeIds()) {
-        OS << "  node" << NId << " [ label=\""
-           << NId << ": " << getNodeCosts(NId) << "\" ]\n";
-      }
-      OS << "  edge [ len=" << nodeIds().size() << " ]\n";
-      for (auto EId : edgeIds()) {
-        OS << "  node" << getEdgeNode1Id(EId)
-           << " -- node" << getEdgeNode2Id(EId)
-           << " [ label=\"";
-        const Matrix &EdgeCosts = getEdgeCosts(EId);
-        for (unsigned i = 0; i < EdgeCosts.getRows(); ++i) {
-          OS << EdgeCosts.getRowAsVector(i) << "\\n";
-        }
-        OS << "\" ]\n";
-      }
-      OS << "}\n";
-    }
   };
 
 }  // namespace PBQP
diff --git a/include/llvm/CodeGen/PBQP/ReductionRules.h b/include/llvm/CodeGen/PBQP/ReductionRules.h
index 21fde4d8a5cd1..d4a544bfe7214 100644
--- a/include/llvm/CodeGen/PBQP/ReductionRules.h
+++ b/include/llvm/CodeGen/PBQP/ReductionRules.h
@@ -132,9 +132,9 @@ namespace PBQP {
     } else {
       const Matrix &YZECosts = G.getEdgeCosts(YZEId);
       if (YNId == G.getEdgeNode1Id(YZEId)) {
-        G.setEdgeCosts(YZEId, Delta + YZECosts);
+        G.updateEdgeCosts(YZEId, Delta + YZECosts);
       } else {
-        G.setEdgeCosts(YZEId, Delta.transpose() + YZECosts);
+        G.updateEdgeCosts(YZEId, Delta.transpose() + YZECosts);
       }
     }
 
@@ -144,6 +144,25 @@ namespace PBQP {
     // TODO: Try to normalize newly added/modified edge.
   }
 
+#ifndef NDEBUG
+  // Does this Cost vector have any register options ?
+  template <typename VectorT>
+  bool hasRegisterOptions(const VectorT &V) {
+    unsigned VL = V.getLength();
+
+    // An empty or spill only cost vector does not provide any register option.
+    if (VL <= 1)
+      return false;
+
+    // If there are registers in the cost vector, but all of them have infinite
+    // costs, then ... there is no available register.
+    for (unsigned i = 1; i < VL; ++i)
+      if (V[i] != std::numeric_limits<PBQP::PBQPNum>::infinity())
+        return true;
+
+    return false;
+  }
+#endif
 
   // \brief Find a solution to a fully reduced graph by backpropagation.
   //
@@ -170,6 +189,15 @@ namespace PBQP {
 
       RawVector v = G.getNodeCosts(NId);
 
+#ifndef NDEBUG
+      // Although a conservatively allocatable node can be allocated to a register,
+      // spilling it may provide a lower cost solution. Assert here that spilling
+      // is done by choice, not because there were no register available.
+      if (G.getNodeMetadata(NId).wasConservativelyAllocatable())
+        assert(hasRegisterOptions(v) && "A conservatively allocatable node "
+                                        "must have available register options");
+#endif
+
       for (auto EId : G.adjEdgeIds(NId)) {
         const Matrix& edgeCosts = G.getEdgeCosts(EId);
         if (NId == G.getEdgeNode1Id(EId)) {
diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h
index 8ed32b8a8dd55..39f69549d7cf3 100644
--- a/include/llvm/CodeGen/Passes.h
+++ b/include/llvm/CodeGen/Passes.h
@@ -120,12 +120,15 @@ protected:
   /// Default setting for -enable-tail-merge on this target.
   bool EnableTailMerge;
 
+  /// Default setting for -enable-shrink-wrap on this target.
+  bool EnableShrinkWrap;
+
 public:
   TargetPassConfig(TargetMachine *tm, PassManagerBase &pm);
   // Dummy constructor.
   TargetPassConfig();
 
-  virtual ~TargetPassConfig();
+  ~TargetPassConfig() override;
 
   static char ID;
 
@@ -179,6 +182,9 @@ public:
   /// Return true if the optimized regalloc pipeline is enabled.
   bool getOptimizeRegAlloc() const;
 
+  /// Return true if shrink wrapping is enabled.
+  bool getEnableShrinkWrap() const;
+
   /// Return true if the default global register allocator is in use and
   /// has not be overriden on the command line with '-regalloc=...'
   bool usingDefaultRegAlloc() const;
@@ -214,7 +220,7 @@ public:
   ///
   /// This can also be used to plug a new MachineSchedStrategy into an instance
   /// of the standard ScheduleDAGMI:
-  ///   return new ScheduleDAGMI(C, new MyStrategy(C))
+  ///   return new ScheduleDAGMI(C, make_unique<MyStrategy>(C), /* IsPostRA= */false)
   ///
   /// Return NULL to select the default (generic) machine scheduler.
   virtual ScheduleDAGInstrs *
@@ -354,13 +360,6 @@ protected:
 namespace llvm {
   FunctionPass *createAtomicExpandPass(const TargetMachine *TM);
 
-  /// \brief Create a basic TargetTransformInfo analysis pass.
-  ///
-  /// This pass implements the target transform info analysis using the target
-  /// independent information available to the LLVM code generator.
-  ImmutablePass *
-  createBasicTargetTransformInfoPass(const TargetMachine *TM);
-
   /// createUnreachableBlockEliminationPass - The LLVM code generator does not
   /// work well with unreachable basic blocks (what live ranges make sense for a
   /// block that cannot be reached?).  As such, a code generator should either
@@ -433,6 +432,10 @@ namespace llvm {
   /// basic blocks.
   extern char &SpillPlacementID;
 
+  /// ShrinkWrap pass. Look for the best place to insert save and restore
+  // instruction and update the MachineFunctionInfo with that information.
+  extern char &ShrinkWrapID;
+
   /// VirtRegRewriter pass. Rewrite virtual registers to physical registers as
   /// assigned in VirtRegMap.
   extern char &VirtRegRewriterID;
@@ -517,11 +520,15 @@ namespace llvm {
   /// information.
   extern char &MachineBlockPlacementStatsID;
 
-  /// GCLowering Pass - Performs target-independent LLVM IR transformations for
-  /// highly portable strategies.
-  ///
+  /// GCLowering Pass - Used by gc.root to perform its default lowering
+  /// operations.
   FunctionPass *createGCLoweringPass();
 
+  /// ShadowStackGCLowering - Implements the custom lowering mechanism
+  /// used by the shadow stack GC.  Only runs on functions which opt in to
+  /// the shadow stack collector.
+  FunctionPass *createShadowStackGCLoweringPass();
+
   /// GCMachineCodeAnalysis - Target-independent pass to mark safe points
   /// in machine code. Must be added very late during code generation, just
   /// prior to output, and importantly after all CFG transformations (such as
@@ -569,6 +576,10 @@ namespace llvm {
   /// adapted to code generation.  Required if using dwarf exception handling.
   FunctionPass *createDwarfEHPass(const TargetMachine *TM);
 
+  /// createWinEHPass - Prepares personality functions used by MSVC on Windows,
+  /// in addition to the Itanium LSDA based personalities.
+  FunctionPass *createWinEHPass(const TargetMachine *TM);
+
   /// createSjLjEHPreparePass - This pass adapts exception handling code to use
   /// the GCC-style builtin setjmp/longjmp (sjlj) to handling EH control flow.
   ///
@@ -611,13 +622,13 @@ namespace llvm {
   ModulePass *createForwardControlFlowIntegrityPass();
 } // End llvm namespace
 
-/// This initializer registers TargetMachine constructor, so the pass being
-/// initialized can use target dependent interfaces. Please do not move this
-/// macro to be together with INITIALIZE_PASS, which is a complete target
-/// independent initializer, and we don't want to make libScalarOpts depend
-/// on libCodeGen.
-#define INITIALIZE_TM_PASS(passName, arg, name, cfg, analysis) \
-  static void* initialize##passName##PassOnce(PassRegistry &Registry) { \
+/// Target machine pass initializer for passes with dependencies. Use with
+/// INITIALIZE_TM_PASS_END.
+#define INITIALIZE_TM_PASS_BEGIN INITIALIZE_PASS_BEGIN
+
+/// Target machine pass initializer for passes with dependencies. Use with
+/// INITIALIZE_TM_PASS_BEGIN.
+#define INITIALIZE_TM_PASS_END(passName, arg, name, cfg, analysis) \
     PassInfo *PI = new PassInfo(name, arg, & passName ::ID, \
       PassInfo::NormalCtor_t(callDefaultCtor< passName >), cfg, analysis, \
       PassInfo::TargetMachineCtor_t(callTargetMachineCtor< passName >)); \
@@ -628,4 +639,13 @@ namespace llvm {
     CALL_ONCE_INITIALIZATION(initialize##passName##PassOnce) \
   }
 
+/// This initializer registers TargetMachine constructor, so the pass being
+/// initialized can use target dependent interfaces. Please do not move this
+/// macro to be together with INITIALIZE_PASS, which is a complete target
+/// independent initializer, and we don't want to make libScalarOpts depend
+/// on libCodeGen.
+#define INITIALIZE_TM_PASS(passName, arg, name, cfg, analysis) \
+    INITIALIZE_TM_PASS_BEGIN(passName, arg, name, cfg, analysis) \
+    INITIALIZE_TM_PASS_END(passName, arg, name, cfg, analysis)
+
 #endif
diff --git a/include/llvm/CodeGen/RegAllocPBQP.h b/include/llvm/CodeGen/RegAllocPBQP.h
index 5c0e9859915f9..6046e46547b20 100644
--- a/include/llvm/CodeGen/RegAllocPBQP.h
+++ b/include/llvm/CodeGen/RegAllocPBQP.h
@@ -23,6 +23,9 @@
 #include "llvm/Support/ErrorHandling.h"
 
 namespace llvm {
+
+class raw_ostream;
+
 namespace PBQP {
 namespace RegAlloc {
 
@@ -177,21 +180,34 @@ class NodeMetadata {
 public:
   typedef RegAlloc::AllowedRegVector AllowedRegVector;
 
-  typedef enum { Unprocessed,
-                 OptimallyReducible,
-                 ConservativelyAllocatable,
-                 NotProvablyAllocatable } ReductionState;
+  // The node's reduction state. The order in this enum is important,
+  // as it is assumed nodes can only progress up (i.e. towards being
+  // optimally reducible) when reducing the graph.
+  typedef enum {
+    Unprocessed,
+    NotProvablyAllocatable,
+    ConservativelyAllocatable,
+    OptimallyReducible
+  } ReductionState;
 
   NodeMetadata()
     : RS(Unprocessed), NumOpts(0), DeniedOpts(0), OptUnsafeEdges(nullptr),
-      VReg(0) {}
+      VReg(0)
+#ifndef NDEBUG
+      , everConservativelyAllocatable(false)
+#endif
+      {}
 
   // FIXME: Re-implementing default behavior to work around MSVC. Remove once
   // MSVC synthesizes move constructors properly.
   NodeMetadata(const NodeMetadata &Other)
     : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts),
       OptUnsafeEdges(new unsigned[NumOpts]), VReg(Other.VReg),
-      AllowedRegs(Other.AllowedRegs) {
+      AllowedRegs(Other.AllowedRegs)
+#ifndef NDEBUG
+      , everConservativelyAllocatable(Other.everConservativelyAllocatable)
+#endif
+  {
     if (NumOpts > 0) {
       std::copy(&Other.OptUnsafeEdges[0], &Other.OptUnsafeEdges[NumOpts],
                 &OptUnsafeEdges[0]);
@@ -203,7 +219,11 @@ public:
   NodeMetadata(NodeMetadata &&Other)
     : RS(Other.RS), NumOpts(Other.NumOpts), DeniedOpts(Other.DeniedOpts),
       OptUnsafeEdges(std::move(Other.OptUnsafeEdges)), VReg(Other.VReg),
-      AllowedRegs(std::move(Other.AllowedRegs)) {}
+      AllowedRegs(std::move(Other.AllowedRegs))
+#ifndef NDEBUG
+      , everConservativelyAllocatable(Other.everConservativelyAllocatable)
+#endif
+  {}
 
   // FIXME: Re-implementing default behavior to work around MSVC. Remove once
   // MSVC synthesizes move constructors properly.
@@ -216,6 +236,9 @@ public:
               OptUnsafeEdges.get());
     VReg = Other.VReg;
     AllowedRegs = Other.AllowedRegs;
+#ifndef NDEBUG
+    everConservativelyAllocatable = Other.everConservativelyAllocatable;
+#endif
     return *this;
   }
 
@@ -228,6 +251,9 @@ public:
     OptUnsafeEdges = std::move(Other.OptUnsafeEdges);
     VReg = Other.VReg;
     AllowedRegs = std::move(Other.AllowedRegs);
+#ifndef NDEBUG
+    everConservativelyAllocatable = Other.everConservativelyAllocatable;
+#endif
     return *this;
   }
 
@@ -245,7 +271,18 @@ public:
   }
 
   ReductionState getReductionState() const { return RS; }
-  void setReductionState(ReductionState RS) { this->RS = RS; }
+  void setReductionState(ReductionState RS) {
+    assert(RS >= this->RS && "A node's reduction state can not be downgraded");
+    this->RS = RS;
+
+#ifndef NDEBUG
+    // Remember this state to assert later that a non-infinite register
+    // option was available.
+    if (RS == ConservativelyAllocatable)
+      everConservativelyAllocatable = true;
+#endif
+  }
+
 
   void handleAddEdge(const MatrixMetadata& MD, bool Transpose) {
     DeniedOpts += Transpose ? MD.getWorstRow() : MD.getWorstCol();
@@ -269,6 +306,12 @@ public:
        &OptUnsafeEdges[NumOpts]);
   }
 
+#ifndef NDEBUG
+  bool wasConservativelyAllocatable() const {
+    return everConservativelyAllocatable;
+  }
+#endif
+
 private:
   ReductionState RS;
   unsigned NumOpts;
@@ -276,6 +319,10 @@ private:
   std::unique_ptr<unsigned[]> OptUnsafeEdges;
   unsigned VReg;
   GraphMetadata::AllowedRegVecRef AllowedRegs;
+
+#ifndef NDEBUG
+  bool everConservativelyAllocatable;
+#endif
 };
 
 class RegAllocSolverImpl {
@@ -309,6 +356,8 @@ public:
   }
 
   void handleAddNode(NodeId NId) {
+    assert(G.getNodeCosts(NId).getLength() > 1 &&
+           "PBQP Graph should not contain single or zero-option nodes");
     G.getNodeMetadata(NId).setup(G.getNodeCosts(NId));
   }
   void handleRemoveNode(NodeId NId) {}
@@ -328,15 +377,7 @@ public:
     NodeMetadata& NMd = G.getNodeMetadata(NId);
     const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata();
     NMd.handleRemoveEdge(MMd, NId == G.getEdgeNode2Id(EId));
-    if (G.getNodeDegree(NId) == 3) {
-      // This node is becoming optimally reducible.
-      moveToOptimallyReducibleNodes(NId);
-    } else if (NMd.getReductionState() ==
-               NodeMetadata::NotProvablyAllocatable &&
-               NMd.isConservativelyAllocatable()) {
-      // This node just became conservatively allocatable.
-      moveToConservativelyAllocatableNodes(NId);
-    }
+    promote(NId, NMd);
   }
 
   void handleReconnectEdge(EdgeId EId, NodeId NId) {
@@ -345,20 +386,44 @@ public:
     NMd.handleAddEdge(MMd, NId == G.getEdgeNode2Id(EId));
   }
 
-  void handleSetEdgeCosts(EdgeId EId, const Matrix& NewCosts) {
-    handleRemoveEdge(EId);
-
+  void handleUpdateCosts(EdgeId EId, const Matrix& NewCosts) {
     NodeId N1Id = G.getEdgeNode1Id(EId);
     NodeId N2Id = G.getEdgeNode2Id(EId);
     NodeMetadata& N1Md = G.getNodeMetadata(N1Id);
     NodeMetadata& N2Md = G.getNodeMetadata(N2Id);
+    bool Transpose = N1Id != G.getEdgeNode1Id(EId);
+
+    // Metadata are computed incrementally. First, update them
+    // by removing the old cost.
+    const MatrixMetadata& OldMMd = G.getEdgeCosts(EId).getMetadata();
+    N1Md.handleRemoveEdge(OldMMd, Transpose);
+    N2Md.handleRemoveEdge(OldMMd, !Transpose);
+
+    // And update now the metadata with the new cost.
     const MatrixMetadata& MMd = NewCosts.getMetadata();
-    N1Md.handleAddEdge(MMd, N1Id != G.getEdgeNode1Id(EId));
-    N2Md.handleAddEdge(MMd, N2Id != G.getEdgeNode1Id(EId));
+    N1Md.handleAddEdge(MMd, Transpose);
+    N2Md.handleAddEdge(MMd, !Transpose);
+
+    // As the metadata may have changed with the update, the nodes may have
+    // become ConservativelyAllocatable or OptimallyReducible.
+    promote(N1Id, N1Md);
+    promote(N2Id, N2Md);
   }
 
 private:
 
+  void promote(NodeId NId, NodeMetadata& NMd) {
+    if (G.getNodeDegree(NId) == 3) {
+      // This node is becoming optimally reducible.
+      moveToOptimallyReducibleNodes(NId);
+    } else if (NMd.getReductionState() ==
+               NodeMetadata::NotProvablyAllocatable &&
+               NMd.isConservativelyAllocatable()) {
+      // This node just became conservatively allocatable.
+      moveToConservativelyAllocatableNodes(NId);
+    }
+  }
+
   void removeFromCurrentSet(NodeId NId) {
     switch (G.getNodeMetadata(NId).getReductionState()) {
     case NodeMetadata::Unprocessed: break;
@@ -479,8 +544,10 @@ private:
   public:
     SpillCostComparator(const Graph& G) : G(G) {}
     bool operator()(NodeId N1Id, NodeId N2Id) {
-      PBQPNum N1SC = G.getNodeCosts(N1Id)[0] / G.getNodeDegree(N1Id);
-      PBQPNum N2SC = G.getNodeCosts(N2Id)[0] / G.getNodeDegree(N2Id);
+      PBQPNum N1SC = G.getNodeCosts(N1Id)[0];
+      PBQPNum N2SC = G.getNodeCosts(N2Id)[0];
+      if (N1SC == N2SC)
+        return G.getNodeDegree(N1Id) < G.getNodeDegree(N2Id);
       return N1SC < N2SC;
     }
   private:
@@ -499,6 +566,17 @@ private:
   typedef PBQP::Graph<RegAllocSolverImpl> BaseT;
 public:
   PBQPRAGraph(GraphMetadata Metadata) : BaseT(Metadata) {}
+
+  /// @brief Dump this graph to dbgs().
+  void dump() const;
+
+  /// @brief Dump this graph to an output stream.
+  /// @param OS Output stream to print on.
+  void dump(raw_ostream &OS) const;
+
+  /// @brief Print a representation of this graph in DOT format.
+  /// @param OS Output stream to print on.
+  void printDot(raw_ostream &OS) const;
 };
 
 inline Solution solve(PBQPRAGraph& G) {
diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h
index cc9e000551464..fcb6feed68ca2 100644
--- a/include/llvm/CodeGen/RegisterPressure.h
+++ b/include/llvm/CodeGen/RegisterPressure.h
@@ -35,21 +35,6 @@ struct RegisterPressure {
   SmallVector<unsigned,8> LiveInRegs;
   SmallVector<unsigned,8> LiveOutRegs;
 
-  /// Increase register pressure for each pressure set impacted by this register
-  /// class. Normally called by RegPressureTracker, but may be called manually
-  /// to account for live through (global liveness).
-  ///
-  /// \param Reg is either a virtual register number or register unit number.
-  void increase(unsigned Reg, const TargetRegisterInfo *TRI,
-                const MachineRegisterInfo *MRI);
-
-  /// Decrease register pressure for each pressure set impacted by this register
-  /// class. This is only useful to account for spilling or rematerialization.
-  ///
-  /// \param Reg is either a virtual register number or register unit number.
-  void decrease(unsigned Reg, const TargetRegisterInfo *TRI,
-                const MachineRegisterInfo *MRI);
-
   void dump(const TargetRegisterInfo *TRI) const;
 };
 
diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h
index 474861e45df1c..df3fd34e0af67 100644
--- a/include/llvm/CodeGen/RegisterScavenging.h
+++ b/include/llvm/CodeGen/RegisterScavenging.h
@@ -9,7 +9,7 @@
 //
 // This file declares the machine register scavenger class. It can provide
 // information such as unused register at any point in a machine basic block.
-// It also provides a mechanism to make registers availbale by evicting them
+// It also provides a mechanism to make registers available by evicting them
 // to spill slots.
 //
 //===----------------------------------------------------------------------===//
@@ -36,8 +36,7 @@ class RegScavenger {
   MachineBasicBlock::iterator MBBI;
   unsigned NumRegUnits;
 
-  /// Tracking - True if RegScavenger is currently tracking the liveness of 
-  /// registers.
+  /// True if RegScavenger is currently tracking the liveness of registers.
   bool Tracking;
 
   /// Information on scavenged registers (held in a spill slot).
@@ -58,8 +57,8 @@ class RegScavenger {
   /// A vector of information on scavenged registers.
   SmallVector<ScavengedInfo, 2> Scavenged;
 
-  /// RegUnitsAvailable - The current state of each reg unit immediatelly
-  /// before MBBI. One bit per register unit. If bit is not set it means any
+  /// The current state of each reg unit immediately before MBBI.
+  /// One bit per register unit. If bit is not set it means any
   /// register containing that register unit is currently being used.
   BitVector RegUnitsAvailable;
 
@@ -72,18 +71,17 @@ public:
   RegScavenger()
     : MBB(nullptr), NumRegUnits(0), Tracking(false) {}
 
-  /// enterBasicBlock - Start tracking liveness from the begin of the specific
-  /// basic block.
+  /// Start tracking liveness from the begin of the specific basic block.
   void enterBasicBlock(MachineBasicBlock *mbb);
 
-  /// initRegState - allow resetting register state info for multiple
+  /// Allow resetting register state info for multiple
   /// passes over/within the same function.
   void initRegState();
 
-  /// forward - Move the internal MBB iterator and update register states.
+  /// Move the internal MBB iterator and update register states.
   void forward();
 
-  /// forward - Move the internal MBB iterator and update register states until
+  /// Move the internal MBB iterator and update register states until
   /// it has processed the specific iterator.
   void forward(MachineBasicBlock::iterator I) {
     if (!Tracking && MBB->begin() != I) forward();
@@ -99,7 +97,7 @@ public:
     while (MBBI != I) unprocess();
   }
 
-  /// skipTo - Move the internal MBB iterator but do not update register states.
+  /// Move the internal MBB iterator but do not update register states.
   void skipTo(MachineBasicBlock::iterator I) {
     if (I == MachineBasicBlock::iterator(nullptr))
       Tracking = false;
@@ -110,14 +108,13 @@ public:
     return MBBI;
   }
   
-  /// isRegUsed - return if a specific register is currently used.
+  /// Return if a specific register is currently used.
   bool isRegUsed(unsigned Reg, bool includeReserved = true) const;
 
-  /// getRegsAvailable - Return all available registers in the register class
-  /// in Mask.
+  /// Return all available registers in the register class in Mask.
   BitVector getRegsAvailable(const TargetRegisterClass *RC);
 
-  /// FindUnusedReg - Find a unused register of the specified register class.
+  /// Find an unused register of the specified register class.
   /// Return 0 if none is found.
   unsigned FindUnusedReg(const TargetRegisterClass *RegClass) const;
 
@@ -144,7 +141,7 @@ public:
         A.push_back(I->FrameIndex);
   }
 
-  /// scavengeRegister - Make a register of the specific register class
+  /// Make a register of the specific register class
   /// available and do the appropriate bookkeeping. SPAdj is the stack
   /// adjustment due to call frame, it's passed along to eliminateFrameIndex().
   /// Returns the scavenged register.
@@ -154,11 +151,10 @@ public:
     return scavengeRegister(RegClass, MBBI, SPAdj);
   }
 
-  /// setRegUsed - Tell the scavenger a register is used.
-  ///
+  /// Tell the scavenger a register is used.
   void setRegUsed(unsigned Reg);
 private:
-  /// isReserved - Returns true if a register is reserved. It is never "unused".
+  /// Returns true if a register is reserved. It is never "unused".
   bool isReserved(unsigned Reg) const { return MRI->isReserved(Reg); }
 
   /// setUsed / setUnused - Mark the state of one or a number of register units.
@@ -177,9 +173,8 @@ private:
   /// Add all Reg Units that Reg contains to BV.
   void addRegUnits(BitVector &BV, unsigned Reg);
   
-  /// findSurvivorReg - Return the candidate register that is unused for the
-  /// longest after StartMI. UseMI is set to the instruction where the search
-  /// stopped.
+  /// Return the candidate register that is unused for the longest after
+  /// StartMI. UseMI is set to the instruction where the search stopped.
   ///
   /// No more than InstrLimit instructions are inspected.
   unsigned findSurvivorReg(MachineBasicBlock::iterator StartMI,
diff --git a/include/llvm/CodeGen/ResourcePriorityQueue.h b/include/llvm/CodeGen/ResourcePriorityQueue.h
index 114fe7c35b827..0097e0472e5c5 100644
--- a/include/llvm/CodeGen/ResourcePriorityQueue.h
+++ b/include/llvm/CodeGen/ResourcePriorityQueue.h
@@ -64,7 +64,7 @@ namespace llvm {
     /// ResourcesModel - Represents VLIW state.
     /// Not limited to VLIW targets per say, but assumes
     /// definition of DFA by a target.
-    DFAPacketizer *ResourcesModel;
+    std::unique_ptr<DFAPacketizer> ResourcesModel;
 
     /// Resource model - packet/bundle model. Purely
     /// internal at the time.
@@ -77,10 +77,6 @@ namespace llvm {
   public:
     ResourcePriorityQueue(SelectionDAGISel *IS);
 
-    ~ResourcePriorityQueue() {
-      delete ResourcesModel;
-    }
-
     bool isBottomUp() const override { return false; }
 
     void initNodes(std::vector<SUnit> &sunits) override;
@@ -124,8 +120,6 @@ namespace llvm {
 
     void remove(SUnit *SU) override;
 
-    void dump(ScheduleDAG* DAG) const override;
-
     /// scheduledNode - Main resource tracking point.
     void scheduledNode(SUnit *Node) override;
     bool isResourceAvailable(SUnit *SU);
diff --git a/include/llvm/CodeGen/RuntimeLibcalls.h b/include/llvm/CodeGen/RuntimeLibcalls.h
index 64c9c4729e922..2be5de640e292 100644
--- a/include/llvm/CodeGen/RuntimeLibcalls.h
+++ b/include/llvm/CodeGen/RuntimeLibcalls.h
@@ -425,6 +425,10 @@ namespace RTLIB {
   /// getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or
   /// UNKNOWN_LIBCALL if there is none.
   Libcall getUINTTOFP(EVT OpVT, EVT RetVT);
+
+  /// Return the SYNC_FETCH_AND_* value for the given opcode and type, or
+  /// UNKNOWN_LIBCALL if there is none.
+  Libcall getATOMIC(unsigned Opc, MVT VT);
 }
 }
 
diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h
index 80aee8c628809..839131416560f 100644
--- a/include/llvm/CodeGen/ScheduleDAG.h
+++ b/include/llvm/CodeGen/ScheduleDAG.h
@@ -628,12 +628,6 @@ namespace llvm {
     }
     bool operator!=(const SUnitIterator& x) const { return !operator==(x); }
 
-    const SUnitIterator &operator=(const SUnitIterator &I) {
-      assert(I.Node==Node && "Cannot assign iterators to two different nodes!");
-      Operand = I.Operand;
-      return *this;
-    }
-
     pointer operator*() const {
       return Node->Preds[Operand].getSUnit();
     }
diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h
index 00dd8f9a633e2..1196783e820b8 100644
--- a/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -158,7 +158,7 @@ namespace llvm {
                                bool RemoveKillFlags = false,
                                LiveIntervals *LIS = nullptr);
 
-    virtual ~ScheduleDAGInstrs() {}
+    ~ScheduleDAGInstrs() override {}
 
     bool isPostRA() const { return IsPostRA; }
 
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index ee4bc08495ec9..89f9005b8b211 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -41,14 +41,16 @@ class TargetSelectionDAGInfo;
 
 class SDVTListNode : public FoldingSetNode {
   friend struct FoldingSetTrait<SDVTListNode>;
-  /// FastID - A reference to an Interned FoldingSetNodeID for this node.
+  /// A reference to an Interned FoldingSetNodeID for this node.
   /// The Allocator in SelectionDAG holds the data.
   /// SDVTList contains all types which are frequently accessed in SelectionDAG.
-  /// The size of this list is not expected big so it won't introduce memory penalty.
+  /// The size of this list is not expected to be big so it won't introduce
+  /// a memory penalty.
   FoldingSetNodeIDRef FastID;
   const EVT *VTs;
   unsigned int NumVTs;
-  /// The hash value for SDVTList is fixed so cache it to avoid hash calculation
+  /// The hash value for SDVTList is fixed, so cache it to avoid
+  /// hash calculation.
   unsigned HashValue;
 public:
   SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) :
@@ -61,8 +63,8 @@ public:
   }
 };
 
-// Specialize FoldingSetTrait for SDVTListNode
-// To avoid computing temp FoldingSetNodeID and hash value.
+/// Specialize FoldingSetTrait for SDVTListNode
+/// to avoid computing temp FoldingSetNodeID and hash value.
 template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> {
   static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) {
     ID = X.FastID;
@@ -98,7 +100,7 @@ private:
   static void createNode(const SDNode &);
 };
 
-/// SDDbgInfo - Keeps track of dbg_value information through SDISel.  We do
+/// Keeps track of dbg_value information through SDISel.  We do
 /// not build SDNodes for these so as not to perturb the generated code;
 /// instead the info is kept off to the side in this structure. Each SDNode may
 /// have one or more associated dbg_value entries. This information is kept in
@@ -110,13 +112,14 @@ private:
 /// motion, and debug info for them is potentially useful even if the parameter
 /// is unused.  Right now only byval parameters are handled separately.
 class SDDbgInfo {
+  BumpPtrAllocator Alloc;
   SmallVector<SDDbgValue*, 32> DbgValues;
   SmallVector<SDDbgValue*, 32> ByvalParmDbgValues;
   typedef DenseMap<const SDNode*, SmallVector<SDDbgValue*, 2> > DbgValMapType;
   DbgValMapType DbgValMap;
 
-  void operator=(const SDDbgInfo&) LLVM_DELETED_FUNCTION;
-  SDDbgInfo(const SDDbgInfo&) LLVM_DELETED_FUNCTION;
+  void operator=(const SDDbgInfo&) = delete;
+  SDDbgInfo(const SDDbgInfo&) = delete;
 public:
   SDDbgInfo() {}
 
@@ -136,8 +139,11 @@ public:
     DbgValMap.clear();
     DbgValues.clear();
     ByvalParmDbgValues.clear();
+    Alloc.Reset();
   }
 
+  BumpPtrAllocator &getAlloc() { return Alloc; }
+
   bool empty() const {
     return DbgValues.empty() && ByvalParmDbgValues.empty();
   }
@@ -159,10 +165,10 @@ public:
 class SelectionDAG;
 void checkForCycles(const SelectionDAG *DAG, bool force = false);
 
-/// SelectionDAG class - This is used to represent a portion of an LLVM function
-/// in a low-level Data Dependence DAG representation suitable for instruction
-/// selection.  This DAG is constructed as the first step of instruction
-/// selection in order to allow implementation of machine specific optimizations
+/// This is used to represent a portion of an LLVM function in a low-level
+/// Data Dependence DAG representation suitable for instruction selection.
+/// This DAG is constructed as the first step of instruction selection in order
+/// to allow implementation of machine specific optimizations
 /// and code simplifications.
 ///
 /// The representation used by the SelectionDAG is a target-independent
@@ -178,40 +184,39 @@ class SelectionDAG {
   LLVMContext *Context;
   CodeGenOpt::Level OptLevel;
 
-  /// EntryNode - The starting token.
+  /// The starting token.
   SDNode EntryNode;
 
-  /// Root - The root of the entire DAG.
+  /// The root of the entire DAG.
   SDValue Root;
 
-  /// AllNodes - A linked list of nodes in the current DAG.
+  /// A linked list of nodes in the current DAG.
   ilist<SDNode> AllNodes;
 
-  /// NodeAllocatorType - The AllocatorType for allocating SDNodes. We use
+  /// The AllocatorType for allocating SDNodes. We use
   /// pool allocation with recycling.
   typedef RecyclingAllocator<BumpPtrAllocator, SDNode, sizeof(LargestSDNode),
                              AlignOf<MostAlignedSDNode>::Alignment>
     NodeAllocatorType;
 
-  /// NodeAllocator - Pool allocation for nodes.
+  /// Pool allocation for nodes.
   NodeAllocatorType NodeAllocator;
 
-  /// CSEMap - This structure is used to memoize nodes, automatically performing
+  /// This structure is used to memoize nodes, automatically performing
   /// CSE with existing nodes when a duplicate is requested.
   FoldingSet<SDNode> CSEMap;
 
-  /// OperandAllocator - Pool allocation for machine-opcode SDNode operands.
+  /// Pool allocation for machine-opcode SDNode operands.
   BumpPtrAllocator OperandAllocator;
 
-  /// Allocator - Pool allocation for misc. objects that are created once per
-  /// SelectionDAG.
+  /// Pool allocation for misc. objects that are created once per SelectionDAG.
   BumpPtrAllocator Allocator;
 
-  /// DbgInfo - Tracks dbg_value information through SDISel.
+  /// Tracks dbg_value information through SDISel.
   SDDbgInfo *DbgInfo;
 
 public:
-  /// DAGUpdateListener - Clients of various APIs that cause global effects on
+  /// Clients of various APIs that cause global effects on
   /// the DAG can optionally implement this interface.  This allows the clients
   /// to handle the various sorts of updates that happen.
   ///
@@ -232,15 +237,15 @@ public:
       DAG.UpdateListeners = Next;
     }
 
-    /// NodeDeleted - The node N that was deleted and, if E is not null, an
+    /// The node N that was deleted and, if E is not null, an
     /// equivalent node E that replaced it.
     virtual void NodeDeleted(SDNode *N, SDNode *E);
 
-    /// NodeUpdated - The node N that was updated.
+    /// The node N that was updated.
     virtual void NodeUpdated(SDNode *N);
   };
 
-  /// NewNodesMustHaveLegalTypes - When true, additional steps are taken to
+  /// When true, additional steps are taken to
   /// ensure that getConstant() and similar functions return DAG nodes that
   /// have legal types. This is important after type legalization since
   /// any illegally typed nodes generated after this point will not experience
@@ -251,32 +256,28 @@ private:
   /// DAGUpdateListener is a friend so it can manipulate the listener stack.
   friend struct DAGUpdateListener;
 
-  /// UpdateListeners - Linked list of registered DAGUpdateListener instances.
+  /// Linked list of registered DAGUpdateListener instances.
   /// This stack is maintained by DAGUpdateListener RAII.
   DAGUpdateListener *UpdateListeners;
 
-  /// setGraphColorHelper - Implementation of setSubgraphColor.
+  /// Implementation of setSubgraphColor.
   /// Return whether we had to truncate the search.
-  ///
   bool setSubgraphColorHelper(SDNode *N, const char *Color,
                               DenseSet<SDNode *> &visited,
                               int level, bool &printed);
 
-  void operator=(const SelectionDAG&) LLVM_DELETED_FUNCTION;
-  SelectionDAG(const SelectionDAG&) LLVM_DELETED_FUNCTION;
+  void operator=(const SelectionDAG&) = delete;
+  SelectionDAG(const SelectionDAG&) = delete;
 
 public:
   explicit SelectionDAG(const TargetMachine &TM, llvm::CodeGenOpt::Level);
   ~SelectionDAG();
 
-  /// init - Prepare this SelectionDAG to process code in the given
-  /// MachineFunction.
-  ///
+  /// Prepare this SelectionDAG to process code in the given MachineFunction.
   void init(MachineFunction &mf);
 
-  /// clear - Clear state and free memory necessary to make this
+  /// Clear state and free memory necessary to make this
   /// SelectionDAG ready to process a new block.
-  ///
   void clear();
 
   MachineFunction &getMachineFunction() const { return *MF; }
@@ -286,8 +287,7 @@ public:
   const TargetSelectionDAGInfo &getSelectionDAGInfo() const { return *TSI; }
   LLVMContext *getContext() const {return Context; }
 
-  /// viewGraph - Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
-  ///
+  /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
   void viewGraph(const std::string &Title);
   void viewGraph();
 
@@ -295,24 +295,21 @@ public:
   std::map<const SDNode *, std::string> NodeGraphAttrs;
 #endif
 
-  /// clearGraphAttrs - Clear all previously defined node graph attributes.
+  /// Clear all previously defined node graph attributes.
   /// Intended to be used from a debugging tool (eg. gdb).
   void clearGraphAttrs();
 
-  /// setGraphAttrs - Set graph attributes for a node. (eg. "color=red".)
-  ///
+  /// Set graph attributes for a node. (eg. "color=red".)
   void setGraphAttrs(const SDNode *N, const char *Attrs);
 
-  /// getGraphAttrs - Get graph attributes for a node. (eg. "color=red".)
+  /// Get graph attributes for a node. (eg. "color=red".)
   /// Used from getNodeAttributes.
   const std::string getGraphAttrs(const SDNode *N) const;
 
-  /// setGraphColor - Convenience for setting node color attribute.
-  ///
+  /// Convenience for setting node color attribute.
   void setGraphColor(const SDNode *N, const char *Color);
 
-  /// setGraphColor - Convenience for setting subgraph color attribute.
-  ///
+  /// Convenience for setting subgraph color attribute.
   void setSubgraphColor(SDNode *N, const char *Color);
 
   typedef ilist<SDNode>::const_iterator allnodes_const_iterator;
@@ -325,17 +322,15 @@ public:
     return AllNodes.size();
   }
 
-  /// getRoot - Return the root tag of the SelectionDAG.
-  ///
+  /// Return the root tag of the SelectionDAG.
   const SDValue &getRoot() const { return Root; }
 
-  /// getEntryNode - Return the token chain corresponding to the entry of the
-  /// function.
+  /// Return the token chain corresponding to the entry of the function.
   SDValue getEntryNode() const {
     return SDValue(const_cast<SDNode *>(&EntryNode), 0);
   }
 
-  /// setRoot - Set the current root tag of the SelectionDAG.
+  /// Set the current root tag of the SelectionDAG.
   ///
   const SDValue &setRoot(SDValue N) {
     assert((!N.getNode() || N.getValueType() == MVT::Other) &&
@@ -348,22 +343,22 @@ public:
     return Root;
   }
 
-  /// Combine - This iterates over the nodes in the SelectionDAG, folding
+  /// This iterates over the nodes in the SelectionDAG, folding
   /// certain types of nodes together, or eliminating superfluous nodes.  The
   /// Level argument controls whether Combine is allowed to produce nodes and
   /// types that are illegal on the target.
   void Combine(CombineLevel Level, AliasAnalysis &AA,
                CodeGenOpt::Level OptLevel);
 
-  /// LegalizeTypes - This transforms the SelectionDAG into a SelectionDAG that
-  /// only uses types natively supported by the target.  Returns "true" if it
-  /// made any changes.
+  /// This transforms the SelectionDAG into a SelectionDAG that
+  /// only uses types natively supported by the target.
+  /// Returns "true" if it made any changes.
   ///
   /// Note that this is an involved process that may invalidate pointers into
   /// the graph.
   bool LegalizeTypes();
 
-  /// Legalize - This transforms the SelectionDAG into a SelectionDAG that is
+  /// This transforms the SelectionDAG into a SelectionDAG that is
   /// compatible with the target instruction selector, as indicated by the
   /// TargetLowering object.
   ///
@@ -392,7 +387,7 @@ public:
   /// UpdatedNodes with any new nodes replacing those originally in the DAG.
   bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes);
 
-  /// LegalizeVectors - This transforms the SelectionDAG into a SelectionDAG
+  /// This transforms the SelectionDAG into a SelectionDAG
   /// that only uses vector math operations supported by the target.  This is
   /// necessary as a separate step from Legalize because unrolling a vector
   /// operation can introduce illegal types, which requires running
@@ -405,16 +400,14 @@ public:
   /// the graph.
   bool LegalizeVectors();
 
-  /// RemoveDeadNodes - This method deletes all unreachable nodes in the
-  /// SelectionDAG.
+  /// This method deletes all unreachable nodes in the SelectionDAG.
   void RemoveDeadNodes();
 
-  /// DeleteNode - Remove the specified node from the system.  This node must
+  /// Remove the specified node from the system.  This node must
   /// have no referrers.
   void DeleteNode(SDNode *N);
 
-  /// getVTList - Return an SDVTList that represents the list of values
-  /// specified.
+  /// Return an SDVTList that represents the list of values specified.
   SDVTList getVTList(EVT VT);
   SDVTList getVTList(EVT VT1, EVT VT2);
   SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3);
@@ -424,36 +417,40 @@ public:
   //===--------------------------------------------------------------------===//
   // Node creation methods.
   //
-  SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false,
+  SDValue getConstant(uint64_t Val, SDLoc DL, EVT VT, bool isTarget = false,
                       bool isOpaque = false);
-  SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false,
+  SDValue getConstant(const APInt &Val, SDLoc DL, EVT VT, bool isTarget = false,
                       bool isOpaque = false);
-  SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false,
-                      bool isOpaque = false);
-  SDValue getIntPtrConstant(uint64_t Val, bool isTarget = false);
-  SDValue getTargetConstant(uint64_t Val, EVT VT, bool isOpaque = false) {
-    return getConstant(Val, VT, true, isOpaque);
+  SDValue getConstant(const ConstantInt &Val, SDLoc DL, EVT VT,
+                      bool isTarget = false, bool isOpaque = false);
+  SDValue getIntPtrConstant(uint64_t Val, SDLoc DL, bool isTarget = false);
+  SDValue getTargetConstant(uint64_t Val, SDLoc DL, EVT VT,
+                            bool isOpaque = false) {
+    return getConstant(Val, DL, VT, true, isOpaque);
   }
-  SDValue getTargetConstant(const APInt &Val, EVT VT, bool isOpaque = false) {
-    return getConstant(Val, VT, true, isOpaque);
+  SDValue getTargetConstant(const APInt &Val, SDLoc DL, EVT VT,
+                            bool isOpaque = false) {
+    return getConstant(Val, DL, VT, true, isOpaque);
   }
-  SDValue getTargetConstant(const ConstantInt &Val, EVT VT,
+  SDValue getTargetConstant(const ConstantInt &Val, SDLoc DL, EVT VT,
                             bool isOpaque = false) {
-    return getConstant(Val, VT, true, isOpaque);
+    return getConstant(Val, DL, VT, true, isOpaque);
   }
   // The forms below that take a double should only be used for simple
   // constants that can be exactly represented in VT.  No checks are made.
-  SDValue getConstantFP(double Val, EVT VT, bool isTarget = false);
-  SDValue getConstantFP(const APFloat& Val, EVT VT, bool isTarget = false);
-  SDValue getConstantFP(const ConstantFP &CF, EVT VT, bool isTarget = false);
-  SDValue getTargetConstantFP(double Val, EVT VT) {
-    return getConstantFP(Val, VT, true);
+  SDValue getConstantFP(double Val, SDLoc DL, EVT VT, bool isTarget = false);
+  SDValue getConstantFP(const APFloat& Val, SDLoc DL, EVT VT,
+                        bool isTarget = false);
+  SDValue getConstantFP(const ConstantFP &CF, SDLoc DL, EVT VT,
+                        bool isTarget = false);
+  SDValue getTargetConstantFP(double Val, SDLoc DL, EVT VT) {
+    return getConstantFP(Val, DL, VT, true);
   }
-  SDValue getTargetConstantFP(const APFloat& Val, EVT VT) {
-    return getConstantFP(Val, VT, true);
+  SDValue getTargetConstantFP(const APFloat& Val, SDLoc DL, EVT VT) {
+    return getConstantFP(Val, DL, VT, true);
   }
-  SDValue getTargetConstantFP(const ConstantFP &Val, EVT VT) {
-    return getConstantFP(Val, VT, true);
+  SDValue getTargetConstantFP(const ConstantFP &Val, SDLoc DL, EVT VT) {
+    return getConstantFP(Val, DL, VT, true);
   }
   SDValue getGlobalAddress(const GlobalValue *GV, SDLoc DL, EVT VT,
                            int64_t offset = 0, bool isTargetGA = false,
@@ -561,10 +558,9 @@ public:
                            SDValue STy,
                            SDValue Rnd, SDValue Sat, ISD::CvtCode Code);
 
-  /// getVectorShuffle - Return an ISD::VECTOR_SHUFFLE node.  The number of
-  /// elements in VT, which must be a vector type, must match the number of
-  /// mask elements NumElts.  A integer mask element equal to -1 is treated as
-  /// undefined.
+  /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT,
+  /// which must be a vector type, must match the number of mask elements
+  /// NumElts. An integer mask element equal to -1 is treated as undefined.
   SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2,
                            const int *MaskElts);
   SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2,
@@ -580,63 +576,62 @@ public:
   /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3>
   SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV);
 
-  /// getAnyExtOrTrunc - Convert Op, which must be of integer type, to the
+  /// Convert Op, which must be of integer type, to the
   /// integer type VT, by either any-extending or truncating it.
   SDValue getAnyExtOrTrunc(SDValue Op, SDLoc DL, EVT VT);
 
-  /// getSExtOrTrunc - Convert Op, which must be of integer type, to the
+  /// Convert Op, which must be of integer type, to the
   /// integer type VT, by either sign-extending or truncating it.
   SDValue getSExtOrTrunc(SDValue Op, SDLoc DL, EVT VT);
 
-  /// getZExtOrTrunc - Convert Op, which must be of integer type, to the
+  /// Convert Op, which must be of integer type, to the
   /// integer type VT, by either zero-extending or truncating it.
   SDValue getZExtOrTrunc(SDValue Op, SDLoc DL, EVT VT);
 
-  /// getZeroExtendInReg - Return the expression required to zero extend the Op
+  /// Return the expression required to zero extend the Op
   /// value assuming it was the smaller SrcTy value.
   SDValue getZeroExtendInReg(SDValue Op, SDLoc DL, EVT SrcTy);
 
-  /// getAnyExtendVectorInReg - Return an operation which will any-extend the
-  /// low lanes of the operand into the specified vector type. For example,
+  /// Return an operation which will any-extend the low lanes of the operand
+  /// into the specified vector type. For example,
   /// this can convert a v16i8 into a v4i32 by any-extending the low four
   /// lanes of the operand from i8 to i32.
   SDValue getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT);
 
-  /// getSignExtendVectorInReg - Return an operation which will sign extend the
-  /// low lanes of the operand into the specified vector type. For example,
+  /// Return an operation which will sign extend the low lanes of the operand
+  /// into the specified vector type. For example,
   /// this can convert a v16i8 into a v4i32 by sign extending the low four
   /// lanes of the operand from i8 to i32.
   SDValue getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT);
 
-  /// getZeroExtendVectorInReg - Return an operation which will zero extend the
-  /// low lanes of the operand into the specified vector type. For example,
+  /// Return an operation which will zero extend the low lanes of the operand
+  /// into the specified vector type. For example,
   /// this can convert a v16i8 into a v4i32 by zero extending the low four
   /// lanes of the operand from i8 to i32.
   SDValue getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT);
 
-  /// getBoolExtOrTrunc - Convert Op, which must be of integer type, to the
-  /// integer type VT, by using an extension appropriate for the target's
+  /// Convert Op, which must be of integer type, to the integer type VT,
+  /// by using an extension appropriate for the target's
   /// BooleanContent for type OpVT or truncating it.
   SDValue getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT, EVT OpVT);
 
-  /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
+  /// Create a bitwise NOT operation as (XOR Val, -1).
   SDValue getNOT(SDLoc DL, SDValue Val, EVT VT);
 
   /// \brief Create a logical NOT operation as (XOR Val, BooleanOne).
   SDValue getLogicalNOT(SDLoc DL, SDValue Val, EVT VT);
 
-  /// getCALLSEQ_START - Return a new CALLSEQ_START node, which always must have
-  /// a glue result (to ensure it's not CSE'd).  CALLSEQ_START does not have a
-  /// useful SDLoc.
+  /// Return a new CALLSEQ_START node, which always must have a glue result
+  /// (to ensure it's not CSE'd).  CALLSEQ_START does not have a useful SDLoc.
   SDValue getCALLSEQ_START(SDValue Chain, SDValue Op, SDLoc DL) {
     SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
     SDValue Ops[] = { Chain,  Op };
     return getNode(ISD::CALLSEQ_START, DL, VTs, Ops);
   }
 
-  /// getCALLSEQ_END - Return a new CALLSEQ_END node, which always must have a
-  /// glue result (to ensure it's not CSE'd).  CALLSEQ_END does not have
-  /// a useful SDLoc.
+  /// Return a new CALLSEQ_END node, which always must have a
+  /// glue result (to ensure it's not CSE'd).
+  /// CALLSEQ_END does not have a useful SDLoc.
   SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2,
                            SDValue InGlue, SDLoc DL) {
     SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue);
@@ -649,19 +644,28 @@ public:
     return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops);
   }
 
-  /// getUNDEF - Return an UNDEF node.  UNDEF does not have a useful SDLoc.
+  /// Return an UNDEF node. UNDEF does not have a useful SDLoc.
   SDValue getUNDEF(EVT VT) {
     return getNode(ISD::UNDEF, SDLoc(), VT);
   }
 
-  /// getGLOBAL_OFFSET_TABLE - Return a GLOBAL_OFFSET_TABLE node.  This does
-  /// not have a useful SDLoc.
+  /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
   SDValue getGLOBAL_OFFSET_TABLE(EVT VT) {
     return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT);
   }
 
-  /// getNode - Gets or creates the specified node.
+  /// Gets or creates the specified node.
   ///
+  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
+                  ArrayRef<SDUse> Ops);
+  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
+                  ArrayRef<SDValue> Ops);
+  SDValue getNode(unsigned Opcode, SDLoc DL, ArrayRef<EVT> ResultTys,
+                  ArrayRef<SDValue> Ops);
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
+                  ArrayRef<SDValue> Ops);
+
+  // Specialize based on number of operands.
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
@@ -672,47 +676,40 @@ public:
                   SDValue N3, SDValue N4);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
                   SDValue N3, SDValue N4, SDValue N5);
-  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef<SDUse> Ops);
-  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
-                  ArrayRef<SDValue> Ops);
-  SDValue getNode(unsigned Opcode, SDLoc DL,
-                  ArrayRef<EVT> ResultTys,
-                  ArrayRef<SDValue> Ops);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  ArrayRef<SDValue> Ops);
+  
+  // Specialize again based on number of operands for nodes with a VTList
+  // rather than a single VT.
   SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs);
   SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  SDValue N1, SDValue N2);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  SDValue N1, SDValue N2, SDValue N3);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  SDValue N1, SDValue N2, SDValue N3, SDValue N4);
-  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  SDValue N1, SDValue N2, SDValue N3, SDValue N4,
-                  SDValue N5);
-
-  /// getStackArgumentTokenFactor - Compute a TokenFactor to force all
-  /// the incoming stack arguments to be loaded from the stack. This is
-  /// used in tail call lowering to protect stack arguments from being
-  /// clobbered.
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
+                  SDValue N2);
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
+                  SDValue N2, SDValue N3);
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
+                  SDValue N2, SDValue N3, SDValue N4);
+  SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
+                  SDValue N2, SDValue N3, SDValue N4, SDValue N5);
+
+  /// Compute a TokenFactor to force all the incoming stack arguments to be
+  /// loaded from the stack. This is used in tail call lowering to protect
+  /// stack arguments from being clobbered.
   SDValue getStackArgumentTokenFactor(SDValue Chain);
 
   SDValue getMemcpy(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src,
                     SDValue Size, unsigned Align, bool isVol, bool AlwaysInline,
-                    MachinePointerInfo DstPtrInfo,
+                    bool isTailCall, MachinePointerInfo DstPtrInfo,
                     MachinePointerInfo SrcPtrInfo);
 
   SDValue getMemmove(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src,
-                     SDValue Size, unsigned Align, bool isVol,
+                     SDValue Size, unsigned Align, bool isVol, bool isTailCall,
                      MachinePointerInfo DstPtrInfo,
                      MachinePointerInfo SrcPtrInfo);
 
   SDValue getMemset(SDValue Chain, SDLoc dl, SDValue Dst, SDValue Src,
-                    SDValue Size, unsigned Align, bool isVol,
+                    SDValue Size, unsigned Align, bool isVol, bool isTailCall,
                     MachinePointerInfo DstPtrInfo);
 
-  /// getSetCC - Helper function to make it easier to build SetCC's if you just
+  /// Helper function to make it easier to build SetCC's if you just
   /// have an ISD::CondCode instead of an SDValue.
   ///
   SDValue getSetCC(SDLoc DL, EVT VT, SDValue LHS, SDValue RHS,
@@ -726,8 +723,8 @@ public:
     return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond));
   }
 
-  // getSelect - Helper function to make it easier to build Select's if you just
-  // have operands and don't want to check for vector.
+  /// Helper function to make it easier to build Select's if you just
+  /// have operands and don't want to check for vector.
   SDValue getSelect(SDLoc DL, EVT VT, SDValue Cond,
                     SDValue LHS, SDValue RHS) {
     assert(LHS.getValueType() == RHS.getValueType() &&
@@ -738,7 +735,7 @@ public:
                    Cond, LHS, RHS);
   }
 
-  /// getSelectCC - Helper function to make it easier to build SelectCC's if you
+  /// Helper function to make it easier to build SelectCC's if you
   /// just have an ISD::CondCode instead of an SDValue.
   ///
   SDValue getSelectCC(SDLoc DL, SDValue LHS, SDValue RHS,
@@ -747,12 +744,12 @@ public:
                    LHS, RHS, True, False, getCondCode(Cond));
   }
 
-  /// getVAArg - VAArg produces a result and token chain, and takes a pointer
+  /// VAArg produces a result and token chain, and takes a pointer
   /// and a source value as input.
   SDValue getVAArg(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr,
                    SDValue SV, unsigned Align);
 
-  /// getAtomicCmpSwap - Gets a node for an atomic cmpxchg op. There are two
+  /// Gets a node for an atomic cmpxchg op. There are two
   /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a
   /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded,
   /// a success flag (initially i1), and a chain.
@@ -769,7 +766,7 @@ public:
                            AtomicOrdering FailureOrdering,
                            SynchronizationScope SynchScope);
 
-  /// getAtomic - Gets a node for an atomic op, produces result (if relevant)
+  /// Gets a node for an atomic op, produces result (if relevant)
   /// and chain and takes 2 operands.
   SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
                     SDValue Ptr, SDValue Val, const Value *PtrVal,
@@ -780,15 +777,15 @@ public:
                     AtomicOrdering Ordering,
                     SynchronizationScope SynchScope);
 
-  /// getAtomic - Gets a node for an atomic op, produces result and chain and
+  /// Gets a node for an atomic op, produces result and chain and
   /// takes 1 operand.
   SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, EVT VT,
                     SDValue Chain, SDValue Ptr, MachineMemOperand *MMO,
                     AtomicOrdering Ordering,
                     SynchronizationScope SynchScope);
 
-  /// getAtomic - Gets a node for an atomic op, produces result and chain and
-  /// takes N operands.
+  /// Gets a node for an atomic op, produces result and chain and takes N
+  /// operands.
   SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTList,
                     ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
                     AtomicOrdering SuccessOrdering,
@@ -798,7 +795,7 @@ public:
                     ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
                     AtomicOrdering Ordering, SynchronizationScope SynchScope);
 
-  /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a
+  /// Creates a MemIntrinsicNode that may produce a
   /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
   /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
   /// less than FIRST_TARGET_MEMORY_OPCODE.
@@ -813,10 +810,10 @@ public:
                               ArrayRef<SDValue> Ops,
                               EVT MemVT, MachineMemOperand *MMO);
 
-  /// getMergeValues - Create a MERGE_VALUES node from the given operands.
+  /// Create a MERGE_VALUES node from the given operands.
   SDValue getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl);
 
-  /// getLoad - Loads are not normal binary operators: their result type is not
+  /// Loads are not normal binary operators: their result type is not
   /// determined by their operands, and they produce a value AND a token chain.
   ///
   SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr,
@@ -848,8 +845,7 @@ public:
                   SDValue Chain, SDValue Ptr, SDValue Offset,
                   EVT MemVT, MachineMemOperand *MMO);
 
-  /// getStore - Helper function to build ISD::STORE nodes.
-  ///
+  /// Helper function to build ISD::STORE nodes.
   SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr,
                    MachinePointerInfo PtrInfo, bool isVolatile,
                    bool isNonTemporal, unsigned Alignment,
@@ -872,21 +868,25 @@ public:
   SDValue getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
                          SDValue Ptr, SDValue Mask, EVT MemVT,
                          MachineMemOperand *MMO, bool IsTrunc);
-  /// getSrcValue - Construct a node to track a Value* through the backend.
+  SDValue getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
+                          ArrayRef<SDValue> Ops, MachineMemOperand *MMO);
+  SDValue getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
+                           ArrayRef<SDValue> Ops, MachineMemOperand *MMO);
+  /// Construct a node to track a Value* through the backend.
   SDValue getSrcValue(const Value *v);
 
-  /// getMDNode - Return an MDNodeSDNode which holds an MDNode.
+  /// Return an MDNodeSDNode which holds an MDNode.
   SDValue getMDNode(const MDNode *MD);
 
-  /// getAddrSpaceCast - Return an AddrSpaceCastSDNode.
+  /// Return an AddrSpaceCastSDNode.
   SDValue getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
                            unsigned SrcAS, unsigned DestAS);
 
-  /// getShiftAmountOperand - Return the specified value casted to
+  /// Return the specified value casted to
   /// the target's desired shift amount type.
   SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op);
 
-  /// UpdateNodeOperands - *Mutate* the specified node in-place to have the
+  /// *Mutate* the specified node in-place to have the
   /// specified operands.  If the resultant node already exists in the DAG,
   /// this does not modify the specified node, instead it returns the node that
   /// already exists.  If the resultant node does not exist in the DAG, the
@@ -902,7 +902,7 @@ public:
                                SDValue Op3, SDValue Op4, SDValue Op5);
   SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops);
 
-  /// SelectNodeTo - These are used for target selectors to *mutate* the
+  /// These are used for target selectors to *mutate* the
   /// specified node to have the specified return type, Target opcode, and
   /// operands.  Note that target opcodes are stored as
   /// ~TargetOpcode in the node opcode field.  The resultant node is returned.
@@ -932,12 +932,12 @@ public:
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, SDVTList VTs,
                        ArrayRef<SDValue> Ops);
 
-  /// MorphNodeTo - This *mutates* the specified node to have the specified
+  /// This *mutates* the specified node to have the specified
   /// return type, opcode, and operands.
   SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs,
                       ArrayRef<SDValue> Ops);
 
-  /// getMachineNode - These are used for target selectors to create a new node
+  /// These are used for target selectors to create a new node
   /// with specified return type(s), MachineInstr opcode, and operands.
   ///
   /// Note that getMachineNode returns the resultant node.  If there is already
@@ -976,25 +976,20 @@ public:
   MachineSDNode *getMachineNode(unsigned Opcode, SDLoc dl, SDVTList VTs,
                                 ArrayRef<SDValue> Ops);
 
-  /// getTargetExtractSubreg - A convenience function for creating
-  /// TargetInstrInfo::EXTRACT_SUBREG nodes.
+  /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
   SDValue getTargetExtractSubreg(int SRIdx, SDLoc DL, EVT VT,
                                  SDValue Operand);
 
-  /// getTargetInsertSubreg - A convenience function for creating
-  /// TargetInstrInfo::INSERT_SUBREG nodes.
+  /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
   SDValue getTargetInsertSubreg(int SRIdx, SDLoc DL, EVT VT,
                                 SDValue Operand, SDValue Subreg);
 
-  /// getNodeIfExists - Get the specified node if it's already available, or
-  /// else return NULL.
+  /// Get the specified node if it's already available, or else return NULL.
   SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTs, ArrayRef<SDValue> Ops,
                           bool nuw = false, bool nsw = false,
                           bool exact = false);
 
-  /// getDbgValue - Creates a SDDbgValue node.
-  ///
-  /// SDNode
+  /// Creates a SDDbgValue node.
   SDDbgValue *getDbgValue(MDNode *Var, MDNode *Expr, SDNode *N, unsigned R,
                           bool IsIndirect, uint64_t Off, DebugLoc DL,
                           unsigned O);
@@ -1007,16 +1002,16 @@ public:
   SDDbgValue *getFrameIndexDbgValue(MDNode *Var, MDNode *Expr, unsigned FI,
                                     uint64_t Off, DebugLoc DL, unsigned O);
 
-  /// RemoveDeadNode - Remove the specified node from the system. If any of its
+  /// Remove the specified node from the system. If any of its
   /// operands then becomes dead, remove them as well. Inform UpdateListener
   /// for each node deleted.
   void RemoveDeadNode(SDNode *N);
 
-  /// RemoveDeadNodes - This method deletes the unreachable nodes in the
+  /// This method deletes the unreachable nodes in the
   /// given list, and any nodes that become unreachable as a result.
   void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes);
 
-  /// ReplaceAllUsesWith - Modify anything using 'From' to use 'To' instead.
+  /// Modify anything using 'From' to use 'To' instead.
   /// This can cause recursive merging of nodes in the DAG.  Use the first
   /// version if 'From' is known to have a single result, use the second
   /// if you have two nodes with identical results (or if 'To' has a superset
@@ -1035,30 +1030,29 @@ public:
   void ReplaceAllUsesWith(SDNode *From, SDNode *To);
   void ReplaceAllUsesWith(SDNode *From, const SDValue *To);
 
-  /// ReplaceAllUsesOfValueWith - Replace any uses of From with To, leaving
+  /// Replace any uses of From with To, leaving
   /// uses of other values produced by From.Val alone.
   void ReplaceAllUsesOfValueWith(SDValue From, SDValue To);
 
-  /// ReplaceAllUsesOfValuesWith - Like ReplaceAllUsesOfValueWith, but
-  /// for multiple values at once. This correctly handles the case where
+  /// Like ReplaceAllUsesOfValueWith, but for multiple values at once.
+  /// This correctly handles the case where
   /// there is an overlap between the From values and the To values.
   void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To,
                                   unsigned Num);
 
-  /// AssignTopologicalOrder - Topological-sort the AllNodes list and a
+  /// Topological-sort the AllNodes list and a
   /// assign a unique node id for each node in the DAG based on their
   /// topological order. Returns the number of nodes.
   unsigned AssignTopologicalOrder();
 
-  /// RepositionNode - Move node N in the AllNodes list to be immediately
+  /// Move node N in the AllNodes list to be immediately
   /// before the given iterator Position. This may be used to update the
   /// topological ordering when the list of nodes is modified.
   void RepositionNode(allnodes_iterator Position, SDNode *N) {
     AllNodes.insert(Position, AllNodes.remove(N));
   }
 
-  /// isCommutativeBinOp - Returns true if the opcode is a commutative binary
-  /// operation.
+  /// Returns true if the opcode is a commutative binary operation.
   static bool isCommutativeBinOp(unsigned Opcode) {
     // FIXME: This should get its info from the td file, so that we can include
     // target info.
@@ -1099,19 +1093,19 @@ public:
     }
   }
 
-  /// AddDbgValue - Add a dbg_value SDNode. If SD is non-null that means the
+  /// Add a dbg_value SDNode. If SD is non-null that means the
   /// value is produced by SD.
   void AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter);
 
-  /// GetDbgValues - Get the debug values which reference the given SDNode.
+  /// Get the debug values which reference the given SDNode.
   ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) {
     return DbgInfo->getSDDbgValues(SD);
   }
 
-  /// TransferDbgValues - Transfer SDDbgValues.
+  /// Transfer SDDbgValues.
   void TransferDbgValues(SDValue From, SDValue To);
 
-  /// hasDebugValues - Return true if there are any SDDbgValue nodes associated
+  /// Return true if there are any SDDbgValue nodes associated
   /// with this SelectionDAG.
   bool hasDebugValues() const { return !DbgInfo->empty(); }
 
@@ -1126,28 +1120,31 @@ public:
 
   void dump() const;
 
-  /// CreateStackTemporary - Create a stack temporary, suitable for holding the
+  /// Create a stack temporary, suitable for holding the
   /// specified value type.  If minAlign is specified, the slot size will have
   /// at least that alignment.
   SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1);
 
-  /// CreateStackTemporary - Create a stack temporary suitable for holding
+  /// Create a stack temporary suitable for holding
   /// either of the specified value types.
   SDValue CreateStackTemporary(EVT VT1, EVT VT2);
 
-  /// FoldConstantArithmetic -
-  SDValue FoldConstantArithmetic(unsigned Opcode, EVT VT,
+  SDValue FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
                                  SDNode *Cst1, SDNode *Cst2);
 
-  /// FoldSetCC - Constant fold a setcc to true or false.
+  SDValue FoldConstantArithmetic(unsigned Opcode, SDLoc DL, EVT VT,
+                                 const ConstantSDNode *Cst1,
+                                 const ConstantSDNode *Cst2);
+
+  /// Constant fold a setcc to true or false.
   SDValue FoldSetCC(EVT VT, SDValue N1,
                     SDValue N2, ISD::CondCode Cond, SDLoc dl);
 
-  /// SignBitIsZero - Return true if the sign bit of Op is known to be zero.  We
-  /// use this predicate to simplify operations downstream.
+  /// Return true if the sign bit of Op is known to be zero.
+  /// We use this predicate to simplify operations downstream.
   bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const;
 
-  /// MaskedValueIsZero - Return true if 'Op & Mask' is known to be zero.  We
+  /// Return true if 'Op & Mask' is known to be zero.  We
   /// use this predicate to simplify operations downstream.  Op and Mask are
   /// known to be the same type.
   bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth = 0)
@@ -1160,7 +1157,7 @@ public:
   void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne,
                         unsigned Depth = 0) const;
 
-  /// ComputeNumSignBits - Return the number of times the sign bit of the
+  /// Return the number of times the sign bit of the
   /// register is replicated into the other bits.  We know that at least 1 bit
   /// is always equal to the sign bit (itself), but other cases can give us
   /// information.  For example, immediately after an "SRA X, 2", we know that
@@ -1169,26 +1166,26 @@ public:
   /// class to allow target nodes to be understood.
   unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const;
 
-  /// isBaseWithConstantOffset - Return true if the specified operand is an
+  /// Return true if the specified operand is an
   /// ISD::ADD with a ConstantSDNode on the right-hand side, or if it is an
   /// ISD::OR with a ConstantSDNode that is guaranteed to have the same
   /// semantics as an ADD.  This handles the equivalence:
   ///     X|Cst == X+Cst iff X&Cst = 0.
   bool isBaseWithConstantOffset(SDValue Op) const;
 
-  /// isKnownNeverNan - Test whether the given SDValue is known to never be NaN.
+  /// Test whether the given SDValue is known to never be NaN.
   bool isKnownNeverNaN(SDValue Op) const;
 
-  /// isKnownNeverZero - Test whether the given SDValue is known to never be
+  /// Test whether the given SDValue is known to never be
   /// positive or negative Zero.
   bool isKnownNeverZero(SDValue Op) const;
 
-  /// isEqualTo - Test whether two SDValues are known to compare equal. This
+  /// Test whether two SDValues are known to compare equal. This
   /// is true if they are the same value, or if one is negative zero and the
   /// other positive zero.
   bool isEqualTo(SDValue A, SDValue B) const;
 
-  /// UnrollVectorOp - Utility function used by legalize and lowering to
+  /// Utility function used by legalize and lowering to
   /// "unroll" a vector operation by splitting out the scalars and operating
   /// on each element individually.  If the ResNE is 0, fully unroll the vector
   /// op. If ResNE is less than the width of the vector op, unroll up to ResNE.
@@ -1196,43 +1193,40 @@ public:
   /// vector op and fill the end of the resulting vector with UNDEFS.
   SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0);
 
-  /// isConsecutiveLoad - Return true if LD is loading 'Bytes' bytes from a
-  /// location that is 'Dist' units away from the location that the 'Base' load
-  /// is loading from.
+  /// Return true if LD is loading 'Bytes' bytes from a location that is 'Dist'
+  /// units away from the location that the 'Base' load is loading from.
   bool isConsecutiveLoad(LoadSDNode *LD, LoadSDNode *Base,
                          unsigned Bytes, int Dist) const;
 
-  /// InferPtrAlignment - Infer alignment of a load / store address. Return 0 if
+  /// Infer alignment of a load / store address. Return 0 if
   /// it cannot be inferred.
   unsigned InferPtrAlignment(SDValue Ptr) const;
 
-  /// GetSplitDestVTs - Compute the VTs needed for the low/hi parts of a type
+  /// Compute the VTs needed for the low/hi parts of a type
   /// which is split (or expanded) into two not necessarily identical pieces.
   std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const;
 
-  /// SplitVector - Split the vector with EXTRACT_SUBVECTOR using the provides
+  /// Split the vector with EXTRACT_SUBVECTOR using the provides
   /// VTs and return the low/high part.
   std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL,
                                           const EVT &LoVT, const EVT &HiVT);
 
-  /// SplitVector - Split the vector with EXTRACT_SUBVECTOR and return the
-  /// low/high part.
+  /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part.
   std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
     EVT LoVT, HiVT;
     std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType());
     return SplitVector(N, DL, LoVT, HiVT);
   }
 
-  /// SplitVectorOperand - Split the node's operand with EXTRACT_SUBVECTOR and
+  /// Split the node's operand with EXTRACT_SUBVECTOR and
   /// return the low/high part.
   std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo)
   {
     return SplitVector(N->getOperand(OpNo), SDLoc(N));
   }
 
-  /// ExtractVectorElements - Append the extracted elements from Start to Count
-  /// out of the vector Op in Args. If Count is 0, all of the elements will be
-  /// extracted.
+  /// Append the extracted elements from Start to Count out of the vector Op
+  /// in Args. If Count is 0, all of the elements will be extracted.
   void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
                              unsigned Start = 0, unsigned Count = 0);
 
@@ -1258,10 +1252,22 @@ private:
                                 SDValue N1, SDValue N2, bool nuw, bool nsw,
                                 bool exact);
 
-  /// VTList - List of non-single value types.
+  /// Look up the node specified by ID in CSEMap.  If it exists, return it.  If
+  /// not, return the insertion token that will make insertion faster.  This
+  /// overload is for nodes other than Constant or ConstantFP, use the other one
+  /// for those.
+  SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos);
+
+  /// Look up the node specified by ID in CSEMap.  If it exists, return it.  If
+  /// not, return the insertion token that will make insertion faster.  Performs
+  /// additional processing for constant nodes.
+  SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, DebugLoc DL,
+                              void *&InsertPos);
+
+  /// List of non-single value types.
   FoldingSet<SDVTListNode> VTListMap;
 
-  /// CondCodeNodes - Maps to auto-CSE operations.
+  /// Maps to auto-CSE operations.
   std::vector<CondCodeSDNode*> CondCodeNodes;
 
   std::vector<SDNode*> ValueTypeNodes;
diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h
index d53e66da5a71c..a011e4c338c4a 100644
--- a/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/include/llvm/CodeGen/SelectionDAGISel.h
@@ -58,7 +58,7 @@ public:
 
   explicit SelectionDAGISel(TargetMachine &tm,
                             CodeGenOpt::Level OL = CodeGenOpt::Default);
-  virtual ~SelectionDAGISel();
+  ~SelectionDAGISel() override;
 
   const TargetLowering *getTargetLowering() const { return TLI; }
 
@@ -80,12 +80,12 @@ public:
   virtual SDNode *Select(SDNode *N) = 0;
 
   /// SelectInlineAsmMemoryOperand - Select the specified address as a target
-  /// addressing mode, according to the specified constraint code.  If this does
+  /// addressing mode, according to the specified constraint.  If this does
   /// not match or is not implemented, return true.  The resultant operands
   /// (which will appear in the machine instruction) should be added to the
   /// OutOps vector.
   virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
-                                            char ConstraintCode,
+                                            unsigned ConstraintID,
                                             std::vector<SDValue> &OutOps) {
     return true;
   }
@@ -199,7 +199,7 @@ protected:
 
   /// SelectInlineAsmMemoryOperands - Calls to this are automatically generated
   /// by tblgen.  Others should not call it.
-  void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops);
+  void SelectInlineAsmMemoryOperands(std::vector<SDValue> &Ops, SDLoc DL);
 
 
 public:
@@ -260,7 +260,10 @@ private:
   SDNode *MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTs,
                     ArrayRef<SDValue> Ops, unsigned EmitNodeInfo);
 
-  void PrepareEHLandingPad();
+  /// Prepares the landing pad to take incoming values or do other EH
+  /// personality specific tasks. Returns true if the block should be
+  /// instruction selected, false if no code should be emitted for it.
+  bool PrepareEHLandingPad();
 
   /// \brief Perform instruction selection on all basic blocks in the function.
   void SelectAllBasicBlocks(const Function &Fn);
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index 933ec7fff4839..daa19430192a5 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -50,28 +50,10 @@ template <typename T> struct DenseMapInfo;
 template <typename T> struct simplify_type;
 template <typename T> struct ilist_traits;
 
-/// isBinOpWithFlags - Returns true if the opcode is a binary operation
-/// with flags.
-static bool isBinOpWithFlags(unsigned Opcode) {
-  switch (Opcode) {
-  case ISD::SDIV:
-  case ISD::UDIV:
-  case ISD::SRA:
-  case ISD::SRL:
-  case ISD::MUL:
-  case ISD::ADD:
-  case ISD::SUB:
-  case ISD::SHL:
-    return true;
-  default:
-    return false;
-  }
-}
-
 void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
                     bool force = false);
 
-/// SDVTList - This represents a list of ValueType's that has been intern'd by
+/// This represents a list of ValueType's that has been intern'd by
 /// a SelectionDAG.  Instances of this simple value class are returned by
 /// SelectionDAG::getVTList(...).
 ///
@@ -83,11 +65,11 @@ struct SDVTList {
 namespace ISD {
   /// Node predicates
 
-  /// isBuildVectorAllOnes - Return true if the specified node is a
+  /// Return true if the specified node is a
   /// BUILD_VECTOR where all of the elements are ~0 or undef.
   bool isBuildVectorAllOnes(const SDNode *N);
 
-  /// isBuildVectorAllZeros - Return true if the specified node is a
+  /// Return true if the specified node is a
   /// BUILD_VECTOR where all of the elements are 0 or undef.
   bool isBuildVectorAllZeros(const SDNode *N);
 
@@ -95,18 +77,22 @@ namespace ISD {
   /// all ConstantSDNode or undef.
   bool isBuildVectorOfConstantSDNodes(const SDNode *N);
 
-  /// isScalarToVector - Return true if the specified node is a
+  /// \brief Return true if the specified node is a BUILD_VECTOR node of
+  /// all ConstantFPSDNode or undef.
+  bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
+
+  /// Return true if the specified node is a
   /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
   /// element is not an undef.
   bool isScalarToVector(const SDNode *N);
 
-  /// allOperandsUndef - Return true if the node has at least one operand
+  /// Return true if the node has at least one operand
   /// and all operands of the specified node are ISD::UNDEF.
   bool allOperandsUndef(const SDNode *N);
 }  // end llvm:ISD namespace
 
 //===----------------------------------------------------------------------===//
-/// SDValue - Unlike LLVM values, Selection DAG nodes may return multiple
+/// Unlike LLVM values, Selection DAG nodes may return multiple
 /// values as the result of a computation.  Many nodes return multiple values,
 /// from loads (which define a token and a return value) to ADDC (which returns
 /// a result and a carry value), to calls (which may return an arbitrary number
@@ -145,7 +131,7 @@ public:
   bool operator<(const SDValue &O) const {
     return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
   }
-  LLVM_EXPLICIT operator bool() const {
+  explicit operator bool() const {
     return Node != nullptr;
   }
 
@@ -153,11 +139,10 @@ public:
     return SDValue(Node, R);
   }
 
-  // isOperandOf - Return true if this node is an operand of N.
+  // Return true if this node is an operand of N.
   bool isOperandOf(SDNode *N) const;
 
-  /// getValueType - Return the ValueType of the referenced return value.
-  ///
+  /// Return the ValueType of the referenced return value.
   inline EVT getValueType() const;
 
   /// Return the simple ValueType of the referenced return value.
@@ -165,8 +150,7 @@ public:
     return getValueType().getSimpleVT();
   }
 
-  /// getValueSizeInBits - Returns the size of the value in bits.
-  ///
+  /// Returns the size of the value in bits.
   unsigned getValueSizeInBits() const {
     return getValueType().getSizeInBits();
   }
@@ -184,26 +168,22 @@ public:
   inline bool isTargetOpcode() const;
   inline bool isMachineOpcode() const;
   inline unsigned getMachineOpcode() const;
-  inline const DebugLoc getDebugLoc() const;
+  inline const DebugLoc &getDebugLoc() const;
   inline void dump() const;
   inline void dumpr() const;
 
-  /// reachesChainWithoutSideEffects - Return true if this operand (which must
-  /// be a chain) reaches the specified operand without crossing any
-  /// side-effecting instructions.  In practice, this looks through token
-  /// factors and non-volatile loads.  In order to remain efficient, this only
+  /// Return true if this operand (which must be a chain) reaches the
+  /// specified operand without crossing any side-effecting instructions.
+  /// In practice, this looks through token factors and non-volatile loads.
+  /// In order to remain efficient, this only
   /// looks a couple of nodes in, it does not do an exhaustive search.
   bool reachesChainWithoutSideEffects(SDValue Dest,
                                       unsigned Depth = 2) const;
 
-  /// use_empty - Return true if there are no nodes using value ResNo
-  /// of Node.
-  ///
+  /// Return true if there are no nodes using value ResNo of Node.
   inline bool use_empty() const;
 
-  /// hasOneUse - Return true if there is exactly one node using value
-  /// ResNo of Node.
-  ///
+  /// Return true if there is exactly one node using value ResNo of Node.
   inline bool hasOneUse() const;
 };
 
@@ -230,7 +210,7 @@ template<> struct DenseMapInfo<SDValue> {
 template <> struct isPodLike<SDValue> { static const bool value = true; };
 
 
-/// simplify_type specializations - Allow casting operators to work directly on
+/// Allow casting operators to work directly on
 /// SDValues as if they were SDNode*'s.
 template<> struct simplify_type<SDValue> {
   typedef SDNode* SimpleType;
@@ -245,7 +225,7 @@ template<> struct simplify_type<const SDValue> {
   }
 };
 
-/// SDUse - Represents a use of a SDNode. This class holds an SDValue,
+/// Represents a use of a SDNode. This class holds an SDValue,
 /// which records the SDNode being used and the result number, a
 /// pointer to the SDNode using the value, and Next and Prev pointers,
 /// which link together all the uses of an SDNode.
@@ -259,8 +239,8 @@ class SDUse {
   /// this operand.
   SDUse **Prev, *Next;
 
-  SDUse(const SDUse &U) LLVM_DELETED_FUNCTION;
-  void operator=(const SDUse &U) LLVM_DELETED_FUNCTION;
+  SDUse(const SDUse &U) = delete;
+  void operator=(const SDUse &U) = delete;
 
 public:
   SDUse() : Val(), User(nullptr), Prev(nullptr), Next(nullptr) {}
@@ -272,30 +252,30 @@ public:
   /// the SDValue.
   const SDValue &get() const { return Val; }
 
-  /// getUser - This returns the SDNode that contains this Use.
+  /// This returns the SDNode that contains this Use.
   SDNode *getUser() { return User; }
 
-  /// getNext - Get the next SDUse in the use list.
+  /// Get the next SDUse in the use list.
   SDUse *getNext() const { return Next; }
 
-  /// getNode - Convenience function for get().getNode().
+  /// Convenience function for get().getNode().
   SDNode *getNode() const { return Val.getNode(); }
-  /// getResNo - Convenience function for get().getResNo().
+  /// Convenience function for get().getResNo().
   unsigned getResNo() const { return Val.getResNo(); }
-  /// getValueType - Convenience function for get().getValueType().
+  /// Convenience function for get().getValueType().
   EVT getValueType() const { return Val.getValueType(); }
 
-  /// operator== - Convenience function for get().operator==
+  /// Convenience function for get().operator==
   bool operator==(const SDValue &V) const {
     return Val == V;
   }
 
-  /// operator!= - Convenience function for get().operator!=
+  /// Convenience function for get().operator!=
   bool operator!=(const SDValue &V) const {
     return Val != V;
   }
 
-  /// operator< - Convenience function for get().operator<
+  /// Convenience function for get().operator<
   bool operator<(const SDValue &V) const {
     return Val < V;
   }
@@ -306,13 +286,13 @@ private:
 
   void setUser(SDNode *p) { User = p; }
 
-  /// set - Remove this use from its existing use list, assign it the
+  /// Remove this use from its existing use list, assign it the
   /// given value, and add it to the new value's node's use list.
   inline void set(const SDValue &V);
-  /// setInitial - like set, but only supports initializing a newly-allocated
+  /// Like set, but only supports initializing a newly-allocated
   /// SDUse with a non-null value.
   inline void setInitial(const SDValue &V);
-  /// setNode - like set, but only sets the Node portion of the value,
+  /// Like set, but only sets the Node portion of the value,
   /// leaving the ResNo portion unmodified.
   inline void setNode(SDNode *N);
 
@@ -339,47 +319,45 @@ template<> struct simplify_type<SDUse> {
 };
 
 
-/// SDNode - Represents one node in the SelectionDAG.
+/// Represents one node in the SelectionDAG.
 ///
 class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
 private:
-  /// NodeType - The operation that this node performs.
-  ///
+  /// The operation that this node performs.
   int16_t NodeType;
 
-  /// OperandsNeedDelete - This is true if OperandList was new[]'d.  If true,
+  /// This is true if OperandList was new[]'d.  If true,
   /// then they will be delete[]'d when the node is destroyed.
   uint16_t OperandsNeedDelete : 1;
 
-  /// HasDebugValue - This tracks whether this node has one or more dbg_value
+  /// This tracks whether this node has one or more dbg_value
   /// nodes corresponding to it.
   uint16_t HasDebugValue : 1;
 
 protected:
-  /// SubclassData - This member is defined by this class, but is not used for
+  /// This member is defined by this class, but is not used for
   /// anything.  Subclasses can use it to hold whatever state they find useful.
   /// This field is initialized to zero by the ctor.
   uint16_t SubclassData : 14;
 
 private:
-  /// NodeId - Unique id per SDNode in the DAG.
+  /// Unique id per SDNode in the DAG.
   int NodeId;
 
-  /// OperandList - The values that are used by this operation.
-  ///
+  /// The values that are used by this operation.
   SDUse *OperandList;
 
-  /// ValueList - The types of the values this node defines.  SDNode's may
+  /// The types of the values this node defines.  SDNode's may
   /// define multiple values simultaneously.
   const EVT *ValueList;
 
-  /// UseList - List of uses for this SDNode.
+  /// List of uses for this SDNode.
   SDUse *UseList;
 
-  /// NumOperands/NumValues - The number of entries in the Operand/Value list.
+  /// The number of entries in the Operand/Value list.
   unsigned short NumOperands, NumValues;
 
-  /// debugLoc - source line information.
+  /// Source line information.
   DebugLoc debugLoc;
 
   // The ordering of the SDNodes. It roughly corresponds to the ordering of the
@@ -389,7 +367,7 @@ private:
   // this ordering.
   unsigned IROrder;
 
-  /// getValueTypeList - Return a pointer to the specified value type.
+  /// Return a pointer to the specified value type.
   static const EVT *getValueTypeList(EVT VT);
 
   friend class SelectionDAG;
@@ -400,17 +378,17 @@ public:
   //  Accessors
   //
 
-  /// getOpcode - Return the SelectionDAG opcode value for this node. For
+  /// Return the SelectionDAG opcode value for this node. For
   /// pre-isel nodes (those for which isMachineOpcode returns false), these
   /// are the opcode values in the ISD and <target>ISD namespaces. For
   /// post-isel opcodes, see getMachineOpcode.
   unsigned getOpcode()  const { return (unsigned short)NodeType; }
 
-  /// isTargetOpcode - Test if this node has a target-specific opcode (in the
+  /// Test if this node has a target-specific opcode (in the
   /// \<target\>ISD namespace).
   bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
 
-  /// isTargetMemoryOpcode - Test if this node has a target-specific
+  /// Test if this node has a target-specific
   /// memory-referencing opcode (in the \<target\>ISD namespace and
   /// greater than FIRST_TARGET_MEMORY_OPCODE).
   bool isTargetMemoryOpcode() const {
@@ -427,11 +405,11 @@ public:
             NodeType == ISD::INTRINSIC_VOID) && ((SubclassData >> 13) & 1);
   }
 
-  /// isMachineOpcode - Test if this node has a post-isel opcode, directly
+  /// Test if this node has a post-isel opcode, directly
   /// corresponding to a MachineInstr opcode.
   bool isMachineOpcode() const { return NodeType < 0; }
 
-  /// getMachineOpcode - This may only be called if isMachineOpcode returns
+  /// This may only be called if isMachineOpcode returns
   /// true. It returns the MachineInstr opcode value that the node's opcode
   /// corresponds to.
   unsigned getMachineOpcode() const {
@@ -439,50 +417,44 @@ public:
     return ~NodeType;
   }
 
-  /// getHasDebugValue - get this bit.
+  /// Get this bit.
   bool getHasDebugValue() const { return HasDebugValue; }
 
-  /// setHasDebugValue - set this bit.
+  /// Set this bit.
   void setHasDebugValue(bool b) { HasDebugValue = b; }
 
-  /// use_empty - Return true if there are no uses of this node.
-  ///
+  /// Return true if there are no uses of this node.
   bool use_empty() const { return UseList == nullptr; }
 
-  /// hasOneUse - Return true if there is exactly one use of this node.
-  ///
+  /// Return true if there is exactly one use of this node.
   bool hasOneUse() const {
     return !use_empty() && std::next(use_begin()) == use_end();
   }
 
-  /// use_size - Return the number of uses of this node. This method takes
+  /// Return the number of uses of this node. This method takes
   /// time proportional to the number of uses.
-  ///
   size_t use_size() const { return std::distance(use_begin(), use_end()); }
 
-  /// getNodeId - Return the unique node id.
-  ///
+  /// Return the unique node id.
   int getNodeId() const { return NodeId; }
 
-  /// setNodeId - Set unique node id.
+  /// Set unique node id.
   void setNodeId(int Id) { NodeId = Id; }
 
-  /// getIROrder - Return the node ordering.
-  ///
+  /// Return the node ordering.
   unsigned getIROrder() const { return IROrder; }
 
-  /// setIROrder - Set the node ordering.
-  ///
+  /// Set the node ordering.
   void setIROrder(unsigned Order) { IROrder = Order; }
 
-  /// getDebugLoc - Return the source location info.
-  const DebugLoc getDebugLoc() const { return debugLoc; }
+  /// Return the source location info.
+  const DebugLoc &getDebugLoc() const { return debugLoc; }
 
-  /// setDebugLoc - Set source location info.  Try to avoid this, putting
+  /// Set source location info.  Try to avoid this, putting
   /// it in the constructor is preferable.
-  void setDebugLoc(const DebugLoc dl) { debugLoc = dl; }
+  void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
 
-  /// use_iterator - This class provides iterator support for SDUse
+  /// This class provides iterator support for SDUse
   /// operands that use a specific SDNode.
   class use_iterator
     : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> {
@@ -506,7 +478,7 @@ public:
       return !operator==(x);
     }
 
-    /// atEnd - return true if this iterator is at the end of uses list.
+    /// Return true if this iterator is at the end of uses list.
     bool atEnd() const { return Op == nullptr; }
 
     // Iterator traversal: forward iteration only.
@@ -530,17 +502,14 @@ public:
 
     SDUse &getUse() const { return *Op; }
 
-    /// getOperandNo - Retrieve the operand # of this use in its user.
-    ///
+    /// Retrieve the operand # of this use in its user.
     unsigned getOperandNo() const {
       assert(Op && "Cannot dereference end iterator!");
       return (unsigned)(Op - Op->getUser()->OperandList);
     }
   };
 
-  /// use_begin/use_end - Provide iteration support to walk over all uses
-  /// of an SDNode.
-
+  /// Provide iteration support to walk over all uses of an SDNode.
   use_iterator use_begin() const {
     return use_iterator(UseList);
   }
@@ -554,37 +523,34 @@ public:
     return iterator_range<use_iterator>(use_begin(), use_end());
   }
 
-  /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
-  /// indicated value.  This method ignores uses of other values defined by this
-  /// operation.
+  /// Return true if there are exactly NUSES uses of the indicated value.
+  /// This method ignores uses of other values defined by this operation.
   bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
 
-  /// hasAnyUseOfValue - Return true if there are any use of the indicated
-  /// value. This method ignores uses of other values defined by this operation.
+  /// Return true if there are any use of the indicated value.
+  /// This method ignores uses of other values defined by this operation.
   bool hasAnyUseOfValue(unsigned Value) const;
 
-  /// isOnlyUserOf - Return true if this node is the only use of N.
-  ///
+  /// Return true if this node is the only use of N.
   bool isOnlyUserOf(SDNode *N) const;
 
-  /// isOperandOf - Return true if this node is an operand of N.
-  ///
+  /// Return true if this node is an operand of N.
   bool isOperandOf(SDNode *N) const;
 
-  /// isPredecessorOf - Return true if this node is a predecessor of N.
+  /// Return true if this node is a predecessor of N.
   /// NOTE: Implemented on top of hasPredecessor and every bit as
   /// expensive. Use carefully.
   bool isPredecessorOf(const SDNode *N) const {
     return N->hasPredecessor(this);
   }
 
-  /// hasPredecessor - Return true if N is a predecessor of this node.
+  /// Return true if N is a predecessor of this node.
   /// N is either an operand of this node, or can be reached by recursively
   /// traversing up the operands.
   /// NOTE: This is an expensive method. Use it carefully.
   bool hasPredecessor(const SDNode *N) const;
 
-  /// hasPredecesorHelper - Return true if N is a predecessor of this node.
+  /// Return true if N is a predecessor of this node.
   /// N is either an operand of this node, or can be reached by recursively
   /// traversing up the operands.
   /// In this helper the Visited and worklist sets are held externally to
@@ -597,12 +563,10 @@ public:
                             SmallPtrSetImpl<const SDNode *> &Visited,
                             SmallVectorImpl<const SDNode *> &Worklist) const;
 
-  /// getNumOperands - Return the number of values used by this operation.
-  ///
+  /// Return the number of values used by this operation.
   unsigned getNumOperands() const { return NumOperands; }
 
-  /// getConstantOperandVal - Helper method returns the integer value of a
-  /// ConstantSDNode operand.
+  /// Helper method returns the integer value of a ConstantSDNode operand.
   uint64_t getConstantOperandVal(unsigned Num) const;
 
   const SDValue &getOperand(unsigned Num) const {
@@ -620,7 +584,7 @@ public:
     return X;
   }
 
-  /// getGluedNode - If this node has a glue operand, return the node
+  /// If this node has a glue operand, return the node
   /// to which the glue operand points. Otherwise return NULL.
   SDNode *getGluedNode() const {
     if (getNumOperands() != 0 &&
@@ -645,7 +609,7 @@ public:
     return FoundNode;
   }
 
-  /// getGluedUser - If this node has a glue value with a user, return
+  /// If this node has a glue value with a user, return
   /// the user (there is at most one). Otherwise return NULL.
   SDNode *getGluedUser() const {
     for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
@@ -654,26 +618,21 @@ public:
     return nullptr;
   }
 
-  /// getNumValues - Return the number of values defined/returned by this
-  /// operator.
-  ///
+  /// Return the number of values defined/returned by this operator.
   unsigned getNumValues() const { return NumValues; }
 
-  /// getValueType - Return the type of a specified result.
-  ///
+  /// Return the type of a specified result.
   EVT getValueType(unsigned ResNo) const {
     assert(ResNo < NumValues && "Illegal result number!");
     return ValueList[ResNo];
   }
 
   /// Return the type of a specified result as a simple type.
-  ///
   MVT getSimpleValueType(unsigned ResNo) const {
     return getValueType(ResNo).getSimpleVT();
   }
 
-  /// getValueSizeInBits - Returns MVT::getSizeInBits(getValueType(ResNo)).
-  ///
+  /// Returns MVT::getSizeInBits(getValueType(ResNo)).
   unsigned getValueSizeInBits(unsigned ResNo) const {
     return getValueType(ResNo).getSizeInBits();
   }
@@ -682,8 +641,7 @@ public:
   value_iterator value_begin() const { return ValueList; }
   value_iterator value_end() const { return ValueList+NumValues; }
 
-  /// getOperationName - Return the opcode of this operation for printing.
-  ///
+  /// Return the opcode of this operation for printing.
   std::string getOperationName(const SelectionDAG *G = nullptr) const;
   static const char* getIndexedModeName(ISD::MemIndexedMode AM);
   void print_types(raw_ostream &OS, const SelectionDAG *G) const;
@@ -691,7 +649,7 @@ public:
   void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
   void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
 
-  /// printrFull - Print a SelectionDAG node and all children down to
+  /// Print a SelectionDAG node and all children down to
   /// the leaves.  The given SelectionDAG allows target-specific nodes
   /// to be printed in human-readable form.  Unlike printr, this will
   /// print the whole DAG, including children that appear multiple
@@ -699,7 +657,7 @@ public:
   ///
   void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
 
-  /// printrWithDepth - Print a SelectionDAG node and children up to
+  /// Print a SelectionDAG node and children up to
   /// depth "depth."  The given SelectionDAG allows target-specific
   /// nodes to be printed in human-readable form.  Unlike printr, this
   /// will print children that appear multiple times wherever they are
@@ -709,30 +667,29 @@ public:
                        unsigned depth = 100) const;
 
 
-  /// dump - Dump this node, for debugging.
+  /// Dump this node, for debugging.
   void dump() const;
 
-  /// dumpr - Dump (recursively) this node and its use-def subgraph.
+  /// Dump (recursively) this node and its use-def subgraph.
   void dumpr() const;
 
-  /// dump - Dump this node, for debugging.
+  /// Dump this node, for debugging.
   /// The given SelectionDAG allows target-specific nodes to be printed
   /// in human-readable form.
   void dump(const SelectionDAG *G) const;
 
-  /// dumpr - Dump (recursively) this node and its use-def subgraph.
+  /// Dump (recursively) this node and its use-def subgraph.
   /// The given SelectionDAG allows target-specific nodes to be printed
   /// in human-readable form.
   void dumpr(const SelectionDAG *G) const;
 
-  /// dumprFull - printrFull to dbgs().  The given SelectionDAG allows
+  /// printrFull to dbgs().  The given SelectionDAG allows
   /// target-specific nodes to be printed in human-readable form.
   /// Unlike dumpr, this will print the whole DAG, including children
   /// that appear multiple times.
-  ///
   void dumprFull(const SelectionDAG *G = nullptr) const;
 
-  /// dumprWithDepth - printrWithDepth to dbgs().  The given
+  /// printrWithDepth to dbgs().  The given
   /// SelectionDAG allows target-specific nodes to be printed in
   /// human-readable form.  Unlike dumpr, this will print children
   /// that appear multiple times wherever they are used.
@@ -740,12 +697,10 @@ public:
   void dumprWithDepth(const SelectionDAG *G = nullptr,
                       unsigned depth = 100) const;
 
-  /// Profile - Gather unique data for the node.
-  ///
+  /// Gather unique data for the node.
   void Profile(FoldingSetNodeID &ID) const;
 
-  /// addUse - This method should only be used by the SDUse class.
-  ///
+  /// This method should only be used by the SDUse class.
   void addUse(SDUse &U) { U.addToList(&UseList); }
 
 protected:
@@ -754,14 +709,13 @@ protected:
     return Ret;
   }
 
-  SDNode(unsigned Opc, unsigned Order, const DebugLoc dl, SDVTList VTs,
+  SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
          ArrayRef<SDValue> Ops)
-    : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false),
-      SubclassData(0), NodeId(-1),
-      OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr),
-      ValueList(VTs.VTs), UseList(nullptr),
-      NumOperands(Ops.size()), NumValues(VTs.NumVTs),
-      debugLoc(dl), IROrder(Order) {
+      : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false),
+        SubclassData(0), NodeId(-1),
+        OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr),
+        ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()),
+        NumValues(VTs.NumVTs), debugLoc(std::move(dl)), IROrder(Order) {
     assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
     assert(NumOperands == Ops.size() &&
            "NumOperands wasn't wide enough for its operands!");
@@ -777,17 +731,17 @@ protected:
 
   /// This constructor adds no operands itself; operands can be
   /// set later with InitOperands.
-  SDNode(unsigned Opc, unsigned Order, const DebugLoc dl, SDVTList VTs)
-    : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false),
-      SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs),
-      UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), debugLoc(dl),
-      IROrder(Order) {
+  SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
+      : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false),
+        SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs),
+        UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs),
+        debugLoc(std::move(dl)), IROrder(Order) {
     assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
     assert(NumValues == VTs.NumVTs &&
            "NumValues wasn't wide enough for its operands!");
   }
 
-  /// InitOperands - Initialize the operands list of this with 1 operand.
+  /// Initialize the operands list of this with 1 operand.
   void InitOperands(SDUse *Ops, const SDValue &Op0) {
     Ops[0].setUser(this);
     Ops[0].setInitial(Op0);
@@ -796,7 +750,7 @@ protected:
     checkForCycles(this);
   }
 
-  /// InitOperands - Initialize the operands list of this with 2 operands.
+  /// Initialize the operands list of this with 2 operands.
   void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1) {
     Ops[0].setUser(this);
     Ops[0].setInitial(Op0);
@@ -807,7 +761,7 @@ protected:
     checkForCycles(this);
   }
 
-  /// InitOperands - Initialize the operands list of this with 3 operands.
+  /// Initialize the operands list of this with 3 operands.
   void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1,
                     const SDValue &Op2) {
     Ops[0].setUser(this);
@@ -821,7 +775,7 @@ protected:
     checkForCycles(this);
   }
 
-  /// InitOperands - Initialize the operands list of this with 4 operands.
+  /// Initialize the operands list of this with 4 operands.
   void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1,
                     const SDValue &Op2, const SDValue &Op3) {
     Ops[0].setUser(this);
@@ -837,7 +791,7 @@ protected:
     checkForCycles(this);
   }
 
-  /// InitOperands - Initialize the operands list of this with N operands.
+  /// Initialize the operands list of this with N operands.
   void InitOperands(SDUse *Ops, const SDValue *Vals, unsigned N) {
     for (unsigned i = 0; i != N; ++i) {
       Ops[i].setUser(this);
@@ -850,8 +804,7 @@ protected:
     checkForCycles(this);
   }
 
-  /// DropOperands - Release the operands and set this node to have
-  /// zero operands.
+  /// Release the operands and set this node to have zero operands.
   void DropOperands();
 };
 
@@ -945,7 +898,7 @@ inline bool SDValue::use_empty() const {
 inline bool SDValue::hasOneUse() const {
   return Node->hasNUsesOfValue(1, ResNo);
 }
-inline const DebugLoc SDValue::getDebugLoc() const {
+inline const DebugLoc &SDValue::getDebugLoc() const {
   return Node->getDebugLoc();
 }
 inline void SDValue::dump() const {
@@ -973,7 +926,63 @@ inline void SDUse::setNode(SDNode *N) {
   if (N) N->addUse(*this);
 }
 
-/// UnarySDNode - This class is used for single-operand SDNodes.  This is solely
+/// These are IR-level optimization flags that may be propagated to SDNodes.
+/// TODO: This data structure should be shared by the IR optimizer and the
+/// the backend.
+struct SDNodeFlags {
+private:
+  bool NoUnsignedWrap : 1;
+  bool NoSignedWrap : 1;
+  bool Exact : 1;
+  bool UnsafeAlgebra : 1;
+  bool NoNaNs : 1;
+  bool NoInfs : 1;
+  bool NoSignedZeros : 1;
+  bool AllowReciprocal : 1;
+
+public:
+  /// Default constructor turns off all optimization flags.
+  SDNodeFlags() {
+    NoUnsignedWrap = false;
+    NoSignedWrap = false;
+    Exact = false;
+    UnsafeAlgebra = false;
+    NoNaNs = false;
+    NoInfs = false;
+    NoSignedZeros = false;
+    AllowReciprocal = false;
+  }
+
+  // These are mutators for each flag.
+  void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
+  void setNoSignedWrap(bool b) { NoSignedWrap = b; }
+  void setExact(bool b) { Exact = b; }
+  void setUnsafeAlgebra(bool b) { UnsafeAlgebra = b; }
+  void setNoNaNs(bool b) { NoNaNs = b; }
+  void setNoInfs(bool b) { NoInfs = b; }
+  void setNoSignedZeros(bool b) { NoSignedZeros = b; }
+  void setAllowReciprocal(bool b) { AllowReciprocal = b; }
+
+  // These are accessors for each flag.
+  bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
+  bool hasNoSignedWrap() const { return NoSignedWrap; }
+  bool hasExact() const { return Exact; }
+  bool hasUnsafeAlgebra() const { return UnsafeAlgebra; }
+  bool hasNoNaNs() const { return NoNaNs; }
+  bool hasNoInfs() const { return NoInfs; }
+  bool hasNoSignedZeros() const { return NoSignedZeros; }
+  bool hasAllowReciprocal() const { return AllowReciprocal; }
+
+  /// Return a raw encoding of the flags.
+  /// This function should only be used to add data to the NodeID value.
+  unsigned getRawFlags() const {
+    return (NoUnsignedWrap << 0) | (NoSignedWrap << 1) | (Exact << 2) |
+           (UnsafeAlgebra << 3) | (NoNaNs << 4) | (NoInfs << 5) |
+           (NoSignedZeros << 6) | (AllowReciprocal << 7);
+  }
+};
+
+/// This class is used for single-operand SDNodes.  This is solely
 /// to allow co-allocation of node operands with the node itself.
 class UnarySDNode : public SDNode {
   SDUse Op;
@@ -985,7 +994,7 @@ public:
   }
 };
 
-/// BinarySDNode - This class is used for two-operand SDNodes.  This is solely
+/// This class is used for two-operand SDNodes.  This is solely
 /// to allow co-allocation of node operands with the node itself.
 class BinarySDNode : public SDNode {
   SDUse Ops[2];
@@ -997,37 +1006,37 @@ public:
   }
 };
 
-/// BinaryWithFlagsSDNode - This class is an extension of BinarySDNode
+/// Returns true if the opcode is a binary operation with flags.
+static bool isBinOpWithFlags(unsigned Opcode) {
+  switch (Opcode) {
+  case ISD::SDIV:
+  case ISD::UDIV:
+  case ISD::SRA:
+  case ISD::SRL:
+  case ISD::MUL:
+  case ISD::ADD:
+  case ISD::SUB:
+  case ISD::SHL:
+    return true;
+  default:
+    return false;
+  }
+}
+
+/// This class is an extension of BinarySDNode
 /// used from those opcodes that have associated extra flags.
 class BinaryWithFlagsSDNode : public BinarySDNode {
-  enum { NUW = (1 << 0), NSW = (1 << 1), EXACT = (1 << 2) };
-
 public:
+  SDNodeFlags Flags;
   BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
                         SDValue X, SDValue Y)
-      : BinarySDNode(Opc, Order, dl, VTs, X, Y) {}
-  /// getRawSubclassData - Return the SubclassData value, which contains an
-  /// encoding of the flags.
-  /// This function should be used to add subclass data to the NodeID value.
-  unsigned getRawSubclassData() const { return SubclassData; }
-  void setHasNoUnsignedWrap(bool b) {
-    SubclassData = (SubclassData & ~NUW) | (b ? NUW : 0);
-  }
-  void setHasNoSignedWrap(bool b) {
-    SubclassData = (SubclassData & ~NSW) | (b ? NSW : 0);
-  }
-  void setIsExact(bool b) {
-    SubclassData = (SubclassData & ~EXACT) | (b ? EXACT : 0);
-  }
-  bool hasNoUnsignedWrap() const { return SubclassData & NUW; }
-  bool hasNoSignedWrap() const { return SubclassData & NSW; }
-  bool isExact() const { return SubclassData & EXACT; }
+      : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags() {}
   static bool classof(const SDNode *N) {
     return isBinOpWithFlags(N->getOpcode());
   }
 };
 
-/// TernarySDNode - This class is used for three-operand SDNodes. This is solely
+/// This class is used for three-operand SDNodes. This is solely
 /// to allow co-allocation of node operands with the node itself.
 class TernarySDNode : public SDNode {
   SDUse Ops[3];
@@ -1040,7 +1049,7 @@ public:
 };
 
 
-/// HandleSDNode - This class is used to form a handle around another node that
+/// This class is used to form a handle around another node that
 /// is persistent and is updated across invocations of replaceAllUsesWith on its
 /// operand.  This node should be directly created by end-users and not added to
 /// the AllNodes list.
@@ -1072,14 +1081,14 @@ public:
   }
 };
 
-/// Abstact virtual class for operations for memory operations
+/// This is an abstract virtual class for memory operations.
 class MemSDNode : public SDNode {
 private:
-  // MemoryVT - VT of in-memory value.
+  // VT of in-memory value.
   EVT MemoryVT;
 
 protected:
-  /// MMO - Memory reference information.
+  /// Memory reference information.
   MachineMemOperand *MMO;
 
 public:
@@ -1100,7 +1109,7 @@ public:
     return MMO->getAlignment();
   }
 
-  /// getRawSubclassData - Return the SubclassData value, which contains an
+  /// Return the SubclassData value, which contains an
   /// encoding of the volatile flag, as well as bits used by subclasses. This
   /// function should only be used to compute a FoldingSetNodeID value.
   unsigned getRawSubclassData() const {
@@ -1129,10 +1138,10 @@ public:
   /// Returns the Ranges that describes the dereference.
   const MDNode *getRanges() const { return MMO->getRanges(); }
 
-  /// getMemoryVT - Return the type of the in-memory value.
+  /// Return the type of the in-memory value.
   EVT getMemoryVT() const { return MemoryVT; }
 
-  /// getMemOperand - Return a MachineMemOperand object describing the memory
+  /// Return a MachineMemOperand object describing the memory
   /// reference performed by operation.
   MachineMemOperand *getMemOperand() const { return MMO; }
 
@@ -1140,12 +1149,12 @@ public:
     return MMO->getPointerInfo();
   }
 
-  /// getAddressSpace - Return the address space for the associated pointer
+  /// Return the address space for the associated pointer
   unsigned getAddressSpace() const {
     return getPointerInfo().getAddrSpace();
   }
 
-  /// refineAlignment - Update this MemSDNode's MachineMemOperand information
+  /// Update this MemSDNode's MachineMemOperand information
   /// to reflect the alignment of NewMMO, if it has a greater alignment.
   /// This must only be used when the new alignment applies to all users of
   /// this MachineMemOperand.
@@ -1182,13 +1191,14 @@ public:
            N->getOpcode() == ISD::ATOMIC_STORE        ||
            N->getOpcode() == ISD::MLOAD               ||
            N->getOpcode() == ISD::MSTORE              ||
+           N->getOpcode() == ISD::MGATHER             ||
+           N->getOpcode() == ISD::MSCATTER            ||
            N->isMemIntrinsic()                        ||
            N->isTargetMemoryOpcode();
   }
 };
 
-/// AtomicSDNode - A SDNode reprenting atomic operations.
-///
+/// This is an SDNode representing atomic operations.
 class AtomicSDNode : public MemSDNode {
   SDUse Ops[4];
 
@@ -1301,7 +1311,7 @@ public:
   }
 };
 
-/// MemIntrinsicSDNode - This SDNode is used for target intrinsics that touch
+/// This SDNode is used for target intrinsics that touch
 /// memory and need an associated MachineMemOperand. Its opcode may be
 /// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
 /// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
@@ -1324,7 +1334,7 @@ public:
   }
 };
 
-/// ShuffleVectorSDNode - This SDNode is used to implement the code generator
+/// This SDNode is used to implement the code generator
 /// support for the llvm IR shufflevector instruction.  It combines elements
 /// from two input vectors into a new input vector, with the selection and
 /// ordering of elements determined by an array of integers, referred to as
@@ -1368,6 +1378,21 @@ public:
   }
   static bool isSplatMask(const int *Mask, EVT VT);
 
+  /// Change values in a shuffle permute mask assuming
+  /// the two vector operands have swapped position.
+  static void commuteMask(SmallVectorImpl<int> &Mask) {
+    unsigned NumElems = Mask.size();
+    for (unsigned i = 0; i != NumElems; ++i) {
+      int idx = Mask[i];
+      if (idx < 0)
+        continue;
+      else if (idx < (int)NumElems)
+        Mask[i] = idx + NumElems;
+      else
+        Mask[i] = idx - NumElems;
+    }
+  }
+
   static bool classof(const SDNode *N) {
     return N->getOpcode() == ISD::VECTOR_SHUFFLE;
   }
@@ -1376,9 +1401,10 @@ public:
 class ConstantSDNode : public SDNode {
   const ConstantInt *Value;
   friend class SelectionDAG;
-  ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
+  ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val,
+                 DebugLoc DL, EVT VT)
     : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant,
-             0, DebugLoc(), getSDVTList(VT)), Value(val) {
+             0, DL, getSDVTList(VT)), Value(val) {
     SubclassData |= (uint16_t)isOpaque;
   }
 public:
@@ -1403,28 +1429,28 @@ public:
 class ConstantFPSDNode : public SDNode {
   const ConstantFP *Value;
   friend class SelectionDAG;
-  ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
+  ConstantFPSDNode(bool isTarget, const ConstantFP *val, DebugLoc DL, EVT VT)
     : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP,
-             0, DebugLoc(), getSDVTList(VT)), Value(val) {
+             0, DL, getSDVTList(VT)), Value(val) {
   }
 public:
 
   const APFloat& getValueAPF() const { return Value->getValueAPF(); }
   const ConstantFP *getConstantFPValue() const { return Value; }
 
-  /// isZero - Return true if the value is positive or negative zero.
+  /// Return true if the value is positive or negative zero.
   bool isZero() const { return Value->isZero(); }
 
-  /// isNaN - Return true if the value is a NaN.
+  /// Return true if the value is a NaN.
   bool isNaN() const { return Value->isNaN(); }
 
-  /// isInfinity - Return true if the value is an infinity
+  /// Return true if the value is an infinity
   bool isInfinity() const { return Value->isInfinity(); }
 
-  /// isNegative - Return true if the value is negative.
+  /// Return true if the value is negative.
   bool isNegative() const { return Value->isNegative(); }
 
-  /// isExactlyValue - We don't rely on operator== working on double values, as
+  /// We don't rely on operator== working on double values, as
   /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
   /// As such, this method can be used to do an exact bit-for-bit comparison of
   /// two floating point values.
@@ -1608,13 +1634,12 @@ public:
   }
 };
 
-/// BuildVectorSDNode - A "pseudo-class" with methods for operating on
-/// BUILD_VECTORs.
+/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
 class BuildVectorSDNode : public SDNode {
   // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
-  explicit BuildVectorSDNode() LLVM_DELETED_FUNCTION;
+  explicit BuildVectorSDNode() = delete;
 public:
-  /// isConstantSplat - Check if this is a constant splat, and if so, find the
+  /// Check if this is a constant splat, and if so, find the
   /// smallest element size that splats the vector.  If MinSplatBits is
   /// nonzero, the element size must be at least that large.  Note that the
   /// splat element may be the entire vector (i.e., a one element vector).
@@ -1657,7 +1682,7 @@ public:
   }
 };
 
-/// SrcValueSDNode - An SDNode that holds an arbitrary LLVM IR Value. This is
+/// An SDNode that holds an arbitrary LLVM IR Value. This is
 /// used when the SelectionDAG needs to make a simple reference to something
 /// in the LLVM IR representation.
 ///
@@ -1669,7 +1694,7 @@ class SrcValueSDNode : public SDNode {
     : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
 
 public:
-  /// getValue - return the contained Value.
+  /// Return the contained Value.
   const Value *getValue() const { return V; }
 
   static bool classof(const SDNode *N) {
@@ -1796,7 +1821,7 @@ public:
   }
 };
 
-/// CvtRndSatSDNode - NOTE: avoid using this node as this may disappear in the
+/// NOTE: avoid using this node as this may disappear in the
 /// future and most targets don't support it.
 class CvtRndSatSDNode : public SDNode {
   ISD::CvtCode CvtCode;
@@ -1815,7 +1840,7 @@ public:
   }
 };
 
-/// VTSDNode - This class is used to represent EVT's, which are used
+/// This class is used to represent EVT's, which are used
 /// to parameterize some operations.
 class VTSDNode : public SDNode {
   EVT ValueType;
@@ -1833,8 +1858,7 @@ public:
   }
 };
 
-/// LSBaseSDNode - Base class for LoadSDNode and StoreSDNode
-///
+/// Base class for LoadSDNode and StoreSDNode
 class LSBaseSDNode : public MemSDNode {
   //! Operand array for load and store
   /*!
@@ -1860,16 +1884,16 @@ public:
     return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
   }
 
-  /// getAddressingMode - Return the addressing mode for this load or store:
+  /// Return the addressing mode for this load or store:
   /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
   ISD::MemIndexedMode getAddressingMode() const {
     return ISD::MemIndexedMode((SubclassData >> 2) & 7);
   }
 
-  /// isIndexed - Return true if this is a pre/post inc/dec load/store.
+  /// Return true if this is a pre/post inc/dec load/store.
   bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
 
-  /// isUnindexed - Return true if this is NOT a pre/post inc/dec load/store.
+  /// Return true if this is NOT a pre/post inc/dec load/store.
   bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
 
   static bool classof(const SDNode *N) {
@@ -1878,8 +1902,7 @@ public:
   }
 };
 
-/// LoadSDNode - This class is used to represent ISD::LOAD nodes.
-///
+/// This class is used to represent ISD::LOAD nodes.
 class LoadSDNode : public LSBaseSDNode {
   friend class SelectionDAG;
   LoadSDNode(SDValue *ChainPtrOff, unsigned Order, DebugLoc dl, SDVTList VTs,
@@ -1893,7 +1916,7 @@ class LoadSDNode : public LSBaseSDNode {
   }
 public:
 
-  /// getExtensionType - Return whether this is a plain node,
+  /// Return whether this is a plain node,
   /// or one of the varieties of value-extending loads.
   ISD::LoadExtType getExtensionType() const {
     return ISD::LoadExtType(SubclassData & 3);
@@ -1907,8 +1930,7 @@ public:
   }
 };
 
-/// StoreSDNode - This class is used to represent ISD::STORE nodes.
-///
+/// This class is used to represent ISD::STORE nodes.
 class StoreSDNode : public LSBaseSDNode {
   friend class SelectionDAG;
   StoreSDNode(SDValue *ChainValuePtrOff, unsigned Order, DebugLoc dl,
@@ -1923,7 +1945,7 @@ class StoreSDNode : public LSBaseSDNode {
   }
 public:
 
-  /// isTruncatingStore - Return true if the op does a truncation before store.
+  /// Return true if the op does a truncation before store.
   /// For integers this is the same as doing a TRUNCATE and storing the result.
   /// For floats, it is the same as doing an FP_ROUND and storing the result.
   bool isTruncatingStore() const { return SubclassData & 1; }
@@ -1937,9 +1959,7 @@ public:
   }
 };
 
-/// MaskedLoadStoreSDNode - This is a base class is used to represent MLOAD and
-/// MSTORE nodes
-///
+/// This base class is used to represent MLOAD and MSTORE nodes
 class MaskedLoadStoreSDNode : public MemSDNode {
   // Operands
   SDUse Ops[4];
@@ -1965,8 +1985,7 @@ public:
   }
 };
 
-/// MaskedLoadSDNode - This class is used to represent an MLOAD node
-///
+/// This class is used to represent an MLOAD node
 class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
 public:
   friend class SelectionDAG;
@@ -1987,8 +2006,7 @@ public:
   }
 };
 
-/// MaskedStoreSDNode - This class is used to represent an MSTORE node
-///
+/// This class is used to represent an MSTORE node
 class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
 
 public:
@@ -2000,7 +2018,7 @@ public:
                             VTs, MemVT, MMO) {
       SubclassData |= (unsigned short)isTrunc;
   }
-  /// isTruncatingStore - Return true if the op does a truncation before store.
+  /// Return true if the op does a truncation before store.
   /// For integers this is the same as doing a TRUNCATE and storing the result.
   /// For floats, it is the same as doing an FP_ROUND and storing the result.
   bool isTruncatingStore() const { return SubclassData & 1; }
@@ -2012,10 +2030,85 @@ public:
   }
 };
 
-/// MachineSDNode - An SDNode that represents everything that will be needed
+/// This is a base class used to represent
+/// MGATHER and MSCATTER nodes
+///
+class MaskedGatherScatterSDNode : public MemSDNode {
+  // Operands
+  SDUse Ops[5];
+public:
+  friend class SelectionDAG;
+  MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
+                            ArrayRef<SDValue> Operands, SDVTList VTs, EVT MemVT,
+                            MachineMemOperand *MMO)
+    : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
+    assert(Operands.size() == 5 && "Incompatible number of operands");
+    InitOperands(Ops, Operands.data(), Operands.size());
+  }
+
+  // In the both nodes address is Op1, mask is Op2:
+  // MaskedGatherSDNode  (Chain, src0, mask, base, index), src0 is a passthru value
+  // MaskedScatterSDNode (Chain, value, mask, base, index)
+  // Mask is a vector of i1 elements
+  const SDValue &getBasePtr() const { return getOperand(3); }
+  const SDValue &getIndex()   const { return getOperand(4); }
+  const SDValue &getMask()    const { return getOperand(2); }
+  const SDValue &getValue()   const { return getOperand(1); }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::MGATHER ||
+           N->getOpcode() == ISD::MSCATTER;
+  }
+};
+
+/// This class is used to represent an MGATHER node
+///
+class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
+public:
+  friend class SelectionDAG;
+  MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef<SDValue> Operands, 
+                     SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
+    : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT,
+                                MMO) {
+    assert(getValue().getValueType() == getValueType(0) &&
+           "Incompatible type of the PathThru value in MaskedGatherSDNode");
+    assert(getMask().getValueType().getVectorNumElements() == 
+           getValueType(0).getVectorNumElements() && 
+           "Vector width mismatch between mask and data");
+    assert(getMask().getValueType().getScalarType() == MVT::i1 && 
+           "Vector width mismatch between mask and data");
+  }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::MGATHER;
+  }
+};
+
+/// This class is used to represent an MSCATTER node
+///
+class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
+
+public:
+  friend class SelectionDAG;
+  MaskedScatterSDNode(unsigned Order, DebugLoc dl,ArrayRef<SDValue> Operands,
+                      SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
+    : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT,
+                                MMO) {
+    assert(getMask().getValueType().getVectorNumElements() == 
+           getValue().getValueType().getVectorNumElements() && 
+           "Vector width mismatch between mask and data");
+    assert(getMask().getValueType().getScalarType() == MVT::i1 && 
+           "Vector width mismatch between mask and data");
+  }
+
+  static bool classof(const SDNode *N) {
+    return N->getOpcode() == ISD::MSCATTER;
+  }
+};
+
+/// An SDNode that represents everything that will be needed
 /// to construct a MachineInstr. These nodes are created during the
 /// instruction selection proper phase.
-///
 class MachineSDNode : public SDNode {
 public:
   typedef MachineMemOperand **mmo_iterator;
@@ -2025,11 +2118,11 @@ private:
   MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs)
     : SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {}
 
-  /// LocalOperands - Operands for this instruction, if they fit here. If
+  /// Operands for this instruction, if they fit here. If
   /// they don't, this field is unused.
   SDUse LocalOperands[4];
 
-  /// MemRefs - Memory reference descriptions for this instruction.
+  /// Memory reference descriptions for this instruction.
   mmo_iterator MemRefs;
   mmo_iterator MemRefsEnd;
 
@@ -2038,7 +2131,7 @@ public:
   mmo_iterator memoperands_end() const { return MemRefsEnd; }
   bool memoperands_empty() const { return MemRefsEnd == MemRefs; }
 
-  /// setMemRefs - Assign this MachineSDNodes's memory reference descriptor
+  /// Assign this MachineSDNodes's memory reference descriptor
   /// list. This does not transfer ownership.
   void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) {
     for (mmo_iterator MMI = NewMemRefs, MME = NewMemRefsEnd; MMI != MME; ++MMI)
@@ -2064,12 +2157,6 @@ public:
   }
   bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
 
-  const SDNodeIterator &operator=(const SDNodeIterator &I) {
-    assert(I.Node == Node && "Cannot assign iterators to two different nodes!");
-    Operand = I.Operand;
-    return *this;
-  }
-
   pointer operator*() const {
     return Node->getOperand(Operand).getNode();
   }
@@ -2109,60 +2196,51 @@ template <> struct GraphTraits<SDNode*> {
   }
 };
 
-/// LargestSDNode - The largest SDNode class.
-///
-typedef AtomicSDNode LargestSDNode;
+/// The largest SDNode class.
+typedef MaskedGatherScatterSDNode LargestSDNode;
 
-/// MostAlignedSDNode - The SDNode class with the greatest alignment
-/// requirement.
-///
+/// The SDNode class with the greatest alignment requirement.
 typedef GlobalAddressSDNode MostAlignedSDNode;
 
 namespace ISD {
-  /// isNormalLoad - Returns true if the specified node is a non-extending
-  /// and unindexed load.
+  /// Returns true if the specified node is a non-extending and unindexed load.
   inline bool isNormalLoad(const SDNode *N) {
     const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
     return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
       Ld->getAddressingMode() == ISD::UNINDEXED;
   }
 
-  /// isNON_EXTLoad - Returns true if the specified node is a non-extending
-  /// load.
+  /// Returns true if the specified node is a non-extending load.
   inline bool isNON_EXTLoad(const SDNode *N) {
     return isa<LoadSDNode>(N) &&
       cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
   }
 
-  /// isEXTLoad - Returns true if the specified node is a EXTLOAD.
-  ///
+  /// Returns true if the specified node is a EXTLOAD.
   inline bool isEXTLoad(const SDNode *N) {
     return isa<LoadSDNode>(N) &&
       cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
   }
 
-  /// isSEXTLoad - Returns true if the specified node is a SEXTLOAD.
-  ///
+  /// Returns true if the specified node is a SEXTLOAD.
   inline bool isSEXTLoad(const SDNode *N) {
     return isa<LoadSDNode>(N) &&
       cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
   }
 
-  /// isZEXTLoad - Returns true if the specified node is a ZEXTLOAD.
-  ///
+  /// Returns true if the specified node is a ZEXTLOAD.
   inline bool isZEXTLoad(const SDNode *N) {
     return isa<LoadSDNode>(N) &&
       cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
   }
 
-  /// isUNINDEXEDLoad - Returns true if the specified node is an unindexed load.
-  ///
+  /// Returns true if the specified node is an unindexed load.
   inline bool isUNINDEXEDLoad(const SDNode *N) {
     return isa<LoadSDNode>(N) &&
       cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
   }
 
-  /// isNormalStore - Returns true if the specified node is a non-truncating
+  /// Returns true if the specified node is a non-truncating
   /// and unindexed store.
   inline bool isNormalStore(const SDNode *N) {
     const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
@@ -2170,20 +2248,17 @@ namespace ISD {
       St->getAddressingMode() == ISD::UNINDEXED;
   }
 
-  /// isNON_TRUNCStore - Returns true if the specified node is a non-truncating
-  /// store.
+  /// Returns true if the specified node is a non-truncating store.
   inline bool isNON_TRUNCStore(const SDNode *N) {
     return isa<StoreSDNode>(N) && !cast<StoreSDNode>(N)->isTruncatingStore();
   }
 
-  /// isTRUNCStore - Returns true if the specified node is a truncating
-  /// store.
+  /// Returns true if the specified node is a truncating store.
   inline bool isTRUNCStore(const SDNode *N) {
     return isa<StoreSDNode>(N) && cast<StoreSDNode>(N)->isTruncatingStore();
   }
 
-  /// isUNINDEXEDStore - Returns true if the specified node is an
-  /// unindexed store.
+  /// Returns true if the specified node is an unindexed store.
   inline bool isUNINDEXEDStore(const SDNode *N) {
     return isa<StoreSDNode>(N) &&
       cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h
index 00bb22b91d32c..9d6d6f5b1be0e 100644
--- a/include/llvm/CodeGen/SlotIndexes.h
+++ b/include/llvm/CodeGen/SlotIndexes.h
@@ -162,7 +162,7 @@ namespace llvm {
     }
 
     /// Return true for a valid index.
-    LLVM_EXPLICIT operator bool() const { return isValid(); }
+    explicit operator bool() const { return isValid(); }
 
     /// Print this index to the given raw_ostream.
     void print(raw_ostream &os) const;
diff --git a/include/llvm/CodeGen/StackMapLivenessAnalysis.h b/include/llvm/CodeGen/StackMapLivenessAnalysis.h
deleted file mode 100644
index f67a6e95191d8..0000000000000
--- a/include/llvm/CodeGen/StackMapLivenessAnalysis.h
+++ /dev/null
@@ -1,64 +0,0 @@
-//===--- StackMapLivenessAnalysis - StackMap Liveness Analysis --*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This pass calculates the liveness for each basic block in a function and
-// attaches the register live-out information to a patchpoint intrinsic (if
-// present).
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_STACKMAPLIVENESSANALYSIS_H
-#define LLVM_CODEGEN_STACKMAPLIVENESSANALYSIS_H
-
-#include "llvm/CodeGen/LivePhysRegs.h"
-#include "llvm/CodeGen/MachineFunctionPass.h"
-
-
-namespace llvm {
-
-/// \brief This pass calculates the liveness information for each basic block in
-/// a function and attaches the register live-out information to a patchpoint
-/// intrinsic if present.
-///
-/// This pass can be disabled via the -enable-patchpoint-liveness=false flag.
-/// The pass skips functions that don't have any patchpoint intrinsics. The
-/// information provided by this pass is optional and not required by the
-/// aformentioned intrinsic to function.
-class StackMapLiveness : public MachineFunctionPass {
-  MachineFunction *MF;
-  const TargetRegisterInfo *TRI;
-  LivePhysRegs LiveRegs;
-public:
-  static char ID;
-
-  /// \brief Default construct and initialize the pass.
-  StackMapLiveness();
-
-  /// \brief Tell the pass manager which passes we depend on and what
-  /// information we preserve.
-  void getAnalysisUsage(AnalysisUsage &AU) const override;
-
-  /// \brief Calculate the liveness information for the given machine function.
-  bool runOnMachineFunction(MachineFunction &MF) override;
-
-private:
-  /// \brief Performs the actual liveness calculation for the function.
-  bool calculateLiveness();
-
-  /// \brief Add the current register live set to the instruction.
-  void addLiveOutSetToMI(MachineInstr &MI);
-
-  /// \brief Create a register mask and initialize it with the registers from
-  /// the register live set.
-  uint32_t *createRegisterMask() const;
-};
-
-} // llvm namespace
-
-#endif
diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h
index 4e48afe140042..46a773f74aaca 100644
--- a/include/llvm/CodeGen/StackMaps.h
+++ b/include/llvm/CodeGen/StackMaps.h
@@ -14,6 +14,7 @@
 #include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/Support/Debug.h"
 #include <map>
 #include <vector>
 
@@ -84,15 +85,23 @@ public:
 /// MI-level Statepoint operands
 ///
 /// Statepoint operands take the form:
-///   <num call arguments>, <call target>, [call arguments],
-///   <StackMaps::ConstantOp>, <flags>,
+///   <id>, <num patch bytes >, <num call arguments>, <call target>,
+///   [call arguments], <StackMaps::ConstantOp>, <calling convention>,
+///   <StackMaps::ConstantOp>, <statepoint flags>,
 ///   <StackMaps::ConstantOp>, <num other args>, [other args],
 ///   [gc values]
 class StatepointOpers {
 private:
+  // These values are aboolute offsets into the operands of the statepoint
+  // instruction.
+  enum { IDPos, NBytesPos, NCallArgsPos, CallTargetPos, MetaEnd };
+
+  // These values are relative offests from the start of the statepoint meta
+  // arguments (i.e. the end of the call arguments).
   enum {
-    NCallArgsPos = 0,
-    CallTargetPos = 1
+    CCOffset = 1,
+    FlagsOffset = 3,
+    NumVMSArgsOffset = 5
   };
 
 public:
@@ -100,22 +109,17 @@ public:
     MI(MI) { }
 
   /// Get starting index of non call related arguments
-  /// (statepoint flags, vm state and gc state).
+  /// (calling convention, statepoint flags, vm state and gc state).
   unsigned getVarIdx() const {
-    return MI->getOperand(NCallArgsPos).getImm() + 2;
+    return MI->getOperand(NCallArgsPos).getImm() + MetaEnd;
   }
 
-  /// Returns the index of the operand containing the number of non-gc non-call
-  /// arguments. 
-  unsigned getNumVMSArgsIdx() const {
-    return getVarIdx() + 3;
-  }
+  /// Return the ID for the given statepoint.
+  uint64_t getID() const { return MI->getOperand(IDPos).getImm(); }
 
-  /// Returns the number of non-gc non-call arguments attached to the
-  /// statepoint.  Note that this is the number of arguments, not the number of
-  /// operands required to represent those arguments.
-  unsigned getNumVMSArgs() const {
-    return MI->getOperand(getNumVMSArgsIdx()).getImm();
+  /// Return the number of patchable bytes the given statepoint should emit.
+  uint32_t getNumPatchBytes() const {
+    return MI->getOperand(NBytesPos).getImm();
   }
 
   /// Returns the target of the underlying call.
@@ -245,7 +249,10 @@ private:
   void emitConstantPoolEntries(MCStreamer &OS);
 
   /// \brief Emit the callsite info for each stackmap/patchpoint intrinsic call.
-  void emitCallsiteEntries(MCStreamer &OS, const TargetRegisterInfo *TRI);
+  void emitCallsiteEntries(MCStreamer &OS);
+
+  void print(raw_ostream &OS);
+  void debug() { print(dbgs()); }
 };
 
 }
diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index 9209e1c67c1bb..9a1b63f91bd46 100644
--- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -34,25 +34,34 @@ namespace llvm {
 
 class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
   bool UseInitArray;
+  mutable unsigned NextUniqueID = 0;
 
 public:
-  virtual ~TargetLoweringObjectFileELF() {}
+  TargetLoweringObjectFileELF() : UseInitArray(false) {}
+
+  ~TargetLoweringObjectFileELF() override {}
 
   void emitPersonalityValue(MCStreamer &Streamer, const TargetMachine &TM,
                             const MCSymbol *Sym) const override;
 
   /// Given a constant with the SectionKind, return a section that it should be
   /// placed in.
-  const MCSection *getSectionForConstant(SectionKind Kind,
-                                         const Constant *C) const override;
+  MCSection *getSectionForConstant(SectionKind Kind,
+                                   const Constant *C) const override;
+
+  MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
+
+  MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                    Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
-  const MCSection *getExplicitSectionGlobal(const GlobalValue *GV,
-                                        SectionKind Kind, Mangler &Mang,
-                                        const TargetMachine &TM) const override;
+  MCSection *getSectionForJumpTable(const Function &F, Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
-  const MCSection *SelectSectionForGlobal(const GlobalValue *GV,
-                                        SectionKind Kind, Mangler &Mang,
-                                        const TargetMachine &TM) const override;
+  bool shouldPutJumpTableInFunctionSection(bool UsesLabelDifference,
+                                           const Function &F) const override;
 
   /// Return an MCExpr to use for a reference to the specified type info global
   /// variable from exception handling information.
@@ -68,17 +77,18 @@ public:
                                     MachineModuleInfo *MMI) const override;
 
   void InitializeELF(bool UseInitArray_);
-  const MCSection *getStaticCtorSection(unsigned Priority,
-                                        const MCSymbol *KeySym) const override;
-  const MCSection *getStaticDtorSection(unsigned Priority,
-                                        const MCSymbol *KeySym) const override;
+  MCSection *getStaticCtorSection(unsigned Priority,
+                                  const MCSymbol *KeySym) const override;
+  MCSection *getStaticDtorSection(unsigned Priority,
+                                  const MCSymbol *KeySym) const override;
 };
 
 
 
 class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile {
 public:
-  virtual ~TargetLoweringObjectFileMachO() {}
+  ~TargetLoweringObjectFileMachO() override {}
+  TargetLoweringObjectFileMachO();
 
   /// Extract the dependent library name from a linker option string. Returns
   /// StringRef() if the option does not specify a library.
@@ -89,18 +99,16 @@ public:
                        ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
                        Mangler &Mang, const TargetMachine &TM) const override;
 
-  const MCSection *
-    SelectSectionForGlobal(const GlobalValue *GV,
-                           SectionKind Kind, Mangler &Mang,
-                           const TargetMachine &TM) const override;
+  MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                    Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
-  const MCSection *
-    getExplicitSectionGlobal(const GlobalValue *GV,
-                             SectionKind Kind, Mangler &Mang,
-                             const TargetMachine &TM) const override;
+  MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
 
-  const MCSection *getSectionForConstant(SectionKind Kind,
-                                         const Constant *C) const override;
+  MCSection *getSectionForConstant(SectionKind Kind,
+                                   const Constant *C) const override;
 
   /// The mach-o version of this method defaults to returning a stub reference.
   const MCExpr *
@@ -113,23 +121,34 @@ public:
   MCSymbol *getCFIPersonalitySymbol(const GlobalValue *GV, Mangler &Mang,
                                     const TargetMachine &TM,
                                     MachineModuleInfo *MMI) const override;
+
+  /// Get MachO PC relative GOT entry relocation
+  const MCExpr *getIndirectSymViaGOTPCRel(const MCSymbol *Sym,
+                                          const MCValue &MV, int64_t Offset,
+                                          MachineModuleInfo *MMI,
+                                          MCStreamer &Streamer) const override;
 };
 
 
 
 class TargetLoweringObjectFileCOFF : public TargetLoweringObjectFile {
 public:
-  virtual ~TargetLoweringObjectFileCOFF() {}
+  ~TargetLoweringObjectFileCOFF() override {}
+
+  MCSection *getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
+                                      Mangler &Mang,
+                                      const TargetMachine &TM) const override;
+
+  MCSection *SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
+                                    Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
-  const MCSection *
-    getExplicitSectionGlobal(const GlobalValue *GV,
-                             SectionKind Kind, Mangler &Mang,
-                             const TargetMachine &TM) const override;
+  void getNameWithPrefix(SmallVectorImpl<char> &OutName, const GlobalValue *GV,
+                         bool CannotUsePrivateLabel, Mangler &Mang,
+                         const TargetMachine &TM) const override;
 
-  const MCSection *
-    SelectSectionForGlobal(const GlobalValue *GV,
-                           SectionKind Kind, Mangler &Mang,
-                           const TargetMachine &TM) const override;
+  MCSection *getSectionForJumpTable(const Function &F, Mangler &Mang,
+                                    const TargetMachine &TM) const override;
 
   /// Extract the dependent library name from a linker option string. Returns
   /// StringRef() if the option does not specify a library.
@@ -141,10 +160,10 @@ public:
                        ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
                        Mangler &Mang, const TargetMachine &TM) const override;
 
-  const MCSection *getStaticCtorSection(unsigned Priority,
-                                        const MCSymbol *KeySym) const override;
-  const MCSection *getStaticDtorSection(unsigned Priority,
-                                        const MCSymbol *KeySym) const override;
+  MCSection *getStaticCtorSection(unsigned Priority,
+                                  const MCSymbol *KeySym) const override;
+  MCSection *getStaticDtorSection(unsigned Priority,
+                                  const MCSymbol *KeySym) const override;
 };
 
 } // end namespace llvm
diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h
index b6136665b968c..751fac411ce6b 100644
--- a/include/llvm/CodeGen/TargetSchedule.h
+++ b/include/llvm/CodeGen/TargetSchedule.h
@@ -40,6 +40,9 @@ class TargetSchedModel {
   SmallVector<unsigned, 16> ResourceFactors;
   unsigned MicroOpFactor; // Multiply to normalize microops to resource units.
   unsigned ResourceLCM;   // Resource units per cycle. Latency normalization factor.
+
+  unsigned computeInstrLatency(const MCSchedClassDesc &SCDesc) const;
+
 public:
   TargetSchedModel(): SchedModel(MCSchedModel::GetDefaultSchedModel()), STI(nullptr), TII(nullptr) {}
 
diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h
index 4e93940e223ef..e1a9fd38290ba 100644
--- a/include/llvm/CodeGen/ValueTypes.h
+++ b/include/llvm/CodeGen/ValueTypes.h
@@ -34,10 +34,9 @@ namespace llvm {
     Type *LLVMTy;
 
   public:
-    EVT() : V((MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE)),
-            LLVMTy(nullptr) {}
-    EVT(MVT::SimpleValueType SVT) : V(SVT), LLVMTy(nullptr) { }
-    EVT(MVT S) : V(S), LLVMTy(nullptr) {}
+    LLVM_CONSTEXPR EVT() : V(MVT::INVALID_SIMPLE_VALUE_TYPE), LLVMTy(nullptr) {}
+    LLVM_CONSTEXPR EVT(MVT::SimpleValueType SVT) : V(SVT), LLVMTy(nullptr) {}
+    LLVM_CONSTEXPR EVT(MVT S) : V(S), LLVMTy(nullptr) {}
 
     bool operator==(EVT VT) const {
       return !(*this != VT);
diff --git a/include/llvm/CodeGen/ValueTypes.td b/include/llvm/CodeGen/ValueTypes.td
index b5fa0e8c6a231..2b30f14f902c6 100644
--- a/include/llvm/CodeGen/ValueTypes.td
+++ b/include/llvm/CodeGen/ValueTypes.td
@@ -62,25 +62,26 @@ def v2i64  : ValueType<128, 38>;   //  2 x i64 vector value
 def v4i64  : ValueType<256, 39>;   //  4 x i64 vector value
 def v8i64  : ValueType<512, 40>;   //  8 x i64 vector value
 def v16i64 : ValueType<1024,41>;   // 16 x i64 vector value
+def v1i128 : ValueType<128, 42>;   //  1 x i128 vector value
 
-def v2f16  : ValueType<32 , 42>;   //  2 x f16 vector value
-def v4f16  : ValueType<64 , 43>;   //  4 x f16 vector value
-def v8f16  : ValueType<128, 44>;   //  8 x f16 vector value
-def v1f32  : ValueType<32 , 45>;   //  1 x f32 vector value
-def v2f32  : ValueType<64 , 46>;   //  2 x f32 vector value
-def v4f32  : ValueType<128, 47>;   //  4 x f32 vector value
-def v8f32  : ValueType<256, 48>;   //  8 x f32 vector value
-def v16f32 : ValueType<512, 49>;   // 16 x f32 vector value
-def v1f64  : ValueType<64, 50>;    //  1 x f64 vector value
-def v2f64  : ValueType<128, 51>;   //  2 x f64 vector value
-def v4f64  : ValueType<256, 52>;   //  4 x f64 vector value
-def v8f64  : ValueType<512, 53>;   //  8 x f64 vector value
+def v2f16  : ValueType<32 , 43>;   //  2 x f16 vector value
+def v4f16  : ValueType<64 , 44>;   //  4 x f16 vector value
+def v8f16  : ValueType<128, 45>;   //  8 x f16 vector value
+def v1f32  : ValueType<32 , 46>;   //  1 x f32 vector value
+def v2f32  : ValueType<64 , 47>;   //  2 x f32 vector value
+def v4f32  : ValueType<128, 48>;   //  4 x f32 vector value
+def v8f32  : ValueType<256, 49>;   //  8 x f32 vector value
+def v16f32 : ValueType<512, 50>;   // 16 x f32 vector value
+def v1f64  : ValueType<64, 51>;    //  1 x f64 vector value
+def v2f64  : ValueType<128, 52>;   //  2 x f64 vector value
+def v4f64  : ValueType<256, 53>;   //  4 x f64 vector value
+def v8f64  : ValueType<512, 54>;   //  8 x f64 vector value
 
 
-def x86mmx : ValueType<64 , 54>;   // X86 MMX value
-def FlagVT : ValueType<0  , 55>;   // Pre-RA sched glue
-def isVoid : ValueType<0  , 56>;   // Produces no value
-def untyped: ValueType<8  , 57>;   // Produces an untyped value
+def x86mmx : ValueType<64 , 55>;   // X86 MMX value
+def FlagVT : ValueType<0  , 56>;   // Pre-RA sched glue
+def isVoid : ValueType<0  , 57>;   // Produces no value
+def untyped: ValueType<8  , 58>;   // Produces an untyped value
 def MetadataVT: ValueType<0, 250>; // Metadata
 
 // Pseudo valuetype mapped to the current pointer size to any address space.
@@ -98,3 +99,6 @@ def iAny   : ValueType<0  , 254>;
 
 // Pseudo valuetype mapped to the current pointer size.
 def iPTR   : ValueType<0  , 255>;
+
+// Pseudo valuetype to represent "any type of any size".
+def Any    : ValueType<0  , 256>;
diff --git a/include/llvm/CodeGen/VirtRegMap.h b/include/llvm/CodeGen/VirtRegMap.h
index eceb8755763e2..d7e92094877d1 100644
--- a/include/llvm/CodeGen/VirtRegMap.h
+++ b/include/llvm/CodeGen/VirtRegMap.h
@@ -63,8 +63,8 @@ namespace llvm {
     /// createSpillSlot - Allocate a spill slot for RC from MFI.
     unsigned createSpillSlot(const TargetRegisterClass *RC);
 
-    VirtRegMap(const VirtRegMap&) LLVM_DELETED_FUNCTION;
-    void operator=(const VirtRegMap&) LLVM_DELETED_FUNCTION;
+    VirtRegMap(const VirtRegMap&) = delete;
+    void operator=(const VirtRegMap&) = delete;
 
   public:
     static char ID;
diff --git a/include/llvm/CodeGen/WinEHFuncInfo.h b/include/llvm/CodeGen/WinEHFuncInfo.h
new file mode 100644
index 0000000000000..e2644edd4d12a
--- /dev/null
+++ b/include/llvm/CodeGen/WinEHFuncInfo.h
@@ -0,0 +1,157 @@
+//===-- llvm/CodeGen/WinEHFuncInfo.h ----------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Data structures and associated state for Windows exception handling schemes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_WINEHFUNCINFO_H
+#define LLVM_CODEGEN_WINEHFUNCINFO_H
+
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/TinyPtrVector.h"
+#include "llvm/ADT/DenseMap.h"
+
+namespace llvm {
+class BasicBlock;
+class Constant;
+class Function;
+class GlobalVariable;
+class IntrinsicInst;
+class LandingPadInst;
+class MCSymbol;
+class Value;
+
+enum ActionType { Catch, Cleanup };
+
+class ActionHandler {
+public:
+  ActionHandler(BasicBlock *BB, ActionType Type)
+      : StartBB(BB), Type(Type), EHState(-1), HandlerBlockOrFunc(nullptr) {}
+
+  ActionType getType() const { return Type; }
+  BasicBlock *getStartBlock() const { return StartBB; }
+
+  bool hasBeenProcessed() { return HandlerBlockOrFunc != nullptr; }
+
+  void setHandlerBlockOrFunc(Constant *F) { HandlerBlockOrFunc = F; }
+  Constant *getHandlerBlockOrFunc() { return HandlerBlockOrFunc; }
+
+  void setEHState(int State) { EHState = State; }
+  int getEHState() const { return EHState; }
+
+private:
+  BasicBlock *StartBB;
+  ActionType Type;
+  int EHState;
+
+  // Can be either a BlockAddress or a Function depending on the EH personality.
+  Constant *HandlerBlockOrFunc;
+};
+
+class CatchHandler : public ActionHandler {
+public:
+  CatchHandler(BasicBlock *BB, Constant *Selector, BasicBlock *NextBB)
+      : ActionHandler(BB, ActionType::Catch), Selector(Selector),
+      NextBB(NextBB), ExceptionObjectVar(nullptr),
+      ExceptionObjectIndex(-1) {}
+
+  // Method for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const ActionHandler *H) {
+    return H->getType() == ActionType::Catch;
+  }
+
+  Constant *getSelector() const { return Selector; }
+  BasicBlock *getNextBB() const { return NextBB; }
+
+  const Value *getExceptionVar() { return ExceptionObjectVar; }
+  TinyPtrVector<BasicBlock *> &getReturnTargets() { return ReturnTargets; }
+
+  void setExceptionVar(const Value *Val) { ExceptionObjectVar = Val; }
+  void setExceptionVarIndex(int Index) { ExceptionObjectIndex = Index;  }
+  int getExceptionVarIndex() const { return ExceptionObjectIndex; }
+  void setReturnTargets(TinyPtrVector<BasicBlock *> &Targets) {
+    ReturnTargets = Targets;
+  }
+
+private:
+  Constant *Selector;
+  BasicBlock *NextBB;
+  // While catch handlers are being outlined the ExceptionObjectVar field will
+  // be populated with the instruction in the parent frame that corresponds
+  // to the exception object (or nullptr if the catch does not use an
+  // exception object) and the ExceptionObjectIndex field will be -1.
+  // When the parseEHActions function is called to populate a vector of
+  // instances of this class, the ExceptionObjectVar field will be nullptr
+  // and the ExceptionObjectIndex will be the index of the exception object in
+  // the parent function's frameescape block.
+  const Value *ExceptionObjectVar;
+  int ExceptionObjectIndex;
+  TinyPtrVector<BasicBlock *> ReturnTargets;
+};
+
+class CleanupHandler : public ActionHandler {
+public:
+  CleanupHandler(BasicBlock *BB) : ActionHandler(BB, ActionType::Cleanup) {}
+
+  // Method for support type inquiry through isa, cast, and dyn_cast:
+  static inline bool classof(const ActionHandler *H) {
+    return H->getType() == ActionType::Cleanup;
+  }
+};
+
+void parseEHActions(const IntrinsicInst *II,
+                    SmallVectorImpl<std::unique_ptr<ActionHandler>> &Actions);
+
+// The following structs respresent the .xdata for functions using C++
+// exceptions on Windows.
+
+struct WinEHUnwindMapEntry {
+  int ToState;
+  Function *Cleanup;
+};
+
+struct WinEHHandlerType {
+  int Adjectives;
+  GlobalVariable *TypeDescriptor;
+  int CatchObjRecoverIdx;
+  Function *Handler;
+};
+
+struct WinEHTryBlockMapEntry {
+  int TryLow;
+  int TryHigh;
+  SmallVector<WinEHHandlerType, 1> HandlerArray;
+};
+
+struct WinEHFuncInfo {
+  DenseMap<const Function *, const LandingPadInst *> RootLPad;
+  DenseMap<const Function *, const InvokeInst *> LastInvoke;
+  DenseMap<const Function *, int> HandlerEnclosedState;
+  DenseMap<const Function *, bool> LastInvokeVisited;
+  DenseMap<const LandingPadInst *, int> LandingPadStateMap;
+  DenseMap<const Function *, int> CatchHandlerParentFrameObjIdx;
+  DenseMap<const Function *, int> CatchHandlerParentFrameObjOffset;
+  DenseMap<const Function *, int> CatchHandlerMaxState;
+  DenseMap<const Function *, int> HandlerBaseState;
+  SmallVector<WinEHUnwindMapEntry, 4> UnwindMap;
+  SmallVector<WinEHTryBlockMapEntry, 4> TryBlockMap;
+  SmallVector<std::pair<MCSymbol *, int>, 4> IPToStateList;
+  int UnwindHelpFrameIdx;
+  int UnwindHelpFrameOffset;
+
+  unsigned NumIPToStateFuncsVisited;
+
+  WinEHFuncInfo()
+      : UnwindHelpFrameIdx(INT_MAX), UnwindHelpFrameOffset(-1),
+        NumIPToStateFuncsVisited(0) {}
+};
+
+}
+#endif // LLVM_CODEGEN_WINEHFUNCINFO_H