vendor/llvm/llvm-release_350-r216957

86 files changed, 5887 insertions, 3971 deletions

diff --git a/include/llvm/CodeGen/Analysis.h b/include/llvm/CodeGen/Analysis.h
index b2cc7047b30b..b791ba09adaf 100644
--- a/include/llvm/CodeGen/Analysis.h
+++ b/include/llvm/CodeGen/Analysis.h
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file declares several CodeGen-specific LLVM IR analysis utilties.
+// This file declares several CodeGen-specific LLVM IR analysis utilities.
 //
 //===----------------------------------------------------------------------===//
 
@@ -17,19 +17,19 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/ISDOpcodes.h"
-#include "llvm/CodeGen/ValueTypes.h"
+#include "llvm/IR/CallSite.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
-#include "llvm/Support/CallSite.h"
 
 namespace llvm {
-
 class GlobalVariable;
-class TargetLowering;
 class TargetLoweringBase;
+class TargetLowering;
+class TargetMachine;
 class SDNode;
 class SDValue;
 class SelectionDAG;
+struct EVT;
 
 /// ComputeLinearIndex - Given an LLVM IR aggregate type and a sequence
 /// of insertvalue or extractvalue indices that identify a member, return
@@ -55,7 +55,7 @@ inline unsigned ComputeLinearIndex(Type *Ty,
 ///
 void ComputeValueVTs(const TargetLowering &TLI, Type *Ty,
                      SmallVectorImpl<EVT> &ValueVTs,
-                     SmallVectorImpl<uint64_t> *Offsets = 0,
+                     SmallVectorImpl<uint64_t> *Offsets = nullptr,
                      uint64_t StartingOffset = 0);
 
 /// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V.
@@ -87,7 +87,7 @@ ISD::CondCode getICmpCondCode(ICmpInst::Predicate Pred);
 /// between it and the return.
 ///
 /// This function only tests target-independent requirements.
-bool isInTailCallPosition(ImmutableCallSite CS, const TargetLowering &TLI);
+bool isInTailCallPosition(ImmutableCallSite CS, const TargetMachine &TM);
 
 /// Test if given that the input instruction is in the tail call position if the
 /// return type or any attributes of the function will inhibit tail call
diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h
index 4bda0f1603ac..e1c9a14c9009 100644
--- a/include/llvm/CodeGen/AsmPrinter.h
+++ b/include/llvm/CodeGen/AsmPrinter.h
@@ -16,486 +16,496 @@
 #ifndef LLVM_CODEGEN_ASMPRINTER_H
 #define LLVM_CODEGEN_ASMPRINTER_H
 
+#include "llvm/ADT/Twine.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
 
 namespace llvm {
-  class BlockAddress;
-  class GCStrategy;
-  class Constant;
-  class ConstantArray;
-  class GCMetadataPrinter;
-  class GlobalValue;
-  class GlobalVariable;
-  class MachineBasicBlock;
-  class MachineFunction;
-  class MachineInstr;
-  class MachineLocation;
-  class MachineLoopInfo;
-  class MachineLoop;
-  class MachineConstantPoolValue;
-  class MachineJumpTableInfo;
-  class MachineModuleInfo;
-  class MCAsmInfo;
-  class MCCFIInstruction;
-  class MCContext;
-  class MCInstrInfo;
-  class MCSection;
-  class MCStreamer;
-  class MCSymbol;
-  class MDNode;
-  class DwarfDebug;
-  class DwarfException;
-  class Mangler;
-  class TargetLoweringObjectFile;
-  class DataLayout;
-  class TargetMachine;
+class AsmPrinterHandler;
+class BlockAddress;
+class ByteStreamer;
+class GCStrategy;
+class Constant;
+class ConstantArray;
+class GCMetadataPrinter;
+class GlobalValue;
+class GlobalVariable;
+class MachineBasicBlock;
+class MachineFunction;
+class MachineInstr;
+class MachineLocation;
+class MachineLoopInfo;
+class MachineLoop;
+class MachineConstantPoolValue;
+class MachineJumpTableInfo;
+class MachineModuleInfo;
+class MCAsmInfo;
+class MCCFIInstruction;
+class MCContext;
+class MCInst;
+class MCInstrInfo;
+class MCSection;
+class MCStreamer;
+class MCSubtargetInfo;
+class MCSymbol;
+class MDNode;
+class DwarfDebug;
+class Mangler;
+class TargetLoweringObjectFile;
+class DataLayout;
+class TargetMachine;
 
-  /// AsmPrinter - This class is intended to be used as a driving class for all
-  /// asm writers.
-  class AsmPrinter : public MachineFunctionPass {
-  public:
-    /// Target machine description.
-    ///
-    TargetMachine &TM;
+/// This class is intended to be used as a driving class for all asm writers.
+class AsmPrinter : public MachineFunctionPass {
+public:
+  /// Target machine description.
+  ///
+  TargetMachine &TM;
 
-    /// Target Asm Printer information.
-    ///
-    const MCAsmInfo *MAI;
+  /// Target Asm Printer information.
+  ///
+  const MCAsmInfo *MAI;
 
-    const MCInstrInfo *MII;
-    /// OutContext - This is the context for the output file that we are
-    /// streaming.  This owns all of the global MC-related objects for the
-    /// generated translation unit.
-    MCContext &OutContext;
+  const MCInstrInfo *MII;
+  /// This is the context for the output file that we are streaming. This owns
+  /// all of the global MC-related objects for the generated translation unit.
+  MCContext &OutContext;
 
-    /// OutStreamer - This is the MCStreamer object for the file we are
-    /// generating.  This contains the transient state for the current
-    /// translation unit that we are generating (such as the current section
-    /// etc).
-    MCStreamer &OutStreamer;
+  /// This is the MCStreamer object for the file we are generating. This
+  /// contains the transient state for the current translation unit that we are
+  /// generating (such as the current section etc).
+  MCStreamer &OutStreamer;
 
-    /// The current machine function.
-    const MachineFunction *MF;
+  /// The current machine function.
+  const MachineFunction *MF;
 
-    /// MMI - This is a pointer to the current MachineModuleInfo.
-    MachineModuleInfo *MMI;
+  /// This is a pointer to the current MachineModuleInfo.
+  MachineModuleInfo *MMI;
 
-    /// Name-mangler for global names.
-    ///
-    Mangler *Mang;
+  /// Name-mangler for global names.
+  ///
+  Mangler *Mang;
 
-    /// The symbol for the current function. This is recalculated at the
-    /// beginning of each call to runOnMachineFunction().
-    ///
-    MCSymbol *CurrentFnSym;
+  /// The symbol for the current function. This is recalculated at the beginning
+  /// of each call to runOnMachineFunction().
+  ///
+  MCSymbol *CurrentFnSym;
 
-    /// The symbol used to represent the start of the current function for the
-    /// purpose of calculating its size (e.g. using the .size directive). By
-    /// default, this is equal to CurrentFnSym.
-    MCSymbol *CurrentFnSymForSize;
+  /// The symbol used to represent the start of the current function for the
+  /// purpose of calculating its size (e.g. using the .size directive). By
+  /// default, this is equal to CurrentFnSym.
+  MCSymbol *CurrentFnSymForSize;
 
-  private:
-    // GCMetadataPrinters - The garbage collection metadata printer table.
-    void *GCMetadataPrinters;  // Really a DenseMap.
+private:
+  // The garbage collection metadata printer table.
+  void *GCMetadataPrinters; // Really a DenseMap.
 
-    /// VerboseAsm - Emit comments in assembly output if this is true.
-    ///
-    bool VerboseAsm;
-    static char ID;
+  /// Emit comments in assembly output if this is true.
+  ///
+  bool VerboseAsm;
+  static char ID;
 
-    /// If VerboseAsm is set, a pointer to the loop info for this
-    /// function.
-    MachineLoopInfo *LI;
+  /// If VerboseAsm is set, a pointer to the loop info for this function.
+  MachineLoopInfo *LI;
 
-    /// DD - If the target supports dwarf debug info, this pointer is non-null.
-    DwarfDebug *DD;
+  struct HandlerInfo {
+    AsmPrinterHandler *Handler;
+    const char *TimerName, *TimerGroupName;
+    HandlerInfo(AsmPrinterHandler *Handler, const char *TimerName,
+                const char *TimerGroupName)
+        : Handler(Handler), TimerName(TimerName),
+          TimerGroupName(TimerGroupName) {}
+  };
+  /// A vector of all debug/EH info emitters we should use. This vector
+  /// maintains ownership of the emitters.
+  SmallVector<HandlerInfo, 1> Handlers;
 
-    /// DE - If the target supports dwarf exception info, this pointer is
-    /// non-null.
-    DwarfException *DE;
+  /// If the target supports dwarf debug info, this pointer is non-null.
+  DwarfDebug *DD;
 
-  protected:
-    explicit AsmPrinter(TargetMachine &TM, MCStreamer &Streamer);
+protected:
+  explicit AsmPrinter(TargetMachine &TM, MCStreamer &Streamer);
 
-  public:
-    virtual ~AsmPrinter();
+public:
+  virtual ~AsmPrinter();
 
-    const DwarfDebug *getDwarfDebug() const { return DD; }
+  DwarfDebug *getDwarfDebug() { return DD; }
 
-    /// isVerbose - Return true if assembly output should contain comments.
-    ///
-    bool isVerbose() const { return VerboseAsm; }
+  /// Return true if assembly output should contain comments.
+  ///
+  bool isVerbose() const { return VerboseAsm; }
 
-    /// getFunctionNumber - Return a unique ID for the current function.
-    ///
-    unsigned getFunctionNumber() const;
+  /// Return a unique ID for the current function.
+  ///
+  unsigned getFunctionNumber() const;
 
-    /// getObjFileLowering - Return information about object file lowering.
-    const TargetLoweringObjectFile &getObjFileLowering() const;
+  /// Return information about object file lowering.
+  const TargetLoweringObjectFile &getObjFileLowering() const;
 
-    /// getDataLayout - Return information about data layout.
-    const DataLayout &getDataLayout() const;
+  /// Return information about data layout.
+  const DataLayout &getDataLayout() const;
 
-    /// getTargetTriple - Return the target triple string.
-    StringRef getTargetTriple() const;
+  /// Return information about subtarget.
+  const MCSubtargetInfo &getSubtargetInfo() const;
 
-    /// getCurrentSection() - Return the current section we are emitting to.
-    const MCSection *getCurrentSection() const;
+  void EmitToStreamer(MCStreamer &S, const MCInst &Inst);
 
-    MCSymbol *getSymbol(const GlobalValue *GV) const;
+  /// Return the target triple string.
+  StringRef getTargetTriple() const;
 
-    //===------------------------------------------------------------------===//
-    // MachineFunctionPass Implementation.
-    //===------------------------------------------------------------------===//
+  /// Return the current section we are emitting to.
+  const MCSection *getCurrentSection() const;
 
-    /// getAnalysisUsage - Record analysis usage.
-    ///
-    void getAnalysisUsage(AnalysisUsage &AU) const;
+  void getNameWithPrefix(SmallVectorImpl<char> &Name,
+                         const GlobalValue *GV) const;
 
-    /// doInitialization - Set up the AsmPrinter when we are working on a new
-    /// module.  If your pass overrides this, it must make sure to explicitly
-    /// call this implementation.
-    bool doInitialization(Module &M);
+  MCSymbol *getSymbol(const GlobalValue *GV) const;
 
-    /// doFinalization - Shut down the asmprinter.  If you override this in your
-    /// pass, you must make sure to call it explicitly.
-    bool doFinalization(Module &M);
+  //===------------------------------------------------------------------===//
+  // MachineFunctionPass Implementation.
+  //===------------------------------------------------------------------===//
 
-    /// runOnMachineFunction - Emit the specified function out to the
-    /// OutStreamer.
-    virtual bool runOnMachineFunction(MachineFunction &MF) {
-      SetupMachineFunction(MF);
-      EmitFunctionHeader();
-      EmitFunctionBody();
-      return false;
-    }
+  /// Record analysis usage.
+  ///
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
 
-    //===------------------------------------------------------------------===//
-    // Coarse grained IR lowering routines.
-    //===------------------------------------------------------------------===//
+  /// Set up the AsmPrinter when we are working on a new module. If your pass
+  /// overrides this, it must make sure to explicitly call this implementation.
+  bool doInitialization(Module &M) override;
 
-    /// SetupMachineFunction - This should be called when a new MachineFunction
-    /// is being processed from runOnMachineFunction.
-    void SetupMachineFunction(MachineFunction &MF);
+  /// Shut down the asmprinter. If you override this in your pass, you must make
+  /// sure to call it explicitly.
+  bool doFinalization(Module &M) override;
 
-    /// EmitFunctionHeader - This method emits the header for the current
-    /// function.
-    void EmitFunctionHeader();
+  /// Emit the specified function out to the OutStreamer.
+  bool runOnMachineFunction(MachineFunction &MF) override {
+    SetupMachineFunction(MF);
+    EmitFunctionHeader();
+    EmitFunctionBody();
+    return false;
+  }
 
-    /// EmitFunctionBody - This method emits the body and trailer for a
-    /// function.
-    void EmitFunctionBody();
+  //===------------------------------------------------------------------===//
+  // Coarse grained IR lowering routines.
+  //===------------------------------------------------------------------===//
 
-    void emitPrologLabel(const MachineInstr &MI);
+  /// This should be called when a new MachineFunction is being processed from
+  /// runOnMachineFunction.
+  void SetupMachineFunction(MachineFunction &MF);
 
-    enum CFIMoveType {
-      CFI_M_None,
-      CFI_M_EH,
-      CFI_M_Debug
-    };
-    CFIMoveType needsCFIMoves();
+  /// This method emits the header for the current function.
+  void EmitFunctionHeader();
 
-    bool needsSEHMoves();
+  /// This method emits the body and trailer for a function.
+  void EmitFunctionBody();
 
-    /// needsRelocationsForDwarfStringPool - Specifies whether the object format
-    /// expects to use relocations to refer to debug entries. Alternatively we
-    /// emit section offsets in bytes from the start of the string pool.
-    bool needsRelocationsForDwarfStringPool() const;
+  void emitCFIInstruction(const MachineInstr &MI);
 
-    /// EmitConstantPool - Print to the current output stream assembly
-    /// representations of the constants in the constant pool MCP. This is
-    /// used to print out constants which have been "spilled to memory" by
-    /// the code generator.
-    ///
-    virtual void EmitConstantPool();
+  enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug };
+  CFIMoveType needsCFIMoves();
 
-    /// EmitJumpTableInfo - Print assembly representations of the jump tables
-    /// used by the current function to the current output stream.
-    ///
-    void EmitJumpTableInfo();
+  bool needsSEHMoves();
 
-    /// EmitGlobalVariable - Emit the specified global variable to the .s file.
-    virtual void EmitGlobalVariable(const GlobalVariable *GV);
+  /// Print to the current output stream assembly representations of the
+  /// constants in the constant pool MCP. This is used to print out constants
+  /// which have been "spilled to memory" by the code generator.
+  ///
+  virtual void EmitConstantPool();
 
-    /// EmitSpecialLLVMGlobal - Check to see if the specified global is a
-    /// special global used by LLVM.  If so, emit it and return true, otherwise
-    /// do nothing and return false.
-    bool EmitSpecialLLVMGlobal(const GlobalVariable *GV);
+  /// Print assembly representations of the jump tables used by the current
+  /// function to the current output stream.
+  ///
+  void EmitJumpTableInfo();
 
-    /// EmitAlignment - Emit an alignment directive to the specified power of
-    /// two boundary.  For example, if you pass in 3 here, you will get an 8
-    /// byte alignment.  If a global value is specified, and if that global has
-    /// an explicit alignment requested, it will override the alignment request
-    /// if required for correctness.
-    ///
-    void EmitAlignment(unsigned NumBits, const GlobalValue *GV = 0) const;
+  /// Emit the specified global variable to the .s file.
+  virtual void EmitGlobalVariable(const GlobalVariable *GV);
 
-    /// EmitBasicBlockStart - This method prints the label for the specified
-    /// MachineBasicBlock, an alignment (if present) and a comment describing
-    /// it if appropriate.
-    void EmitBasicBlockStart(const MachineBasicBlock *MBB) const;
+  /// Check to see if the specified global is a special global used by LLVM. If
+  /// so, emit it and return true, otherwise do nothing and return false.
+  bool EmitSpecialLLVMGlobal(const GlobalVariable *GV);
 
-    /// \brief Print a general LLVM constant to the .s file.
-    void EmitGlobalConstant(const Constant *CV);
+  /// Emit an alignment directive to the specified power of two boundary. For
+  /// example, if you pass in 3 here, you will get an 8 byte alignment. If a
+  /// global value is specified, and if that global has an explicit alignment
+  /// requested, it will override the alignment request if required for
+  /// correctness.
+  ///
+  void EmitAlignment(unsigned NumBits, const GlobalObject *GO = nullptr) const;
 
+  /// This method prints the label for the specified MachineBasicBlock, an
+  /// alignment (if present) and a comment describing it if appropriate.
+  void EmitBasicBlockStart(const MachineBasicBlock &MBB) const;
 
-    //===------------------------------------------------------------------===//
-    // Overridable Hooks
-    //===------------------------------------------------------------------===//
+  /// \brief Print a general LLVM constant to the .s file.
+  void EmitGlobalConstant(const Constant *CV);
 
-    // Targets can, or in the case of EmitInstruction, must implement these to
-    // customize output.
+  //===------------------------------------------------------------------===//
+  // Overridable Hooks
+  //===------------------------------------------------------------------===//
 
-    /// EmitStartOfAsmFile - This virtual method can be overridden by targets
-    /// that want to emit something at the start of their file.
-    virtual void EmitStartOfAsmFile(Module &) {}
+  // Targets can, or in the case of EmitInstruction, must implement these to
+  // customize output.
 
-    /// EmitEndOfAsmFile - This virtual method can be overridden by targets that
-    /// want to emit something at the end of their file.
-    virtual void EmitEndOfAsmFile(Module &) {}
+  /// This virtual method can be overridden by targets that want to emit
+  /// something at the start of their file.
+  virtual void EmitStartOfAsmFile(Module &) {}
 
-    /// EmitFunctionBodyStart - Targets can override this to emit stuff before
-    /// the first basic block in the function.
-    virtual void EmitFunctionBodyStart() {}
+  /// This virtual method can be overridden by targets that want to emit
+  /// something at the end of their file.
+  virtual void EmitEndOfAsmFile(Module &) {}
 
-    /// EmitFunctionBodyEnd - Targets can override this to emit stuff after
-    /// the last basic block in the function.
-    virtual void EmitFunctionBodyEnd() {}
+  /// Targets can override this to emit stuff before the first basic block in
+  /// the function.
+  virtual void EmitFunctionBodyStart() {}
 
-    /// EmitInstruction - Targets should implement this to emit instructions.
-    virtual void EmitInstruction(const MachineInstr *) {
-      llvm_unreachable("EmitInstruction not implemented");
-    }
+  /// Targets can override this to emit stuff after the last basic block in the
+  /// function.
+  virtual void EmitFunctionBodyEnd() {}
 
-    virtual void EmitFunctionEntryLabel();
+  /// Targets should implement this to emit instructions.
+  virtual void EmitInstruction(const MachineInstr *) {
+    llvm_unreachable("EmitInstruction not implemented");
+  }
 
-    virtual void EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV);
+  /// Return the symbol for the specified constant pool entry.
+  virtual MCSymbol *GetCPISymbol(unsigned CPID) const;
 
-    /// EmitXXStructor - Targets can override this to change how global
-    /// constants that are part of a C++ static/global constructor list are
-    /// emitted.
-    virtual void EmitXXStructor(const Constant *CV) {
-      EmitGlobalConstant(CV);
-    }
+  virtual void EmitFunctionEntryLabel();
 
-    /// isBlockOnlyReachableByFallthough - Return true if the basic block has
-    /// exactly one predecessor and the control transfer mechanism between
-    /// the predecessor and this block is a fall-through.
-    virtual bool
-    isBlockOnlyReachableByFallthrough(const MachineBasicBlock *MBB) const;
+  virtual void EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV);
 
-    /// emitImplicitDef - Targets can override this to customize the output of
-    /// IMPLICIT_DEF instructions in verbose mode.
-    virtual void emitImplicitDef(const MachineInstr *MI) const;
+  /// Targets can override this to change how global constants that are part of
+  /// a C++ static/global constructor list are emitted.
+  virtual void EmitXXStructor(const Constant *CV) { EmitGlobalConstant(CV); }
 
-    //===------------------------------------------------------------------===//
-    // Symbol Lowering Routines.
-    //===------------------------------------------------------------------===//
-  public:
+  /// Return true if the basic block has exactly one predecessor and the control
+  /// transfer mechanism between the predecessor and this block is a
+  /// fall-through.
+  virtual bool
+  isBlockOnlyReachableByFallthrough(const MachineBasicBlock *MBB) const;
 
-    /// GetTempSymbol - Return the MCSymbol corresponding to the assembler
-    /// temporary label with the specified stem and unique ID.
-    MCSymbol *GetTempSymbol(StringRef Name, unsigned ID) const;
+  /// Targets can override this to customize the output of IMPLICIT_DEF
+  /// instructions in verbose mode.
+  virtual void emitImplicitDef(const MachineInstr *MI) const;
 
-    /// GetTempSymbol - Return an assembler temporary label with the specified
-    /// stem.
-    MCSymbol *GetTempSymbol(StringRef Name) const;
+  //===------------------------------------------------------------------===//
+  // Symbol Lowering Routines.
+  //===------------------------------------------------------------------===//
+public:
+  /// Return the MCSymbol corresponding to the assembler temporary label with
+  /// the specified stem and unique ID.
+  MCSymbol *GetTempSymbol(Twine Name, unsigned ID) const;
 
+  /// Return an assembler temporary label with the specified stem.
+  MCSymbol *GetTempSymbol(Twine Name) const;
 
-    /// GetSymbolWithGlobalValueBase - Return the MCSymbol for a symbol with
-    /// global value name as its base, with the specified suffix, and where the
-    /// symbol is forced to have private linkage if ForcePrivate is true.
-    MCSymbol *GetSymbolWithGlobalValueBase(const GlobalValue *GV,
-                                           StringRef Suffix,
-                                           bool ForcePrivate = true) const;
+  /// Return the MCSymbol for a private symbol with global value name as its
+  /// base, with the specified suffix.
+  MCSymbol *getSymbolWithGlobalValueBase(const GlobalValue *GV,
+                                         StringRef Suffix) const;
 
-    /// GetExternalSymbolSymbol - Return the MCSymbol for the specified
-    /// ExternalSymbol.
-    MCSymbol *GetExternalSymbolSymbol(StringRef Sym) const;
+  /// Return the MCSymbol for the specified ExternalSymbol.
+  MCSymbol *GetExternalSymbolSymbol(StringRef Sym) const;
 
-    /// GetCPISymbol - Return the symbol for the specified constant pool entry.
-    MCSymbol *GetCPISymbol(unsigned CPID) const;
+  /// Return the symbol for the specified jump table entry.
+  MCSymbol *GetJTISymbol(unsigned JTID, bool isLinkerPrivate = false) const;
 
-    /// GetJTISymbol - Return the symbol for the specified jump table entry.
-    MCSymbol *GetJTISymbol(unsigned JTID, bool isLinkerPrivate = false) const;
+  /// Return the symbol for the specified jump table .set
+  /// FIXME: privatize to AsmPrinter.
+  MCSymbol *GetJTSetSymbol(unsigned UID, unsigned MBBID) const;
 
-    /// GetJTSetSymbol - Return the symbol for the specified jump table .set
-    /// FIXME: privatize to AsmPrinter.
-    MCSymbol *GetJTSetSymbol(unsigned UID, unsigned MBBID) const;
+  /// Return the MCSymbol used to satisfy BlockAddress uses of the specified
+  /// basic block.
+  MCSymbol *GetBlockAddressSymbol(const BlockAddress *BA) const;
+  MCSymbol *GetBlockAddressSymbol(const BasicBlock *BB) const;
 
-    /// GetBlockAddressSymbol - Return the MCSymbol used to satisfy BlockAddress
-    /// uses of the specified basic block.
-    MCSymbol *GetBlockAddressSymbol(const BlockAddress *BA) const;
-    MCSymbol *GetBlockAddressSymbol(const BasicBlock *BB) const;
+  //===------------------------------------------------------------------===//
+  // Emission Helper Routines.
+  //===------------------------------------------------------------------===//
+public:
+  /// This is just convenient handler for printing offsets.
+  void printOffset(int64_t Offset, raw_ostream &OS) const;
 
-    //===------------------------------------------------------------------===//
-    // Emission Helper Routines.
-    //===------------------------------------------------------------------===//
-  public:
-    /// printOffset - This is just convenient handler for printing offsets.
-    void printOffset(int64_t Offset, raw_ostream &OS) const;
+  /// Emit a byte directive and value.
+  ///
+  void EmitInt8(int Value) const;
 
-    /// EmitInt8 - Emit a byte directive and value.
-    ///
-    void EmitInt8(int Value) const;
+  /// Emit a short directive and value.
+  ///
+  void EmitInt16(int Value) const;
 
-    /// EmitInt16 - Emit a short directive and value.
-    ///
-    void EmitInt16(int Value) const;
+  /// Emit a long directive and value.
+  ///
+  void EmitInt32(int Value) const;
 
-    /// EmitInt32 - Emit a long directive and value.
-    ///
-    void EmitInt32(int Value) const;
+  /// Emit something like ".long Hi-Lo" where the size in bytes of the directive
+  /// is specified by Size and Hi/Lo specify the labels.  This implicitly uses
+  /// .set if it is available.
+  void EmitLabelDifference(const MCSymbol *Hi, const MCSymbol *Lo,
+                           unsigned Size) const;
 
-    /// EmitLabelDifference - Emit something like ".long Hi-Lo" where the size
-    /// in bytes of the directive is specified by Size and Hi/Lo specify the
-    /// labels.  This implicitly uses .set if it is available.
-    void EmitLabelDifference(const MCSymbol *Hi, const MCSymbol *Lo,
-                             unsigned Size) const;
+  /// Emit something like ".long Hi+Offset-Lo" where the size in bytes of the
+  /// directive is specified by Size and Hi/Lo specify the labels.  This
+  /// implicitly uses .set if it is available.
+  void EmitLabelOffsetDifference(const MCSymbol *Hi, uint64_t Offset,
+                                 const MCSymbol *Lo, unsigned Size) const;
 
-    /// EmitLabelOffsetDifference - Emit something like ".long Hi+Offset-Lo"
-    /// where the size in bytes of the directive is specified by Size and Hi/Lo
-    /// specify the labels.  This implicitly uses .set if it is available.
-    void EmitLabelOffsetDifference(const MCSymbol *Hi, uint64_t Offset,
-                                   const MCSymbol *Lo, unsigned Size) const;
+  /// Emit something like ".long Label+Offset" where the size in bytes of the
+  /// directive is specified by Size and Label specifies the label.  This
+  /// implicitly uses .set if it is available.
+  void EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset,
+                           unsigned Size, bool IsSectionRelative = false) const;
 
-    /// EmitLabelPlusOffset - Emit something like ".long Label+Offset"
-    /// where the size in bytes of the directive is specified by Size and Label
-    /// specifies the label.  This implicitly uses .set if it is available.
-    void EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset,
-                             unsigned Size,
-                             bool IsSectionRelative = false) const;
+  /// Emit something like ".long Label" where the size in bytes of the directive
+  /// is specified by Size and Label specifies the label.
+  void EmitLabelReference(const MCSymbol *Label, unsigned Size,
+                          bool IsSectionRelative = false) const {
+    EmitLabelPlusOffset(Label, 0, Size, IsSectionRelative);
+  }
 
-    /// EmitLabelReference - Emit something like ".long Label"
-    /// where the size in bytes of the directive is specified by Size and Label
-    /// specifies the label.
-    void EmitLabelReference(const MCSymbol *Label, unsigned Size,
-                            bool IsSectionRelative = false) const {
-      EmitLabelPlusOffset(Label, 0, Size, IsSectionRelative);
-    }
+  //===------------------------------------------------------------------===//
+  // Dwarf Emission Helper Routines
+  //===------------------------------------------------------------------===//
 
-    //===------------------------------------------------------------------===//
-    // Dwarf Emission Helper Routines
-    //===------------------------------------------------------------------===//
+  /// Emit the specified signed leb128 value.
+  void EmitSLEB128(int64_t Value, const char *Desc = nullptr) const;
 
-    /// EmitSLEB128 - emit the specified signed leb128 value.
-    void EmitSLEB128(int64_t Value, const char *Desc = 0) const;
+  /// Emit the specified unsigned leb128 value.
+  void EmitULEB128(uint64_t Value, const char *Desc = nullptr,
+                   unsigned PadTo = 0) const;
 
-    /// EmitULEB128 - emit the specified unsigned leb128 value.
-    void EmitULEB128(uint64_t Value, const char *Desc = 0,
-                     unsigned PadTo = 0) const;
+  /// Emit a .byte 42 directive for a DW_CFA_xxx value.
+  void EmitCFAByte(unsigned Val) const;
 
-    /// EmitCFAByte - Emit a .byte 42 directive for a DW_CFA_xxx value.
-    void EmitCFAByte(unsigned Val) const;
+  /// Emit a .byte 42 directive that corresponds to an encoding.  If verbose
+  /// assembly output is enabled, we output comments describing the encoding.
+  /// Desc is a string saying what the encoding is specifying (e.g. "LSDA").
+  void EmitEncodingByte(unsigned Val, const char *Desc = nullptr) const;
 
-    /// EmitEncodingByte - Emit a .byte 42 directive that corresponds to an
-    /// encoding.  If verbose assembly output is enabled, we output comments
-    /// describing the encoding.  Desc is a string saying what the encoding is
-    /// specifying (e.g. "LSDA").
-    void EmitEncodingByte(unsigned Val, const char *Desc = 0) const;
+  /// Return the size of the encoding in bytes.
+  unsigned GetSizeOfEncodedValue(unsigned Encoding) const;
 
-    /// GetSizeOfEncodedValue - Return the size of the encoding in bytes.
-    unsigned GetSizeOfEncodedValue(unsigned Encoding) const;
+  /// Emit reference to a ttype global with a specified encoding.
+  void EmitTTypeReference(const GlobalValue *GV, unsigned Encoding) const;
 
-    /// EmitReference - Emit reference to a ttype global with a specified encoding.
-    void EmitTTypeReference(const GlobalValue *GV, unsigned Encoding) const;
+  /// Emit the 4-byte offset of Label from the start of its section.  This can
+  /// be done with a special directive if the target supports it (e.g. cygwin)
+  /// or by emitting it as an offset from a label at the start of the section.
+  ///
+  /// SectionLabel is a temporary label emitted at the start of the section
+  /// that Label lives in.
+  void EmitSectionOffset(const MCSymbol *Label,
+                         const MCSymbol *SectionLabel) const;
 
-    /// EmitSectionOffset - Emit the 4-byte offset of Label from the start of
-    /// its section.  This can be done with a special directive if the target
-    /// supports it (e.g. cygwin) or by emitting it as an offset from a label at
-    /// the start of the section.
-    ///
-    /// SectionLabel is a temporary label emitted at the start of the section
-    /// that Label lives in.
-    void EmitSectionOffset(const MCSymbol *Label,
-                           const MCSymbol *SectionLabel) const;
+  /// Get the value for DW_AT_APPLE_isa. Zero if no isa encoding specified.
+  virtual unsigned getISAEncoding() { return 0; }
 
-    /// getISAEncoding - Get the value for DW_AT_APPLE_isa. Zero if no isa
-    /// encoding specified.
-    virtual unsigned getISAEncoding() { return 0; }
+  /// \brief Emit a partial DWARF register operation.
+  /// \param MLoc             the register
+  /// \param PieceSize        size and
+  /// \param PieceOffset      offset of the piece in bits, if this is one
+  ///                         piece of an aggregate value.
+  ///
+  /// If size and offset is zero an operation for the entire
+  /// register is emitted: Some targets do not provide a DWARF
+  /// register number for every register.  If this is the case, this
+  /// function will attempt to emit a DWARF register by emitting a
+  /// piece of a super-register or by piecing together multiple
+  /// subregisters that alias the register.
+  void EmitDwarfRegOpPiece(ByteStreamer &BS, const MachineLocation &MLoc,
+                           unsigned PieceSize = 0,
+                           unsigned PieceOffset = 0) const;
 
-    /// EmitDwarfRegOp - Emit dwarf register operation.
-    virtual void EmitDwarfRegOp(const MachineLocation &MLoc,
-                                bool Indirect) const;
+  /// Emit dwarf register operation.
+  /// \param Indirect   whether this is a register-indirect address
+  virtual void EmitDwarfRegOp(ByteStreamer &BS, const MachineLocation &MLoc,
+                              bool Indirect) const;
 
-    //===------------------------------------------------------------------===//
-    // Dwarf Lowering Routines
-    //===------------------------------------------------------------------===//
+  //===------------------------------------------------------------------===//
+  // Dwarf Lowering Routines
+  //===------------------------------------------------------------------===//
 
-    /// \brief Emit frame instruction to describe the layout of the frame.
-    void emitCFIInstruction(const MCCFIInstruction &Inst) const;
+  /// \brief Emit frame instruction to describe the layout of the frame.
+  void emitCFIInstruction(const MCCFIInstruction &Inst) const;
 
-    //===------------------------------------------------------------------===//
-    // Inline Asm Support
-    //===------------------------------------------------------------------===//
-  public:
-    // These are hooks that targets can override to implement inline asm
-    // support.  These should probably be moved out of AsmPrinter someday.
+  //===------------------------------------------------------------------===//
+  // Inline Asm Support
+  //===------------------------------------------------------------------===//
+public:
+  // These are hooks that targets can override to implement inline asm
+  // support.  These should probably be moved out of AsmPrinter someday.
 
-    /// PrintSpecial - Print information related to the specified machine instr
-    /// that is independent of the operand, and may be independent of the instr
-    /// itself.  This can be useful for portably encoding the comment character
-    /// or other bits of target-specific knowledge into the asmstrings.  The
-    /// syntax used is ${:comment}.  Targets can override this to add support
-    /// for their own strange codes.
-    virtual void PrintSpecial(const MachineInstr *MI, raw_ostream &OS,
-                              const char *Code) const;
+  /// Print information related to the specified machine instr that is
+  /// independent of the operand, and may be independent of the instr itself.
+  /// This can be useful for portably encoding the comment character or other
+  /// bits of target-specific knowledge into the asmstrings.  The syntax used is
+  /// ${:comment}.  Targets can override this to add support for their own
+  /// strange codes.
+  virtual void PrintSpecial(const MachineInstr *MI, raw_ostream &OS,
+                            const char *Code) const;
 
-    /// PrintAsmOperand - Print the specified operand of MI, an INLINEASM
-    /// instruction, using the specified assembler variant.  Targets should
-    /// override this to format as appropriate.  This method can return true if
-    /// the operand is erroneous.
-    virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
-                                 unsigned AsmVariant, const char *ExtraCode,
-                                 raw_ostream &OS);
+  /// Print the specified operand of MI, an INLINEASM instruction, using the
+  /// specified assembler variant.  Targets should override this to format as
+  /// appropriate.  This method can return true if the operand is erroneous.
+  virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
+                               unsigned AsmVariant, const char *ExtraCode,
+                               raw_ostream &OS);
 
-    /// PrintAsmMemoryOperand - Print the specified operand of MI, an INLINEASM
-    /// instruction, using the specified assembler variant as an address.
-    /// Targets should override this to format as appropriate.  This method can
-    /// return true if the operand is erroneous.
-    virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
-                                       unsigned AsmVariant,
-                                       const char *ExtraCode, raw_ostream &OS);
+  /// Print the specified operand of MI, an INLINEASM instruction, using the
+  /// specified assembler variant as an address. Targets should override this to
+  /// format as appropriate.  This method can return true if the operand is
+  /// erroneous.
+  virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
+                                     unsigned AsmVariant, const char *ExtraCode,
+                                     raw_ostream &OS);
 
-  private:
-    /// Private state for PrintSpecial()
-    // Assign a unique ID to this machine instruction.
-    mutable const MachineInstr *LastMI;
-    mutable unsigned LastFn;
-    mutable unsigned Counter;
-    mutable unsigned SetCounter;
+  /// Let the target do anything it needs to do after emitting inlineasm.
+  /// This callback can be used restore the original mode in case the
+  /// inlineasm contains directives to switch modes.
+  /// \p StartInfo - the original subtarget info before inline asm
+  /// \p EndInfo   - the final subtarget info after parsing the inline asm,
+  ///                or NULL if the value is unknown.
+  virtual void emitInlineAsmEnd(const MCSubtargetInfo &StartInfo,
+                                const MCSubtargetInfo *EndInfo) const;
 
-    /// EmitInlineAsm - Emit a blob of inline asm to the output streamer.
-    void EmitInlineAsm(StringRef Str, const MDNode *LocMDNode = 0,
-                       InlineAsm::AsmDialect AsmDialect =
-                           InlineAsm::AD_ATT) const;
+private:
+  /// Private state for PrintSpecial()
+  // Assign a unique ID to this machine instruction.
+  mutable const MachineInstr *LastMI;
+  mutable unsigned LastFn;
+  mutable unsigned Counter;
+  mutable unsigned SetCounter;
 
-    /// EmitInlineAsm - This method formats and emits the specified machine
-    /// instruction that is an inline asm.
-    void EmitInlineAsm(const MachineInstr *MI) const;
+  /// Emit a blob of inline asm to the output streamer.
+  void
+  EmitInlineAsm(StringRef Str, const MDNode *LocMDNode = nullptr,
+                InlineAsm::AsmDialect AsmDialect = InlineAsm::AD_ATT) const;
 
-    //===------------------------------------------------------------------===//
-    // Internal Implementation Details
-    //===------------------------------------------------------------------===//
+  /// This method formats and emits the specified machine instruction that is an
+  /// inline asm.
+  void EmitInlineAsm(const MachineInstr *MI) const;
 
-    /// EmitVisibility - This emits visibility information about symbol, if
-    /// this is suported by the target.
-    void EmitVisibility(MCSymbol *Sym, unsigned Visibility,
-                        bool IsDefinition = true) const;
+  //===------------------------------------------------------------------===//
+  // Internal Implementation Details
+  //===------------------------------------------------------------------===//
 
-    void EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const;
+  /// This emits visibility information about symbol, if this is suported by the
+  /// target.
+  void EmitVisibility(MCSymbol *Sym, unsigned Visibility,
+                      bool IsDefinition = true) const;
 
-    void EmitJumpTableEntry(const MachineJumpTableInfo *MJTI,
-                            const MachineBasicBlock *MBB, unsigned uid) const;
-    void EmitLLVMUsedList(const ConstantArray *InitList);
-    /// Emit llvm.ident metadata in an '.ident' directive.
-    void EmitModuleIdents(Module &M);
-    void EmitXXStructorList(const Constant *List, bool isCtor);
-    GCMetadataPrinter *GetOrCreateGCPrinter(GCStrategy *C);
-  };
+  void EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const;
+
+  void EmitJumpTableEntry(const MachineJumpTableInfo *MJTI,
+                          const MachineBasicBlock *MBB, unsigned uid) const;
+  void EmitLLVMUsedList(const ConstantArray *InitList);
+  /// Emit llvm.ident metadata in an '.ident' directive.
+  void EmitModuleIdents(Module &M);
+  void EmitXXStructorList(const Constant *List, bool isCtor);
+  GCMetadataPrinter *GetOrCreateGCPrinter(GCStrategy &C);
+};
 }
 
 #endif
diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h
index a18f433bda40..04af4bd4d59d 100644
--- a/include/llvm/CodeGen/CallingConvLower.h
+++ b/include/llvm/CodeGen/CallingConvLower.h
@@ -18,14 +18,14 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/CallingConv.h"
 #include "llvm/Target/TargetCallingConv.h"
 
 namespace llvm {
-  class TargetRegisterInfo;
-  class TargetMachine;
-  class CCState;
+class CCState;
+class MVT;
+class TargetMachine;
+class TargetRegisterInfo;
 
 /// CCValAssign - Represent assignment of one arg/retval to a location.
 class CCValAssign {
@@ -39,6 +39,7 @@ public:
     VExt,   // The value is vector-widened in the location.
             // FIXME: Not implemented yet. Code that uses AExt to mean
             // vector-widen should be fixed to use VExt instead.
+    FPExt,  // The floating-point value is fp-extended in the location.
     Indirect // The location contains pointer to the value.
     // TODO: a subset of the value is in the location.
   };
@@ -111,6 +112,23 @@ public:
     return Ret;
   }
 
+  // There is no need to differentiate between a pending CCValAssign and other
+  // kinds, as they are stored in a different list.
+  static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT,
+                                LocInfo HTP) {
+    return getReg(ValNo, ValVT, 0, LocVT, HTP);
+  }
+
+  void convertToReg(unsigned RegNo) {
+    Loc = RegNo;
+    isMem = false;
+  }
+
+  void convertToMem(unsigned Offset) {
+    Loc = Offset;
+    isMem = true;
+  }
+
   unsigned getValNo() const { return ValNo; }
   MVT getValVT() const { return ValVT; }
 
@@ -163,6 +181,7 @@ private:
 
   unsigned StackOffset;
   SmallVector<uint32_t, 16> UsedRegs;
+  SmallVector<CCValAssign, 4> PendingLocs;
 
   // ByValInfo and SmallVector<ByValInfo, 4> ByValRegs:
   //
@@ -278,7 +297,7 @@ public:
 
   /// getFirstUnallocated - Return the first unallocated register in the set, or
   /// NumRegs if they are all allocated.
-  unsigned getFirstUnallocated(const uint16_t *Regs, unsigned NumRegs) const {
+  unsigned getFirstUnallocated(const MCPhysReg *Regs, unsigned NumRegs) const {
     for (unsigned i = 0; i != NumRegs; ++i)
       if (!isAllocated(Regs[i]))
         return i;
@@ -305,7 +324,7 @@ public:
   /// AllocateReg - Attempt to allocate one of the specified registers.  If none
   /// are available, return zero.  Otherwise, return the first one available,
   /// marking it and any aliases as allocated.
-  unsigned AllocateReg(const uint16_t *Regs, unsigned NumRegs) {
+  unsigned AllocateReg(const MCPhysReg *Regs, unsigned NumRegs) {
     unsigned FirstUnalloc = getFirstUnallocated(Regs, NumRegs);
     if (FirstUnalloc == NumRegs)
       return 0;    // Didn't find the reg.
@@ -316,8 +335,33 @@ public:
     return Reg;
   }
 
+  /// AllocateRegBlock - Attempt to allocate a block of RegsRequired consecutive
+  /// registers. If this is not possible, return zero. Otherwise, return the first
+  /// register of the block that were allocated, marking the entire block as allocated.
+  unsigned AllocateRegBlock(const uint16_t *Regs, unsigned NumRegs, unsigned RegsRequired) {
+    for (unsigned StartIdx = 0; StartIdx <= NumRegs - RegsRequired; ++StartIdx) {
+      bool BlockAvailable = true;
+      // Check for already-allocated regs in this block
+      for (unsigned BlockIdx = 0; BlockIdx < RegsRequired; ++BlockIdx) {
+        if (isAllocated(Regs[StartIdx + BlockIdx])) {
+          BlockAvailable = false;
+          break;
+        }
+      }
+      if (BlockAvailable) {
+        // Mark the entire block as allocated
+        for (unsigned BlockIdx = 0; BlockIdx < RegsRequired; ++BlockIdx) {
+          MarkAllocated(Regs[StartIdx + BlockIdx]);
+        }
+        return Regs[StartIdx];
+      }
+    }
+    // No block was available
+    return 0;
+  }
+
   /// Version of AllocateReg with list of registers to be shadowed.
-  unsigned AllocateReg(const uint16_t *Regs, const uint16_t *ShadowRegs,
+  unsigned AllocateReg(const MCPhysReg *Regs, const MCPhysReg *ShadowRegs,
                        unsigned NumRegs) {
     unsigned FirstUnalloc = getFirstUnallocated(Regs, NumRegs);
     if (FirstUnalloc == NumRegs)
@@ -347,6 +391,15 @@ public:
     return AllocateStack(Size, Align);
   }
 
+  /// Version of AllocateStack with list of extra registers to be shadowed.
+  /// Note that, unlike AllocateReg, this shadows ALL of the shadow registers.
+  unsigned AllocateStack(unsigned Size, unsigned Align,
+                         const MCPhysReg *ShadowRegs, unsigned NumShadowRegs) {
+    for (unsigned i = 0; i < NumShadowRegs; ++i)
+      MarkAllocated(ShadowRegs[i]);
+    return AllocateStack(Size, Align);
+  }
+
   // HandleByVal - Allocate a stack slot large enough to pass an argument by
   // value. The size and alignment information of the argument is encoded in its
   // parameter attribute.
@@ -394,8 +447,18 @@ public:
     ByValRegs.clear();
   }
 
+  // Rewind byval registers tracking info.
+  void rewindByValRegsInfo() {
+    InRegsParamsProceed = 0;
+  }
+
   ParmContext getCallOrPrologue() const { return CallOrPrologue; }
 
+  // Get list of pending assignments
+  SmallVectorImpl<llvm::CCValAssign> &getPendingLocs() {
+    return PendingLocs;
+  }
+
 private:
   /// MarkAllocated - Mark a register and all of its aliases as allocated.
   void MarkAllocated(unsigned Reg);
diff --git a/include/llvm/CodeGen/CommandFlags.h b/include/llvm/CodeGen/CommandFlags.h
index bc8dce33356e..449d93418a4c 100644
--- a/include/llvm/CodeGen/CommandFlags.h
+++ b/include/llvm/CodeGen/CommandFlags.h
@@ -16,9 +16,11 @@
 #ifndef LLVM_CODEGEN_COMMANDFLAGS_H
 #define LLVM_CODEGEN_COMMANDFLAGS_H
 
+#include "llvm/MC/MCTargetOptionsCommandFlags.h"
 #include "llvm/Support/CodeGen.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetOptions.h"
 #include <string>
 using namespace llvm;
 
@@ -68,11 +70,6 @@ CMModel("code-model",
                               "Large code model"),
                    clEnumValEnd));
 
-cl::opt<bool>
-RelaxAll("mc-relax-all",
-         cl::desc("When used with filetype=obj, "
-                  "relax all fixups in the emitted object file"));
-
 cl::opt<TargetMachine::CodeGenFileType>
 FileType("filetype", cl::init(TargetMachine::CGFT_AssemblyFile),
   cl::desc("Choose a file type (not all types are supported by all targets):"),
@@ -85,15 +82,6 @@ FileType("filetype", cl::init(TargetMachine::CGFT_AssemblyFile),
                         "Emit nothing, for performance testing"),
              clEnumValEnd));
 
-cl::opt<bool> DisableDotLoc("disable-dot-loc", cl::Hidden,
-                            cl::desc("Do not use .loc entries"));
-
-cl::opt<bool> DisableCFI("disable-cfi", cl::Hidden,
-                         cl::desc("Do not use .cfi_* directives"));
-
-cl::opt<bool> EnableDwarfDirectory("enable-dwarf-directory", cl::Hidden,
-                  cl::desc("Use .file directives with an explicit directory."));
-
 cl::opt<bool>
 DisableRedZone("disable-red-zone",
                cl::desc("Do not emit code that uses the red zone."),
@@ -192,11 +180,6 @@ EnablePIE("enable-pie",
           cl::init(false));
 
 cl::opt<bool>
-SegmentedStacks("segmented-stacks",
-                cl::desc("Use segmented stacks if possible."),
-                cl::init(false));
-
-cl::opt<bool>
 UseInitArray("use-init-array",
              cl::desc("Use .init_array instead of .ctors."),
              cl::init(false));
@@ -210,4 +193,59 @@ cl::opt<std::string> StartAfter("start-after",
                           cl::value_desc("pass-name"),
                           cl::init(""));
 
+cl::opt<bool> DataSections("data-sections",
+                           cl::desc("Emit data into separate sections"),
+                           cl::init(false));
+
+cl::opt<bool>
+FunctionSections("function-sections",
+                 cl::desc("Emit functions into separate sections"),
+                 cl::init(false));
+
+cl::opt<llvm::JumpTable::JumpTableType>
+JTableType("jump-table-type",
+          cl::desc("Choose the type of Jump-Instruction Table for jumptable."),
+          cl::init(JumpTable::Single),
+          cl::values(
+              clEnumValN(JumpTable::Single, "single",
+                         "Create a single table for all jumptable functions"),
+              clEnumValN(JumpTable::Arity, "arity",
+                         "Create one table per number of parameters."),
+              clEnumValN(JumpTable::Simplified, "simplified",
+                         "Create one table per simplified function type."),
+              clEnumValN(JumpTable::Full, "full",
+                         "Create one table per unique function type."),
+              clEnumValEnd));
+
+// Common utility function tightly tied to the options listed here. Initializes
+// a TargetOptions object with CodeGen flags and returns it.
+static inline TargetOptions InitTargetOptionsFromCodeGenFlags() {
+  TargetOptions Options;
+  Options.LessPreciseFPMADOption = EnableFPMAD;
+  Options.NoFramePointerElim = DisableFPElim;
+  Options.AllowFPOpFusion = FuseFPOps;
+  Options.UnsafeFPMath = EnableUnsafeFPMath;
+  Options.NoInfsFPMath = EnableNoInfsFPMath;
+  Options.NoNaNsFPMath = EnableNoNaNsFPMath;
+  Options.HonorSignDependentRoundingFPMathOption =
+      EnableHonorSignDependentRoundingFPMath;
+  Options.UseSoftFloat = GenerateSoftFloatCalls;
+  if (FloatABIForCalls != FloatABI::Default)
+    Options.FloatABIType = FloatABIForCalls;
+  Options.NoZerosInBSS = DontPlaceZerosInBSS;
+  Options.GuaranteedTailCallOpt = EnableGuaranteedTailCallOpt;
+  Options.DisableTailCalls = DisableTailCalls;
+  Options.StackAlignmentOverride = OverrideStackAlignment;
+  Options.TrapFuncName = TrapFuncName;
+  Options.PositionIndependentExecutable = EnablePIE;
+  Options.UseInitArray = UseInitArray;
+  Options.DataSections = DataSections;
+  Options.FunctionSections = FunctionSections;
+
+  Options.MCOptions = InitMCTargetOptionsFromFlags();
+  Options.JTType = JTableType;
+
+  return Options;
+}
+
 #endif
diff --git a/include/llvm/CodeGen/EdgeBundles.h b/include/llvm/CodeGen/EdgeBundles.h
index e8a4a2d2d894..c31fad246c96 100644
--- a/include/llvm/CodeGen/EdgeBundles.h
+++ b/include/llvm/CodeGen/EdgeBundles.h
@@ -55,15 +55,10 @@ public:
   void view() const;
 
 private:
-  virtual bool runOnMachineFunction(MachineFunction&);
-  virtual void getAnalysisUsage(AnalysisUsage&) const;
+  bool runOnMachineFunction(MachineFunction&) override;
+  void getAnalysisUsage(AnalysisUsage&) const override;
 };
 
-/// Specialize WriteGraph, the standard implementation won't work.
-raw_ostream &WriteGraph(raw_ostream &O, const EdgeBundles &G,
-                        bool ShortNames = false,
-                        const Twine &Title = "");
-
 } // end namespace llvm
 
 #endif
diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h
index 1e0ef6b545ea..0d1b1dc09560 100644
--- a/include/llvm/CodeGen/FastISel.h
+++ b/include/llvm/CodeGen/FastISel.h
@@ -16,23 +16,28 @@
 #define LLVM_CODEGEN_FASTISEL_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/ValueTypes.h"
+#include "llvm/Target/TargetLowering.h"
+#include "llvm/IR/CallingConv.h"
 
 namespace llvm {
 
 class AllocaInst;
 class Constant;
 class ConstantFP;
+class CallInst;
+class DataLayout;
 class FunctionLoweringInfo;
 class Instruction;
+class IntrinsicInst;
 class LoadInst;
+class MVT;
 class MachineConstantPool;
+class MachineFrameInfo;
 class MachineFunction;
 class MachineInstr;
-class MachineFrameInfo;
 class MachineRegisterInfo;
-class DataLayout;
 class TargetInstrInfo;
 class TargetLibraryInfo;
 class TargetLowering;
@@ -45,15 +50,154 @@ class Value;
 /// This is a fast-path instruction selection class that generates poor code and
 /// doesn't support illegal types or non-trivial lowering, but runs quickly.
 class FastISel {
+  public:
+  struct ArgListEntry {
+    Value *Val;
+    Type *Ty;
+    bool isSExt     : 1;
+    bool isZExt     : 1;
+    bool isInReg    : 1;
+    bool isSRet     : 1;
+    bool isNest     : 1;
+    bool isByVal    : 1;
+    bool isInAlloca : 1;
+    bool isReturned : 1;
+    uint16_t Alignment;
+
+    ArgListEntry()
+      : Val(nullptr), Ty(nullptr), isSExt(false), isZExt(false), isInReg(false),
+        isSRet(false), isNest(false), isByVal(false), isInAlloca(false),
+        isReturned(false), Alignment(0) { }
+
+    void setAttributes(ImmutableCallSite *CS, unsigned AttrIdx);
+  };
+  typedef std::vector<ArgListEntry> ArgListTy;
+
+  struct CallLoweringInfo {
+    Type *RetTy;
+    bool RetSExt           : 1;
+    bool RetZExt           : 1;
+    bool IsVarArg          : 1;
+    bool IsInReg           : 1;
+    bool DoesNotReturn     : 1;
+    bool IsReturnValueUsed : 1;
+
+    // IsTailCall should be modified by implementations of
+    // FastLowerCall that perform tail call conversions.
+    bool IsTailCall;
+
+    unsigned NumFixedArgs;
+    CallingConv::ID CallConv;
+    const Value *Callee;
+    const char *SymName;
+    ArgListTy Args;
+    ImmutableCallSite *CS;
+    MachineInstr *Call;
+    unsigned ResultReg;
+    unsigned NumResultRegs;
+
+    SmallVector<Value *, 16> OutVals;
+    SmallVector<ISD::ArgFlagsTy, 16> OutFlags;
+    SmallVector<unsigned, 16> OutRegs;
+    SmallVector<ISD::InputArg, 4> Ins;
+    SmallVector<unsigned, 4> InRegs;
+
+    CallLoweringInfo()
+      : RetTy(nullptr), RetSExt(false), RetZExt(false), IsVarArg(false),
+        IsInReg(false), DoesNotReturn(false), IsReturnValueUsed(true),
+        IsTailCall(false), NumFixedArgs(-1), CallConv(CallingConv::C),
+        Callee(nullptr), SymName(nullptr), CS(nullptr), Call(nullptr),
+        ResultReg(0), NumResultRegs(0)
+    {}
+
+    CallLoweringInfo &setCallee(Type *ResultTy, FunctionType *FuncTy,
+                                const Value *Target, ArgListTy &&ArgsList,
+                                ImmutableCallSite &Call) {
+      RetTy = ResultTy;
+      Callee = Target;
+
+      IsInReg = Call.paramHasAttr(0, Attribute::InReg);
+      DoesNotReturn = Call.doesNotReturn();
+      IsVarArg = FuncTy->isVarArg();
+      IsReturnValueUsed = !Call.getInstruction()->use_empty();
+      RetSExt = Call.paramHasAttr(0, Attribute::SExt);
+      RetZExt = Call.paramHasAttr(0, Attribute::ZExt);
+
+      CallConv = Call.getCallingConv();
+      NumFixedArgs = FuncTy->getNumParams();
+      Args = std::move(ArgsList);
+
+      CS = &Call;
+
+      return *this;
+    }
+
+    CallLoweringInfo &setCallee(Type *ResultTy, FunctionType *FuncTy,
+                                const char *Target, ArgListTy &&ArgsList,
+                                ImmutableCallSite &Call,
+                                unsigned FixedArgs = ~0U) {
+      RetTy = ResultTy;
+      Callee = Call.getCalledValue();
+      SymName = Target;
+
+      IsInReg = Call.paramHasAttr(0, Attribute::InReg);
+      DoesNotReturn = Call.doesNotReturn();
+      IsVarArg = FuncTy->isVarArg();
+      IsReturnValueUsed = !Call.getInstruction()->use_empty();
+      RetSExt = Call.paramHasAttr(0, Attribute::SExt);
+      RetZExt = Call.paramHasAttr(0, Attribute::ZExt);
+
+      CallConv = Call.getCallingConv();
+      NumFixedArgs = (FixedArgs == ~0U) ? FuncTy->getNumParams() : FixedArgs;
+      Args = std::move(ArgsList);
+
+      CS = &Call;
+
+      return *this;
+    }
+
+    CallLoweringInfo &setCallee(CallingConv::ID CC, Type *ResultTy,
+                                const Value *Target, ArgListTy &&ArgsList,
+                                unsigned FixedArgs = ~0U) {
+      RetTy = ResultTy;
+      Callee = Target;
+      CallConv = CC;
+      NumFixedArgs = (FixedArgs == ~0U) ? Args.size() : FixedArgs;
+      Args = std::move(ArgsList);
+      return *this;
+    }
+
+    CallLoweringInfo &setTailCall(bool Value = true) {
+      IsTailCall = Value;
+      return *this;
+    }
+
+    ArgListTy &getArgs() {
+      return Args;
+    }
+
+    void clearOuts() {
+      OutVals.clear();
+      OutFlags.clear();
+      OutRegs.clear();
+    }
+
+    void clearIns() {
+      Ins.clear();
+      InRegs.clear();
+    }
+  };
+
 protected:
   DenseMap<const Value *, unsigned> LocalValueMap;
   FunctionLoweringInfo &FuncInfo;
+  MachineFunction *MF;
   MachineRegisterInfo &MRI;
   MachineFrameInfo &MFI;
   MachineConstantPool &MCP;
-  DebugLoc DL;
+  DebugLoc DbgLoc;
   const TargetMachine &TM;
-  const DataLayout &TD;
+  const DataLayout &DL;
   const TargetInstrInfo &TII;
   const TargetLowering &TLI;
   const TargetRegisterInfo &TRI;
@@ -87,7 +231,7 @@ public:
   void startNewBlock();
 
   /// Return current debug location information.
-  DebugLoc getCurDebugLoc() const { return DL; }
+  DebugLoc getCurDebugLoc() const { return DbgLoc; }
   
   /// Do "fast" instruction selection for function arguments and append machine
   /// instructions to the current block. Return true if it is successful.
@@ -170,13 +314,20 @@ protected:
   /// process fails to select an instruction.  This gives targets a chance to
   /// emit code for anything that doesn't fit into FastISel's framework. It
   /// returns true if it was successful.
-  virtual bool
-  TargetSelectInstruction(const Instruction *I) = 0;
+  virtual bool TargetSelectInstruction(const Instruction *I) = 0;
   
   /// This method is called by target-independent code to do target specific
   /// argument lowering. It returns true if it was successful.
   virtual bool FastLowerArguments();
 
+  /// \brief This method is called by target-independent code to do target
+  /// specific call lowering. It returns true if it was successful.
+  virtual bool FastLowerCall(CallLoweringInfo &CLI);
+
+  /// \brief This method is called by target-independent code to do target
+  /// specific intrinsic lowering. It returns true if it was successful.
+  virtual bool FastLowerIntrinsicCall(const IntrinsicInst *II);
+
   /// This method is called by target-independent code to request that an
   /// instruction with the given type and opcode be emitted.
   virtual unsigned FastEmit_(MVT VT,
@@ -343,6 +494,12 @@ protected:
 
   unsigned createResultReg(const TargetRegisterClass *RC);
 
+  /// Try to constrain Op so that it is usable by argument OpNum of the provided
+  /// MCInstrDesc. If this fails, create a new virtual register in the correct
+  /// class and COPY the value there.
+  unsigned constrainOperandRegClass(const MCInstrDesc &II, unsigned Op,
+                                    unsigned OpNum);
+
   /// Emit a constant in a register using target-specific logic, such as
   /// constant pool loads.
   virtual unsigned TargetMaterializeConstant(const Constant* C) {
@@ -367,6 +524,15 @@ protected:
   /// - \c Add has a constant operand.
   bool canFoldAddIntoGEP(const User *GEP, const Value *Add);
 
+  /// Test whether the given value has exactly one use.
+  bool hasTrivialKill(const Value *V) const;
+
+  /// \brief Create a machine mem operand from the given instruction.
+  MachineMemOperand *createMachineMemOperandFor(const Instruction *I) const;
+
+  bool LowerCallTo(const CallInst *CI, const char *SymName, unsigned NumArgs);
+  bool LowerCallTo(CallLoweringInfo &CLI);
+
 private:
   bool SelectBinaryOp(const User *I, unsigned ISDOpcode);
 
@@ -374,7 +540,11 @@ private:
 
   bool SelectGetElementPtr(const User *I);
 
-  bool SelectCall(const User *I);
+  bool SelectStackmap(const CallInst *I);
+  bool SelectPatchpoint(const CallInst *I);
+  bool LowerCall(const CallInst *I);
+  bool SelectCall(const User *Call);
+  bool SelectIntrinsicCall(const IntrinsicInst *II);
 
   bool SelectBitCast(const User *I);
 
@@ -403,8 +573,11 @@ private:
   /// heavy instructions like calls.
   void flushLocalValueMap();
 
-  /// Test whether the given value has exactly one use.
-  bool hasTrivialKill(const Value *V) const;
+  bool addStackMapLiveVars(SmallVectorImpl<MachineOperand> &Ops,
+                           const CallInst *CI, unsigned StartIdx);
+  bool lowerCallOperands(const CallInst *CI, unsigned ArgIdx, unsigned NumArgs,
+                         const Value *Callee, bool ForceRetVoidTy,
+                         CallLoweringInfo &CLI);
 };
 
 }
diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h
index 50d320f8e839..9636b51e303d 100644
--- a/include/llvm/CodeGen/FunctionLoweringInfo.h
+++ b/include/llvm/CodeGen/FunctionLoweringInfo.h
@@ -21,7 +21,6 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Target/TargetRegisterInfo.h"
@@ -41,6 +40,8 @@ class MachineBasicBlock;
 class MachineFunction;
 class MachineModuleInfo;
 class MachineRegisterInfo;
+class SelectionDAG;
+class MVT;
 class TargetLowering;
 class Value;
 
@@ -125,7 +126,7 @@ public:
   /// set - Initialize this FunctionLoweringInfo with the given Function
   /// and its associated MachineFunction.
   ///
-  void set(const Function &Fn, MachineFunction &MF);
+  void set(const Function &Fn, MachineFunction &MF, SelectionDAG *DAG);
 
   /// clear - Clear out all the function-specific state. This returns this
   /// FunctionLoweringInfo to an empty state, ready to be used for a
@@ -152,11 +153,11 @@ public:
   /// register is a PHI destination and the PHI's LiveOutInfo is not valid.
   const LiveOutInfo *GetLiveOutRegInfo(unsigned Reg) {
     if (!LiveOutRegInfo.inBounds(Reg))
-      return NULL;
+      return nullptr;
 
     const LiveOutInfo *LOI = &LiveOutRegInfo[Reg];
     if (!LOI->IsValid)
-      return NULL;
+      return nullptr;
 
     return LOI;
   }
diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h
index 1070d29f7381..ddcc823ecd9e 100644
--- a/include/llvm/CodeGen/GCMetadata.h
+++ b/include/llvm/CodeGen/GCMetadata.h
@@ -35,8 +35,10 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/Pass.h"
-#include "llvm/Support/DebugLoc.h"
+
+#include <memory>
 
 namespace llvm {
   class AsmPrinter;
@@ -163,7 +165,7 @@ namespace llvm {
   ///
   class GCModuleInfo : public ImmutablePass {
     typedef StringMap<GCStrategy*> strategy_map_type;
-    typedef std::vector<GCStrategy*> list_type;
+    typedef std::vector<std::unique_ptr<GCStrategy>> list_type;
     typedef DenseMap<const Function*,GCFunctionInfo*> finfo_map_type;
 
     strategy_map_type StrategyMap;
@@ -178,7 +180,6 @@ namespace llvm {
     static char ID;
 
     GCModuleInfo();
-    ~GCModuleInfo();
 
     /// clear - Resets the pass. Any pass, which uses GCModuleInfo, should
     /// call it in doFinalization().
diff --git a/include/llvm/CodeGen/GCStrategy.h b/include/llvm/CodeGen/GCStrategy.h
index dfc26d72dc9f..81e1f85286e1 100644
--- a/include/llvm/CodeGen/GCStrategy.h
+++ b/include/llvm/CodeGen/GCStrategy.h
@@ -54,7 +54,7 @@ namespace llvm {
   /// be abstractly described.
   class GCStrategy {
   public:
-    typedef std::vector<GCFunctionInfo*> list_type;
+    typedef std::vector<std::unique_ptr<GCFunctionInfo>> list_type;
     typedef list_type::iterator iterator;
     
   private:
@@ -77,7 +77,7 @@ namespace llvm {
   public:
     GCStrategy();
     
-    virtual ~GCStrategy();
+    virtual ~GCStrategy() {}
     
     
     /// getName - The name of the GC strategy, for debugging.
diff --git a/include/llvm/CodeGen/GCs.h b/include/llvm/CodeGen/GCs.h
index 456d2dcb51a6..bb170c85cbf8 100644
--- a/include/llvm/CodeGen/GCs.h
+++ b/include/llvm/CodeGen/GCs.h
@@ -17,13 +17,13 @@
 namespace llvm {
   class GCStrategy;
   class GCMetadataPrinter;
-  
+
   /// FIXME: Collector instances are not useful on their own. These no longer
   ///        serve any purpose except to link in the plugins.
-  
+
   /// Creates an ocaml-compatible garbage collector.
   void linkOcamlGC();
-  
+
   /// Creates an ocaml-compatible metadata printer.
   void linkOcamlGCPrinter();
 
@@ -32,7 +32,7 @@ namespace llvm {
 
   /// Creates an erlang-compatible metadata printer.
   void linkErlangGCPrinter();
-  
+
   /// Creates a shadow stack garbage collector. This collector requires no code
   /// generator support.
   void linkShadowStackGC();
diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h
index 48a0523dc618..84447616c989 100644
--- a/include/llvm/CodeGen/ISDOpcodes.h
+++ b/include/llvm/CodeGen/ISDOpcodes.h
@@ -72,6 +72,11 @@ namespace ISD {
     /// the parent's frame or return address, and so on.
     FRAMEADDR, RETURNADDR,
 
+    /// READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on
+    /// the DAG, which implements the named register global variables extension.
+    READ_REGISTER,
+    WRITE_REGISTER,
+
     /// FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to
     /// first (possible) on-stack argument. This is needed for correct stack
     /// adjustment during unwind.
@@ -374,6 +379,37 @@ namespace ISD {
     /// operand, a ValueType node.
     SIGN_EXTEND_INREG,
 
+    /// ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an
+    /// in-register any-extension of the low lanes of an integer vector. The
+    /// result type must have fewer elements than the operand type, and those
+    /// elements must be larger integer types such that the total size of the
+    /// operand type and the result type match. Each of the low operand
+    /// elements is any-extended into the corresponding, wider result
+    /// elements with the high bits becoming undef.
+    ANY_EXTEND_VECTOR_INREG,
+
+    /// SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an
+    /// in-register sign-extension of the low lanes of an integer vector. The
+    /// result type must have fewer elements than the operand type, and those
+    /// elements must be larger integer types such that the total size of the
+    /// operand type and the result type match. Each of the low operand
+    /// elements is sign-extended into the corresponding, wider result
+    /// elements.
+    // FIXME: The SIGN_EXTEND_INREG node isn't specifically limited to
+    // scalars, but it also doesn't handle vectors well. Either it should be
+    // restricted to scalars or this node (and its handling) should be merged
+    // into it.
+    SIGN_EXTEND_VECTOR_INREG,
+
+    /// ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an
+    /// in-register zero-extension of the low lanes of an integer vector. The
+    /// result type must have fewer elements than the operand type, and those
+    /// elements must be larger integer types such that the total size of the
+    /// operand type and the result type match. Each of the low operand
+    /// elements is zero-extended into the corresponding, wider result
+    /// elements.
+    ZERO_EXTEND_VECTOR_INREG,
+
     /// FP_TO_[US]INT - Convert a floating point value to a signed or unsigned
     /// integer.
     FP_TO_SINT,
@@ -436,11 +472,11 @@ namespace ISD {
     ///   5) ISD::CvtCode indicating the type of conversion to do
     CONVERT_RNDSAT,
 
-    /// FP16_TO_FP32, FP32_TO_FP16 - These operators are used to perform
-    /// promotions and truncation for half-precision (16 bit) floating
-    /// numbers. We need special nodes since FP16 is a storage-only type with
-    /// special semantics of operations.
-    FP16_TO_FP32, FP32_TO_FP16,
+    /// FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions
+    /// and truncation for half-precision (16 bit) floating numbers. These nodes
+    /// form a semi-softened interface for dealing with f16 (as an i16), which
+    /// is often a storage-only type but has native conversions.
+    FP16_TO_FP, FP_TO_FP16,
 
     /// FNEG, FABS, FSQRT, FSIN, FCOS, FPOWI, FPOW,
     /// FLOG, FLOG2, FLOG10, FEXP, FEXP2,
@@ -603,7 +639,7 @@ namespace ISD {
     /// This corresponds to "load atomic" instruction.
     ATOMIC_LOAD,
 
-    /// OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr, val)
+    /// OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val)
     /// This corresponds to "store atomic" instruction.
     ATOMIC_STORE,
 
@@ -614,6 +650,12 @@ namespace ISD {
     /// This corresponds to the cmpxchg instruction.
     ATOMIC_CMP_SWAP,
 
+    /// Val, Success, OUTCHAIN
+    ///     = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap)
+    /// N.b. this is still a strong cmpxchg operation, so
+    /// Success == "Val == cmp".
+    ATOMIC_CMP_SWAP_WITH_SUCCESS,
+
     /// Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt)
     /// Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt)
     /// For double-word atomic operations:
@@ -702,6 +744,8 @@ namespace ISD {
     LAST_LOADEXT_TYPE
   };
 
+  NodeType getExtForLoadExtType(LoadExtType);
+
   //===--------------------------------------------------------------------===//
   /// ISD::CondCode enum - These are ordered carefully to make the bitfields
   /// below work out, when considering SETFALSE (something that never exists
diff --git a/include/llvm/CodeGen/IntrinsicLowering.h b/include/llvm/CodeGen/IntrinsicLowering.h
index 68389dde494f..9e6ab7d45977 100644
--- a/include/llvm/CodeGen/IntrinsicLowering.h
+++ b/include/llvm/CodeGen/IntrinsicLowering.h
@@ -24,13 +24,13 @@ namespace llvm {
   class DataLayout;
 
   class IntrinsicLowering {
-    const DataLayout& TD;
+    const DataLayout& DL;
 
     
     bool Warned;
   public:
-    explicit IntrinsicLowering(const DataLayout &td) :
-      TD(td), Warned(false) {}
+    explicit IntrinsicLowering(const DataLayout &DL) :
+      DL(DL), Warned(false) {}
 
     /// AddPrototypes - This method, if called, causes all of the prototypes
     /// that might be needed by an intrinsic lowering implementation to be
diff --git a/include/llvm/CodeGen/JITCodeEmitter.h b/include/llvm/CodeGen/JITCodeEmitter.h
index 9a7321418698..dc2a0272db4e 100644
--- a/include/llvm/CodeGen/JITCodeEmitter.h
+++ b/include/llvm/CodeGen/JITCodeEmitter.h
@@ -51,7 +51,7 @@ class Function;
 /// occurred, more memory is allocated, and we reemit the code into it.
 /// 
 class JITCodeEmitter : public MachineCodeEmitter {
-  virtual void anchor();
+  void anchor() override;
 public:
   virtual ~JITCodeEmitter() {}
 
@@ -59,15 +59,15 @@ public:
   /// about to be code generated.  This initializes the BufferBegin/End/Ptr
   /// fields.
   ///
-  virtual void startFunction(MachineFunction &F) = 0;
+  void startFunction(MachineFunction &F) override = 0;
 
   /// finishFunction - This callback is invoked when the specified function has
   /// finished code generation.  If a buffer overflow has occurred, this method
   /// returns true (the callee is required to try again), otherwise it returns
   /// false.
   ///
-  virtual bool finishFunction(MachineFunction &F) = 0;
-  
+  bool finishFunction(MachineFunction &F) override = 0;
+
   /// allocIndirectGV - Allocates and fills storage for an indirect
   /// GlobalValue, and returns the address.
   virtual void *allocIndirectGV(const GlobalValue *GV,
@@ -248,19 +248,19 @@ public:
   
   
   /// emitLabel - Emits a label
-  virtual void emitLabel(MCSymbol *Label) = 0;
+  void emitLabel(MCSymbol *Label) override = 0;
 
   /// allocateSpace - Allocate a block of space in the current output buffer,
   /// returning null (and setting conditions to indicate buffer overflow) on
   /// failure.  Alignment is the alignment in bytes of the buffer desired.
-  virtual void *allocateSpace(uintptr_t Size, unsigned Alignment) {
+  void *allocateSpace(uintptr_t Size, unsigned Alignment) override {
     emitAlignment(Alignment);
     void *Result;
     
     // Check for buffer overflow.
     if (Size >= (uintptr_t)(BufferEnd-CurBufferPtr)) {
       CurBufferPtr = BufferEnd;
-      Result = 0;
+      Result = nullptr;
     } else {
       // Allocate the space.
       Result = CurBufferPtr;
@@ -278,18 +278,18 @@ public:
   /// StartMachineBasicBlock - This should be called by the target when a new
   /// basic block is about to be emitted.  This way the MCE knows where the
   /// start of the block is, and can implement getMachineBasicBlockAddress.
-  virtual void StartMachineBasicBlock(MachineBasicBlock *MBB) = 0;
-  
+  void StartMachineBasicBlock(MachineBasicBlock *MBB) override = 0;
+
   /// getCurrentPCValue - This returns the address that the next emitted byte
   /// will be output to.
   ///
-  virtual uintptr_t getCurrentPCValue() const {
+  uintptr_t getCurrentPCValue() const override {
     return (uintptr_t)CurBufferPtr;
   }
 
   /// getCurrentPCOffset - Return the offset from the start of the emitted
   /// buffer that we are currently writing to.
-  uintptr_t getCurrentPCOffset() const {
+  uintptr_t getCurrentPCOffset() const override {
     return CurBufferPtr-BufferBegin;
   }
 
@@ -298,42 +298,45 @@ public:
   /// creates jump tables or constant pools in memory on the fly while the
   /// object code emitters rely on a linker to have real addresses and should
   /// use relocations instead.
-  bool earlyResolveAddresses() const { return true; }
+  bool earlyResolveAddresses() const override { return true; }
 
   /// addRelocation - Whenever a relocatable address is needed, it should be
   /// noted with this interface.
-  virtual void addRelocation(const MachineRelocation &MR) = 0;
-  
+  void addRelocation(const MachineRelocation &MR) override = 0;
+
   /// FIXME: These should all be handled with relocations!
   
   /// getConstantPoolEntryAddress - Return the address of the 'Index' entry in
   /// the constant pool that was last emitted with the emitConstantPool method.
   ///
-  virtual uintptr_t getConstantPoolEntryAddress(unsigned Index) const = 0;
+  uintptr_t getConstantPoolEntryAddress(unsigned Index) const override = 0;
 
   /// getJumpTableEntryAddress - Return the address of the jump table with index
   /// 'Index' in the function that last called initJumpTableInfo.
   ///
-  virtual uintptr_t getJumpTableEntryAddress(unsigned Index) const = 0;
-  
+  uintptr_t getJumpTableEntryAddress(unsigned Index) const override = 0;
+
   /// getMachineBasicBlockAddress - Return the address of the specified
   /// MachineBasicBlock, only usable after the label for the MBB has been
   /// emitted.
   ///
-  virtual uintptr_t getMachineBasicBlockAddress(MachineBasicBlock *MBB) const= 0;
+  uintptr_t
+    getMachineBasicBlockAddress(MachineBasicBlock *MBB) const override = 0;
 
   /// getLabelAddress - Return the address of the specified Label, only usable
   /// after the Label has been emitted.
   ///
-  virtual uintptr_t getLabelAddress(MCSymbol *Label) const = 0;
-  
+  uintptr_t getLabelAddress(MCSymbol *Label) const override = 0;
+
   /// Specifies the MachineModuleInfo object. This is used for exception handling
   /// purposes.
-  virtual void setModuleInfo(MachineModuleInfo* Info) = 0;
+  void setModuleInfo(MachineModuleInfo* Info) override = 0;
 
   /// getLabelLocations - Return the label locations map of the label IDs to
   /// their address.
-  virtual DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() { return 0; }
+  virtual DenseMap<MCSymbol*, uintptr_t> *getLabelLocations() {
+    return nullptr;
+  }
 };
 
 } // End llvm namespace
diff --git a/include/llvm/CodeGen/JumpInstrTables.h b/include/llvm/CodeGen/JumpInstrTables.h
new file mode 100644
index 000000000000..6ca3d7d1765f
--- /dev/null
+++ b/include/llvm/CodeGen/JumpInstrTables.h
@@ -0,0 +1,104 @@
+//===-- JumpInstrTables.h: Jump-Instruction Tables --------------*- C++ -*-===//
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// \brief An implementation of tables consisting of jump instructions
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_JUMPINSTRTABLES_H
+#define LLVM_CODEGEN_JUMPINSTRTABLES_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Pass.h"
+#include "llvm/Target/TargetOptions.h"
+
+namespace llvm {
+class Constant;
+class Function;
+class FunctionType;
+class JumpInstrTableInfo;
+class Module;
+
+/// A class to manage a set of jump tables indexed on function type. It looks at
+/// each function in the module to find all the functions that have the
+/// jumptable attribute set. For each such function, it creates a new
+/// jump-instruction-table function and stores the mapping in the ImmutablePass
+/// JumpInstrTableInfo.
+///
+/// These special functions get lowered in AsmPrinter to assembly of the form:
+/// \verbatim
+///   .globl f
+///   .type f,@function
+///   .align 8,0x90
+/// f:
+///   jmp f_orig@PLT
+/// \endverbatim
+///
+/// Support for an architecture depends on two functions in TargetInstrInfo:
+/// getUnconditionalBranch, and getTrap. AsmPrinter uses these to generate the
+/// appropriate instructions for the jump statement (an unconditional branch)
+/// and for padding to make the table have a size that is a power of two. This
+/// padding uses a trap instruction to ensure that calls to this area halt the
+/// program. The default implementations of these functions call
+/// llvm_unreachable.
+class JumpInstrTables : public ModulePass {
+public:
+  static char ID;
+
+  JumpInstrTables();
+  JumpInstrTables(JumpTable::JumpTableType JTT);
+  virtual ~JumpInstrTables();
+  bool runOnModule(Module &M) override;
+  const char *getPassName() const override { return "Jump-Instruction Tables"; }
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  /// Creates a jump-instruction table function for the Target and adds it to
+  /// the tables.
+  Function *insertEntry(Module &M, Function *Target);
+
+  /// Checks to see if there is already a table for the given FunctionType.
+  bool hasTable(FunctionType *FunTy);
+
+private:
+  /// The metadata used while a jump table is being built
+  struct TableMeta {
+    /// The number of this table
+    unsigned TableNum;
+
+    /// The current number of jump entries in the table.
+    unsigned Count;
+  };
+
+  typedef DenseMap<FunctionType *, struct TableMeta> JumpMap;
+
+  /// Maps the function into a subset of function types, depending on the
+  /// jump-instruction table style selected from JumpTableTypes in
+  /// JumpInstrTables.cpp. The choice of mapping determines the number of
+  /// jump-instruction tables generated by this pass. E.g., the simplest mapping
+  /// converts every function type into void f(); so, all functions end up in a
+  /// single table.
+  FunctionType *transformType(FunctionType *FunTy);
+
+  /// The current state of functions and jump entries in the table(s).
+  JumpMap Metadata;
+
+  /// The ImmutablePass that stores information about the generated tables.
+  JumpInstrTableInfo *JITI;
+
+  /// The total number of tables.
+  unsigned TableCount;
+
+  /// The type of tables to build.
+  JumpTable::JumpTableType JTType;
+};
+
+/// Creates a JumpInstrTables pass for the given type of jump table.
+ModulePass *createJumpInstrTablesPass(JumpTable::JumpTableType JTT);
+}
+
+#endif /* LLVM_CODEGEN_JUMPINSTRTABLES_H */
diff --git a/include/llvm/CodeGen/LatencyPriorityQueue.h b/include/llvm/CodeGen/LatencyPriorityQueue.h
index d454347d0b82..cf601ae5384c 100644
--- a/include/llvm/CodeGen/LatencyPriorityQueue.h
+++ b/include/llvm/CodeGen/LatencyPriorityQueue.h
@@ -47,22 +47,22 @@ namespace llvm {
     LatencyPriorityQueue() : Picker(this) {
     }
 
-    bool isBottomUp() const { return false; }
+    bool isBottomUp() const override { return false; }
 
-    void initNodes(std::vector<SUnit> &sunits) {
+    void initNodes(std::vector<SUnit> &sunits) override {
       SUnits = &sunits;
       NumNodesSolelyBlocking.resize(SUnits->size(), 0);
     }
 
-    void addNode(const SUnit *SU) {
+    void addNode(const SUnit *SU) override {
       NumNodesSolelyBlocking.resize(SUnits->size(), 0);
     }
 
-    void updateNode(const SUnit *SU) {
+    void updateNode(const SUnit *SU) override {
     }
 
-    void releaseState() {
-      SUnits = 0;
+    void releaseState() override {
+      SUnits = nullptr;
     }
 
     unsigned getLatency(unsigned NodeNum) const {
@@ -75,21 +75,21 @@ namespace llvm {
       return NumNodesSolelyBlocking[NodeNum];
     }
 
-    bool empty() const { return Queue.empty(); }
+    bool empty() const override { return Queue.empty(); }
 
-    virtual void push(SUnit *U);
+    void push(SUnit *U) override;
 
-    virtual SUnit *pop();
+    SUnit *pop() override;
 
-    virtual void remove(SUnit *SU);
+    void remove(SUnit *SU) override;
 
-    virtual void dump(ScheduleDAG* DAG) const;
+    void dump(ScheduleDAG* DAG) const override;
 
     // scheduledNode - As nodes are scheduled, we look to see if there are any
     // successor nodes that have a single unscheduled predecessor.  If so, that
     // single predecessor has a higher priority, since scheduling it will make
     // the node available.
-    void scheduledNode(SUnit *Node);
+    void scheduledNode(SUnit *Node) override;
 
 private:
     void AdjustPriorityOfUnscheduledPreds(SUnit *SU);
diff --git a/include/llvm/CodeGen/LexicalScopes.h b/include/llvm/CodeGen/LexicalScopes.h
index 26563a605574..036aea30a510 100644
--- a/include/llvm/CodeGen/LexicalScopes.h
+++ b/include/llvm/CodeGen/LexicalScopes.h
@@ -21,16 +21,17 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Metadata.h"
-#include "llvm/Support/DebugLoc.h"
-#include "llvm/Support/ValueHandle.h"
+#include "llvm/IR/ValueHandle.h"
 #include <utility>
+#include <unordered_map>
 namespace llvm {
 
 class MachineInstr;
 class MachineBasicBlock;
 class MachineFunction;
-class LexicalScope;
 
 //===----------------------------------------------------------------------===//
 /// InsnRange - This is used to track range of instructions with identical
@@ -39,37 +40,130 @@ class LexicalScope;
 typedef std::pair<const MachineInstr *, const MachineInstr *> InsnRange;
 
 //===----------------------------------------------------------------------===//
+/// LexicalScope - This class is used to track scope information.
+///
+class LexicalScope {
+
+public:
+  LexicalScope(LexicalScope *P, const MDNode *D, const MDNode *I, bool A)
+      : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A),
+        LastInsn(nullptr), FirstInsn(nullptr), DFSIn(0), DFSOut(0) {
+    if (Parent)
+      Parent->addChild(this);
+  }
+
+  // Accessors.
+  LexicalScope *getParent() const { return Parent; }
+  const MDNode *getDesc() const { return Desc; }
+  const MDNode *getInlinedAt() const { return InlinedAtLocation; }
+  const MDNode *getScopeNode() const { return Desc; }
+  bool isAbstractScope() const { return AbstractScope; }
+  SmallVectorImpl<LexicalScope *> &getChildren() { return Children; }
+  SmallVectorImpl<InsnRange> &getRanges() { return Ranges; }
+
+  /// addChild - Add a child scope.
+  void addChild(LexicalScope *S) { Children.push_back(S); }
+
+  /// openInsnRange - This scope covers instruction range starting from MI.
+  void openInsnRange(const MachineInstr *MI) {
+    if (!FirstInsn)
+      FirstInsn = MI;
+
+    if (Parent)
+      Parent->openInsnRange(MI);
+  }
+
+  /// extendInsnRange - Extend the current instruction range covered by
+  /// this scope.
+  void extendInsnRange(const MachineInstr *MI) {
+    assert(FirstInsn && "MI Range is not open!");
+    LastInsn = MI;
+    if (Parent)
+      Parent->extendInsnRange(MI);
+  }
+
+  /// closeInsnRange - Create a range based on FirstInsn and LastInsn collected
+  /// until now. This is used when a new scope is encountered while walking
+  /// machine instructions.
+  void closeInsnRange(LexicalScope *NewScope = nullptr) {
+    assert(LastInsn && "Last insn missing!");
+    Ranges.push_back(InsnRange(FirstInsn, LastInsn));
+    FirstInsn = nullptr;
+    LastInsn = nullptr;
+    // If Parent dominates NewScope then do not close Parent's instruction
+    // range.
+    if (Parent && (!NewScope || !Parent->dominates(NewScope)))
+      Parent->closeInsnRange(NewScope);
+  }
+
+  /// dominates - Return true if current scope dominates given lexical scope.
+  bool dominates(const LexicalScope *S) const {
+    if (S == this)
+      return true;
+    if (DFSIn < S->getDFSIn() && DFSOut > S->getDFSOut())
+      return true;
+    return false;
+  }
+
+  // Depth First Search support to walk and manipulate LexicalScope hierarchy.
+  unsigned getDFSOut() const { return DFSOut; }
+  void setDFSOut(unsigned O) { DFSOut = O; }
+  unsigned getDFSIn() const { return DFSIn; }
+  void setDFSIn(unsigned I) { DFSIn = I; }
+
+  /// dump - print lexical scope.
+  void dump(unsigned Indent = 0) const;
+
+private:
+  LexicalScope *Parent;                        // Parent to this scope.
+  AssertingVH<const MDNode> Desc;              // Debug info descriptor.
+  AssertingVH<const MDNode> InlinedAtLocation; // Location at which this
+                                               // scope is inlined.
+  bool AbstractScope;                          // Abstract Scope
+  SmallVector<LexicalScope *, 4> Children;     // Scopes defined in scope.
+                                               // Contents not owned.
+  SmallVector<InsnRange, 4> Ranges;
+
+  const MachineInstr *LastInsn;  // Last instruction of this scope.
+  const MachineInstr *FirstInsn; // First instruction of this scope.
+  unsigned DFSIn, DFSOut;        // In & Out Depth use to determine
+                                 // scope nesting.
+};
+
+//===----------------------------------------------------------------------===//
 /// LexicalScopes -  This class provides interface to collect and use lexical
 /// scoping information from machine instruction.
 ///
 class LexicalScopes {
 public:
-  LexicalScopes() : MF(NULL),  CurrentFnLexicalScope(NULL) { }
-  virtual ~LexicalScopes();
+  LexicalScopes() : MF(nullptr), CurrentFnLexicalScope(nullptr) {}
 
-  /// initialize - Scan machine function and constuct lexical scope nest.
-  virtual void initialize(const MachineFunction &);
+  /// initialize - Scan machine function and constuct lexical scope nest, resets
+  /// the instance if necessary.
+  void initialize(const MachineFunction &);
 
   /// releaseMemory - release memory.
-  virtual void releaseMemory();
-  
+  void reset();
+
   /// empty - Return true if there is any lexical scope information available.
-  bool empty() { return CurrentFnLexicalScope == NULL; }
+  bool empty() { return CurrentFnLexicalScope == nullptr; }
 
-  /// isCurrentFunctionScope - Return true if given lexical scope represents 
+  /// isCurrentFunctionScope - Return true if given lexical scope represents
   /// current function.
-  bool isCurrentFunctionScope(const LexicalScope *LS) { 
+  bool isCurrentFunctionScope(const LexicalScope *LS) {
     return LS == CurrentFnLexicalScope;
   }
 
   /// getCurrentFunctionScope - Return lexical scope for the current function.
-  LexicalScope *getCurrentFunctionScope() const { return CurrentFnLexicalScope;}
+  LexicalScope *getCurrentFunctionScope() const {
+    return CurrentFnLexicalScope;
+  }
 
   /// getMachineBasicBlocks - Populate given set using machine basic blocks
   /// which have machine instructions that belong to lexical scope identified by
   /// DebugLoc.
   void getMachineBasicBlocks(DebugLoc DL,
-                             SmallPtrSet<const MachineBasicBlock*, 4> &MBBs);
+                             SmallPtrSet<const MachineBasicBlock *, 4> &MBBs);
 
   /// dominates - Return true if DebugLoc's lexical scope dominates at least one
   /// machine instruction's lexical scope in a given machine basic block.
@@ -84,27 +178,29 @@ public:
     return AbstractScopesList;
   }
 
-  /// findAbstractScope - Find an abstract scope or return NULL.
+  /// findAbstractScope - Find an abstract scope or return null.
   LexicalScope *findAbstractScope(const MDNode *N) {
-    return AbstractScopeMap.lookup(N);
+    auto I = AbstractScopeMap.find(N);
+    return I != AbstractScopeMap.end() ? &I->second : nullptr;
   }
 
   /// findInlinedScope - Find an inlined scope for the given DebugLoc or return
   /// NULL.
-  LexicalScope *findInlinedScope(DebugLoc DL) {
-    return InlinedLexicalScopeMap.lookup(DL);
-  }
+  LexicalScope *findInlinedScope(DebugLoc DL);
 
-  /// findLexicalScope - Find regular lexical scope or return NULL.
+  /// findLexicalScope - Find regular lexical scope or return null.
   LexicalScope *findLexicalScope(const MDNode *N) {
-    return LexicalScopeMap.lookup(N);
+    auto I = LexicalScopeMap.find(N);
+    return I != LexicalScopeMap.end() ? &I->second : nullptr;
   }
 
   /// dump - Print data structures to dbgs().
   void dump();
 
-private:
+  /// getOrCreateAbstractScope - Find or create an abstract lexical scope.
+  LexicalScope *getOrCreateAbstractScope(const MDNode *N);
 
+private:
   /// getOrCreateLexicalScope - Find lexical scope for the given DebugLoc. If
   /// not available then create new lexical scope.
   LexicalScope *getOrCreateLexicalScope(DebugLoc DL);
@@ -115,30 +211,31 @@ private:
   /// getOrCreateInlinedScope - Find or create an inlined lexical scope.
   LexicalScope *getOrCreateInlinedScope(MDNode *Scope, MDNode *InlinedAt);
 
-  /// getOrCreateAbstractScope - Find or create an abstract lexical scope.
-  LexicalScope *getOrCreateAbstractScope(const MDNode *N);
-
   /// extractLexicalScopes - Extract instruction ranges for each lexical scopes
   /// for the given machine function.
   void extractLexicalScopes(SmallVectorImpl<InsnRange> &MIRanges,
                             DenseMap<const MachineInstr *, LexicalScope *> &M);
   void constructScopeNest(LexicalScope *Scope);
-  void assignInstructionRanges(SmallVectorImpl<InsnRange> &MIRanges,
-                             DenseMap<const MachineInstr *, LexicalScope *> &M);
+  void
+  assignInstructionRanges(SmallVectorImpl<InsnRange> &MIRanges,
+                          DenseMap<const MachineInstr *, LexicalScope *> &M);
 
 private:
   const MachineFunction *MF;
 
-  /// LexicalScopeMap - Tracks the scopes in the current function.  Owns the
-  /// contained LexicalScope*s.
-  DenseMap<const MDNode *, LexicalScope *> LexicalScopeMap;
+  /// LexicalScopeMap - Tracks the scopes in the current function.
+  // Use an unordered_map to ensure value pointer validity over insertion.
+  std::unordered_map<const MDNode *, LexicalScope> LexicalScopeMap;
 
-  /// InlinedLexicalScopeMap - Tracks inlined function scopes in current function.
-  DenseMap<DebugLoc, LexicalScope *> InlinedLexicalScopeMap;
+  /// InlinedLexicalScopeMap - Tracks inlined function scopes in current
+  /// function.
+  std::unordered_map<std::pair<const MDNode *, const MDNode *>, LexicalScope,
+                     pair_hash<const MDNode *, const MDNode *>>
+  InlinedLexicalScopeMap;
 
-  /// AbstractScopeMap - These scopes are  not included LexicalScopeMap.  
-  /// AbstractScopes owns its LexicalScope*s.
-  DenseMap<const MDNode *, LexicalScope *> AbstractScopeMap;
+  /// AbstractScopeMap - These scopes are  not included LexicalScopeMap.
+  // Use an unordered_map to ensure value pointer validity over insertion.
+  std::unordered_map<const MDNode *, LexicalScope> AbstractScopeMap;
 
   /// AbstractScopesList - Tracks abstract scopes constructed while processing
   /// a function.
@@ -149,100 +246,6 @@ private:
   LexicalScope *CurrentFnLexicalScope;
 };
 
-//===----------------------------------------------------------------------===//
-/// LexicalScope - This class is used to track scope information.
-///
-class LexicalScope {
-  virtual void anchor();
-
-public:
-  LexicalScope(LexicalScope *P, const MDNode *D, const MDNode *I, bool A)
-    : Parent(P), Desc(D), InlinedAtLocation(I), AbstractScope(A),
-      LastInsn(0), FirstInsn(0), DFSIn(0), DFSOut(0) {
-    if (Parent)
-      Parent->addChild(this);
-  }
-
-  virtual ~LexicalScope() {}
-
-  // Accessors.
-  LexicalScope *getParent() const                { return Parent; }
-  const MDNode *getDesc() const                  { return Desc; }
-  const MDNode *getInlinedAt() const             { return InlinedAtLocation; }
-  const MDNode *getScopeNode() const             { return Desc; }
-  bool isAbstractScope() const                   { return AbstractScope; }
-  SmallVectorImpl<LexicalScope *> &getChildren() { return Children; }
-  SmallVectorImpl<InsnRange> &getRanges()        { return Ranges; }
-
-  /// addChild - Add a child scope.
-  void addChild(LexicalScope *S) { Children.push_back(S); }
-
-  /// openInsnRange - This scope covers instruction range starting from MI.
-  void openInsnRange(const MachineInstr *MI) {
-    if (!FirstInsn)
-      FirstInsn = MI;
-
-    if (Parent)
-      Parent->openInsnRange(MI);
-  }
-
-  /// extendInsnRange - Extend the current instruction range covered by
-  /// this scope.
-  void extendInsnRange(const MachineInstr *MI) {
-    assert (FirstInsn && "MI Range is not open!");
-    LastInsn = MI;
-    if (Parent)
-      Parent->extendInsnRange(MI);
-  }
-
-  /// closeInsnRange - Create a range based on FirstInsn and LastInsn collected
-  /// until now. This is used when a new scope is encountered while walking
-  /// machine instructions.
-  void closeInsnRange(LexicalScope *NewScope = NULL) {
-    assert (LastInsn && "Last insn missing!");
-    Ranges.push_back(InsnRange(FirstInsn, LastInsn));
-    FirstInsn = NULL;
-    LastInsn = NULL;
-    // If Parent dominates NewScope then do not close Parent's instruction
-    // range.
-    if (Parent && (!NewScope || !Parent->dominates(NewScope)))
-      Parent->closeInsnRange(NewScope);
-  }
-
-  /// dominates - Return true if current scope dominates given lexical scope.
-  bool dominates(const LexicalScope *S) const {
-    if (S == this)
-      return true;
-    if (DFSIn < S->getDFSIn() && DFSOut > S->getDFSOut())
-      return true;
-    return false;
-  }
-
-  // Depth First Search support to walk and manipulate LexicalScope hierarchy.
-  unsigned getDFSOut() const            { return DFSOut; }
-  void setDFSOut(unsigned O)            { DFSOut = O; }
-  unsigned getDFSIn() const             { return DFSIn; }
-  void setDFSIn(unsigned I)             { DFSIn = I; }
-
-  /// dump - print lexical scope.
-  void dump(unsigned Indent = 0) const;
-
-private:
-  LexicalScope *Parent;                          // Parent to this scope.
-  AssertingVH<const MDNode> Desc;                // Debug info descriptor.
-  AssertingVH<const MDNode> InlinedAtLocation;   // Location at which this 
-                                                 // scope is inlined.
-  bool AbstractScope;                            // Abstract Scope
-  SmallVector<LexicalScope *, 4> Children;       // Scopes defined in scope.  
-                                                 // Contents not owned.
-  SmallVector<InsnRange, 4> Ranges;
-
-  const MachineInstr *LastInsn;       // Last instruction of this scope.
-  const MachineInstr *FirstInsn;      // First instruction of this scope.
-  unsigned DFSIn, DFSOut;             // In & Out Depth use to determine
-                                      // scope nesting.
-};
-
 } // end llvm namespace
 
 #endif
diff --git a/include/llvm/CodeGen/LinkAllCodegenComponents.h b/include/llvm/CodeGen/LinkAllCodegenComponents.h
index 916c0f233ef8..372c294da306 100644
--- a/include/llvm/CodeGen/LinkAllCodegenComponents.h
+++ b/include/llvm/CodeGen/LinkAllCodegenComponents.h
@@ -40,12 +40,15 @@ namespace {
       llvm::linkErlangGC();
       llvm::linkShadowStackGC();
 
-      (void) llvm::createBURRListDAGScheduler(NULL, llvm::CodeGenOpt::Default);
-      (void) llvm::createSourceListDAGScheduler(NULL,llvm::CodeGenOpt::Default);
-      (void) llvm::createHybridListDAGScheduler(NULL,llvm::CodeGenOpt::Default);
-      (void) llvm::createFastDAGScheduler(NULL, llvm::CodeGenOpt::Default);
-      (void) llvm::createDefaultScheduler(NULL, llvm::CodeGenOpt::Default);
-      (void) llvm::createVLIWDAGScheduler(NULL, llvm::CodeGenOpt::Default);
+      (void) llvm::createBURRListDAGScheduler(nullptr,
+                                              llvm::CodeGenOpt::Default);
+      (void) llvm::createSourceListDAGScheduler(nullptr,
+                                                llvm::CodeGenOpt::Default);
+      (void) llvm::createHybridListDAGScheduler(nullptr,
+                                                llvm::CodeGenOpt::Default);
+      (void) llvm::createFastDAGScheduler(nullptr, llvm::CodeGenOpt::Default);
+      (void) llvm::createDefaultScheduler(nullptr, llvm::CodeGenOpt::Default);
+      (void) llvm::createVLIWDAGScheduler(nullptr, llvm::CodeGenOpt::Default);
 
     }
   } ForceCodegenLinking; // Force link by creating a global definition.
diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h
index 3a9fef6fbd78..6629e6046532 100644
--- a/include/llvm/CodeGen/LiveInterval.h
+++ b/include/llvm/CodeGen/LiveInterval.h
@@ -116,13 +116,13 @@ namespace llvm {
     /// Return the value leaving the instruction, if any. This can be a
     /// live-through value, or a live def. A dead def returns NULL.
     VNInfo *valueOut() const {
-      return isDeadDef() ? 0 : LateVal;
+      return isDeadDef() ? nullptr : LateVal;
     }
 
     /// Return the value defined by this instruction, if any. This includes
     /// dead defs, it is the value created by the instruction's def operands.
     VNInfo *valueDefined() const {
-      return EarlyVal == LateVal ? 0 : LateVal;
+      return EarlyVal == LateVal ? nullptr : LateVal;
     }
 
     /// Return the end point of the last live range segment to interact with
@@ -154,7 +154,7 @@ namespace llvm {
       SlotIndex end;    // End point of the interval (exclusive)
       VNInfo *valno;    // identifier for the value contained in this segment.
 
-      Segment() : valno(0) {}
+      Segment() : valno(nullptr) {}
 
       Segment(SlotIndex S, SlotIndex E, VNInfo *V)
         : start(S), end(E), valno(V) {
@@ -173,7 +173,7 @@ namespace llvm {
       }
 
       bool operator<(const Segment &Other) const {
-        return start < Other.start || (start == Other.start && end < Other.end);
+        return std::tie(start, end) < std::tie(Other.start, Other.end);
       }
       bool operator==(const Segment &Other) const {
         return start == Other.start && end == Other.end;
@@ -336,20 +336,20 @@ namespace llvm {
     /// is none.
     const Segment *getSegmentContaining(SlotIndex Idx) const {
       const_iterator I = FindSegmentContaining(Idx);
-      return I == end() ? 0 : &*I;
+      return I == end() ? nullptr : &*I;
     }
 
     /// Return the live segment that contains the specified index, or null if
     /// there is none.
     Segment *getSegmentContaining(SlotIndex Idx) {
       iterator I = FindSegmentContaining(Idx);
-      return I == end() ? 0 : &*I;
+      return I == end() ? nullptr : &*I;
     }
 
     /// getVNInfoAt - Return the VNInfo that is live at Idx, or NULL.
     VNInfo *getVNInfoAt(SlotIndex Idx) const {
       const_iterator I = FindSegmentContaining(Idx);
-      return I == end() ? 0 : I->valno;
+      return I == end() ? nullptr : I->valno;
     }
 
     /// getVNInfoBefore - Return the VNInfo that is live up to but not
@@ -357,7 +357,7 @@ namespace llvm {
     /// used by an instruction at this SlotIndex position.
     VNInfo *getVNInfoBefore(SlotIndex Idx) const {
       const_iterator I = FindSegmentContaining(Idx.getPrevSlot());
-      return I == end() ? 0 : I->valno;
+      return I == end() ? nullptr : I->valno;
     }
 
     /// Return an iterator to the segment that contains the specified index, or
@@ -443,13 +443,13 @@ namespace llvm {
       const_iterator I = find(Idx.getBaseIndex());
       const_iterator E = end();
       if (I == E)
-        return LiveQueryResult(0, 0, SlotIndex(), false);
+        return LiveQueryResult(nullptr, nullptr, SlotIndex(), false);
 
       // Is this an instruction live-in segment?
       // If Idx is the start index of a basic block, include live-in segments
       // that start at Idx.getBaseIndex().
-      VNInfo *EarlyVal = 0;
-      VNInfo *LateVal  = 0;
+      VNInfo *EarlyVal = nullptr;
+      VNInfo *LateVal  = nullptr;
       SlotIndex EndPoint;
       bool Kill = false;
       if (I->start <= Idx.getBaseIndex()) {
@@ -466,7 +466,7 @@ namespace llvm {
         // predecessor.
         // Such a value is not live-in.
         if (EarlyVal->def == Idx.getBaseIndex())
-          EarlyVal = 0;
+          EarlyVal = nullptr;
       }
       // I now points to the segment that may be live-through, or defined by
       // this instr. Ignore segments starting after the current instr.
@@ -552,8 +552,7 @@ namespace llvm {
     bool operator<(const LiveInterval& other) const {
       const SlotIndex &thisIndex = beginIndex();
       const SlotIndex &otherIndex = other.beginIndex();
-      return thisIndex < otherIndex ||
-              (thisIndex == otherIndex && reg < other.reg);
+      return std::tie(thisIndex, reg) < std::tie(otherIndex, other.reg);
     }
 
     void print(raw_ostream &OS) const;
@@ -598,7 +597,7 @@ namespace llvm {
   public:
     /// Create a LiveRangeUpdater for adding segments to LR.
     /// LR will temporarily be in an invalid state until flush() is called.
-    LiveRangeUpdater(LiveRange *lr = 0) : LR(lr) {}
+    LiveRangeUpdater(LiveRange *lr = nullptr) : LR(lr) {}
 
     ~LiveRangeUpdater() { flush(); }
 
diff --git a/include/llvm/CodeGen/LiveIntervalAnalysis.h b/include/llvm/CodeGen/LiveIntervalAnalysis.h
index d8437f09aaa7..176665bc2566 100644
--- a/include/llvm/CodeGen/LiveIntervalAnalysis.h
+++ b/include/llvm/CodeGen/LiveIntervalAnalysis.h
@@ -45,6 +45,7 @@ namespace llvm {
   class TargetInstrInfo;
   class TargetRegisterClass;
   class VirtRegMap;
+  class MachineBlockFrequencyInfo;
 
   class LiveIntervals : public MachineFunctionPass {
     MachineFunction* MF;
@@ -100,7 +101,9 @@ namespace llvm {
     virtual ~LiveIntervals();
 
     // Calculate the spill weight to assign to a single instruction.
-    static float getSpillWeight(bool isDef, bool isUse, BlockFrequency freq);
+    static float getSpillWeight(bool isDef, bool isUse,
+                                const MachineBlockFrequencyInfo *MBFI,
+                                const MachineInstr *Instr);
 
     LiveInterval &getInterval(unsigned Reg) {
       if (hasInterval(Reg))
@@ -134,7 +137,7 @@ namespace llvm {
     // Interval removal.
     void removeInterval(unsigned Reg) {
       delete VirtRegIntervals[Reg];
-      VirtRegIntervals[Reg] = 0;
+      VirtRegIntervals[Reg] = nullptr;
     }
 
     /// Given a register and an instruction, adds a live segment from that
@@ -150,7 +153,18 @@ namespace llvm {
     /// Return true if the interval may have been separated into multiple
     /// connected components.
     bool shrinkToUses(LiveInterval *li,
-                      SmallVectorImpl<MachineInstr*> *dead = 0);
+                      SmallVectorImpl<MachineInstr*> *dead = nullptr);
+
+    /// \brief Walk the values in the given interval and compute which ones
+    /// are dead.  Dead values are not deleted, however:
+    /// - Dead PHIDef values are marked as unused.
+    /// - New dead machine instructions are added to the dead vector.
+    /// - CanSeparate is set to true if the interval may have been separated
+    ///   into multiple connected components.
+    void computeDeadValues(LiveInterval *li,
+                           LiveRange &LR,
+                           bool *CanSeparate,
+                           SmallVectorImpl<MachineInstr*> *dead);
 
     /// extendToIndices - Extend the live range of LI to reach all points in
     /// Indices. The points in the Indices array must be jointly dominated by
@@ -252,14 +266,14 @@ namespace llvm {
 
     VNInfo::Allocator& getVNInfoAllocator() { return VNInfoAllocator; }
 
-    virtual void getAnalysisUsage(AnalysisUsage &AU) const;
-    virtual void releaseMemory();
+    void getAnalysisUsage(AnalysisUsage &AU) const override;
+    void releaseMemory() override;
 
     /// runOnMachineFunction - pass entry point
-    virtual bool runOnMachineFunction(MachineFunction&);
+    bool runOnMachineFunction(MachineFunction&) override;
 
     /// print - Implement the dump method.
-    virtual void print(raw_ostream &O, const Module* = 0) const;
+    void print(raw_ostream &O, const Module* = nullptr) const override;
 
     /// intervalIsInOneMBB - If LI is confined to a single basic block, return
     /// a pointer to that block.  If LI is live in to or out of any block,
diff --git a/include/llvm/CodeGen/LiveIntervalUnion.h b/include/llvm/CodeGen/LiveIntervalUnion.h
index 95933d11dbb3..2f40509a1111 100644
--- a/include/llvm/CodeGen/LiveIntervalUnion.h
+++ b/include/llvm/CodeGen/LiveIntervalUnion.h
@@ -122,8 +122,8 @@ public:
     {}
 
     void clear() {
-      LiveUnion = NULL;
-      VirtReg = NULL;
+      LiveUnion = nullptr;
+      VirtReg = nullptr;
       InterferingVRegs.clear();
       CheckedFirstInterference = false;
       SeenAllInterferences = false;
@@ -182,7 +182,7 @@ public:
     unsigned Size;
     LiveIntervalUnion *LIUs;
   public:
-    Array() : Size(0), LIUs(0) {}
+    Array() : Size(0), LIUs(nullptr) {}
     ~Array() { clear(); }
 
     // Initialize the array to have Size entries.
diff --git a/include/llvm/CodeGen/LivePhysRegs.h b/include/llvm/CodeGen/LivePhysRegs.h
new file mode 100644
index 000000000000..847092b1d824
--- /dev/null
+++ b/include/llvm/CodeGen/LivePhysRegs.h
@@ -0,0 +1,146 @@
+//===- llvm/CodeGen/LivePhysRegs.h - Live Physical Register Set -*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the LivePhysRegs utility for tracking liveness of
+// physical registers. This can be used for ad-hoc liveness tracking after
+// register allocation. You can start with the live-ins/live-outs at the
+// beginning/end of a block and update the information while walking the
+// instructions inside the block. This implementation tracks the liveness on a
+// sub-register granularity.
+//
+// We assume that the high bits of a physical super-register are not preserved
+// unless the instruction has an implicit-use operand reading the super-
+// register.
+//
+// X86 Example:
+// %YMM0<def> = ...
+// %XMM0<def> = ... (Kills %XMM0, all %XMM0s sub-registers, and %YMM0)
+//
+// %YMM0<def> = ...
+// %XMM0<def> = ..., %YMM0<imp-use> (%YMM0 and all its sub-registers are alive)
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_LIVE_PHYS_REGS_H
+#define LLVM_CODEGEN_LIVE_PHYS_REGS_H
+
+#include "llvm/ADT/SparseSet.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+#include <cassert>
+
+namespace llvm {
+
+class MachineInstr;
+
+/// \brief A set of live physical registers with functions to track liveness
+/// when walking backward/forward through a basic block.
+class LivePhysRegs {
+  const TargetRegisterInfo *TRI;
+  SparseSet<unsigned> LiveRegs;
+
+  LivePhysRegs(const LivePhysRegs&) LLVM_DELETED_FUNCTION;
+  LivePhysRegs &operator=(const LivePhysRegs&) LLVM_DELETED_FUNCTION;
+public:
+  /// \brief Constructs a new empty LivePhysRegs set.
+  LivePhysRegs() : TRI(nullptr), LiveRegs() {}
+
+  /// \brief Constructs and initialize an empty LivePhysRegs set.
+  LivePhysRegs(const TargetRegisterInfo *TRI) : TRI(TRI) {
+    assert(TRI && "Invalid TargetRegisterInfo pointer.");
+    LiveRegs.setUniverse(TRI->getNumRegs());
+  }
+
+  /// \brief Clear and initialize the LivePhysRegs set.
+  void init(const TargetRegisterInfo *_TRI) {
+    assert(_TRI && "Invalid TargetRegisterInfo pointer.");
+    TRI = _TRI;
+    LiveRegs.clear();
+    LiveRegs.setUniverse(TRI->getNumRegs());
+  }
+
+  /// \brief Clears the LivePhysRegs set.
+  void clear() { LiveRegs.clear(); }
+
+  /// \brief Returns true if the set is empty.
+  bool empty() const { return LiveRegs.empty(); }
+
+  /// \brief Adds a physical register and all its sub-registers to the set.
+  void addReg(unsigned Reg) {
+    assert(TRI && "LivePhysRegs is not initialized.");
+    assert(Reg <= TRI->getNumRegs() && "Expected a physical register.");
+    for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
+         SubRegs.isValid(); ++SubRegs)
+      LiveRegs.insert(*SubRegs);
+  }
+
+  /// \brief Removes a physical register, all its sub-registers, and all its
+  /// super-registers from the set.
+  void removeReg(unsigned Reg) {
+    assert(TRI && "LivePhysRegs is not initialized.");
+    assert(Reg <= TRI->getNumRegs() && "Expected a physical register.");
+    for (MCSubRegIterator SubRegs(Reg, TRI, /*IncludeSelf=*/true);
+         SubRegs.isValid(); ++SubRegs)
+      LiveRegs.erase(*SubRegs);
+    for (MCSuperRegIterator SuperRegs(Reg, TRI, /*IncludeSelf=*/false);
+         SuperRegs.isValid(); ++SuperRegs)
+      LiveRegs.erase(*SuperRegs);
+  }
+
+  /// \brief Removes physical registers clobbered by the regmask operand @p MO.
+  void removeRegsInMask(const MachineOperand &MO);
+
+  /// \brief Returns true if register @p Reg is contained in the set. This also
+  /// works if only the super register of @p Reg has been defined, because we
+  /// always add also all sub-registers to the set.
+  bool contains(unsigned Reg) const { return LiveRegs.count(Reg); }
+
+  /// \brief Simulates liveness when stepping backwards over an
+  /// instruction(bundle): Remove Defs, add uses. This is the recommended way of
+  /// calculating liveness.
+  void stepBackward(const MachineInstr &MI);
+
+  /// \brief Simulates liveness when stepping forward over an
+  /// instruction(bundle): Remove killed-uses, add defs. This is the not
+  /// recommended way, because it depends on accurate kill flags. If possible
+  /// use stepBackwards() instead of this function.
+  void stepForward(const MachineInstr &MI);
+
+  /// \brief Adds all live-in registers of basic block @p MBB.
+  void addLiveIns(const MachineBasicBlock *MBB) {
+    for (MachineBasicBlock::livein_iterator LI = MBB->livein_begin(),
+         LE = MBB->livein_end(); LI != LE; ++LI)
+      addReg(*LI);
+  }
+
+  /// \brief Adds all live-out registers of basic block @p MBB.
+  void addLiveOuts(const MachineBasicBlock *MBB) {
+    for (MachineBasicBlock::const_succ_iterator SI = MBB->succ_begin(),
+         SE = MBB->succ_end(); SI != SE; ++SI)
+      addLiveIns(*SI);
+  }
+
+  typedef SparseSet<unsigned>::const_iterator const_iterator;
+  const_iterator begin() const { return LiveRegs.begin(); }
+  const_iterator end() const { return LiveRegs.end(); }
+
+  /// \brief Prints the currently live registers to @p OS.
+  void print(raw_ostream &OS) const;
+
+  /// \brief Dumps the currently live registers to the debug output.
+  void dump() const;
+};
+
+inline raw_ostream &operator<<(raw_ostream &OS, const LivePhysRegs& LR) {
+  LR.print(OS);
+  return OS;
+}
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_LIVE_PHYS_REGS_H
diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h
index 7edf67cc24d1..5767cab1a4db 100644
--- a/include/llvm/CodeGen/LiveRangeEdit.h
+++ b/include/llvm/CodeGen/LiveRangeEdit.h
@@ -99,7 +99,7 @@ private:
 
   /// MachineRegisterInfo callback to notify when new virtual
   /// registers are created.
-  void MRI_NoteNewVirtualRegister(unsigned VReg);
+  void MRI_NoteNewVirtualRegister(unsigned VReg) override;
 
 public:
   /// Create a LiveRangeEdit for breaking down parent into smaller pieces.
@@ -116,7 +116,7 @@ public:
                 MachineFunction &MF,
                 LiveIntervals &lis,
                 VirtRegMap *vrm,
-                Delegate *delegate = 0)
+                Delegate *delegate = nullptr)
     : Parent(parent), NewRegs(newRegs),
       MRI(MF.getRegInfo()), LIS(lis), VRM(vrm),
       TII(*MF.getTarget().getInstrInfo()),
@@ -174,7 +174,7 @@ public:
   struct Remat {
     VNInfo *ParentVNI;      // parent_'s value at the remat location.
     MachineInstr *OrigMI;   // Instruction defining ParentVNI.
-    explicit Remat(VNInfo *ParentVNI) : ParentVNI(ParentVNI), OrigMI(0) {}
+    explicit Remat(VNInfo *ParentVNI) : ParentVNI(ParentVNI), OrigMI(nullptr) {}
   };
 
   /// canRematerializeAt - Determine if ParentVNI can be rematerialized at
diff --git a/include/llvm/CodeGen/LiveRegMatrix.h b/include/llvm/CodeGen/LiveRegMatrix.h
index 7a3e9e8347f4..878b4d9836f2 100644
--- a/include/llvm/CodeGen/LiveRegMatrix.h
+++ b/include/llvm/CodeGen/LiveRegMatrix.h
@@ -25,7 +25,6 @@
 #define LLVM_CODEGEN_LIVEREGMATRIX_H
 
 #include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/OwningPtr.h"
 #include "llvm/CodeGen/LiveIntervalUnion.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 
@@ -51,7 +50,7 @@ class LiveRegMatrix : public MachineFunctionPass {
   LiveIntervalUnion::Array Matrix;
 
   // Cached queries per register unit.
-  OwningArrayPtr<LiveIntervalUnion::Query> Queries;
+  std::unique_ptr<LiveIntervalUnion::Query[]> Queries;
 
   // Cached register mask interference info.
   unsigned RegMaskTag;
@@ -59,9 +58,9 @@ class LiveRegMatrix : public MachineFunctionPass {
   BitVector RegMaskUsable;
 
   // MachineFunctionPass boilerplate.
-  virtual void getAnalysisUsage(AnalysisUsage&) const;
-  virtual bool runOnMachineFunction(MachineFunction&);
-  virtual void releaseMemory();
+  void getAnalysisUsage(AnalysisUsage&) const override;
+  bool runOnMachineFunction(MachineFunction&) override;
+  void releaseMemory() override;
 public:
   static char ID;
   LiveRegMatrix();
diff --git a/include/llvm/CodeGen/LiveRegUnits.h b/include/llvm/CodeGen/LiveRegUnits.h
deleted file mode 100644
index 02b9c55cc61f..000000000000
--- a/include/llvm/CodeGen/LiveRegUnits.h
+++ /dev/null
@@ -1,88 +0,0 @@
-//===-- llvm/CodeGen/LiveRegUnits.h - Live register unit set ----*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements a Set of live register units. This can be used for ad
-// hoc liveness tracking after register allocation. You can start with the
-// live-ins/live-outs at the beginning/end of a block and update the information
-// while walking the instructions inside the block.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_LIVEREGUNITS_H
-#define LLVM_CODEGEN_LIVEREGUNITS_H
-
-#include "llvm/ADT/SparseSet.h"
-#include "llvm/CodeGen/MachineBasicBlock.h"
-#include "llvm/Target/TargetRegisterInfo.h"
-#include <cassert>
-
-namespace llvm {
-
-class MachineInstr;
-
-/// A set of live register units with functions to track liveness when walking
-/// backward/forward through a basic block.
-class LiveRegUnits {
-  SparseSet<unsigned> LiveUnits;
-
-  LiveRegUnits(const LiveRegUnits&) LLVM_DELETED_FUNCTION;
-  LiveRegUnits &operator=(const LiveRegUnits&) LLVM_DELETED_FUNCTION;
-public:
-  /// \brief Constructs a new empty LiveRegUnits set.
-  LiveRegUnits() {}
-
-  void init(const TargetRegisterInfo *TRI) {
-    LiveUnits.clear();
-    LiveUnits.setUniverse(TRI->getNumRegs());
-  }
-
-  void clear() { LiveUnits.clear(); }
-
-  bool empty() const { return LiveUnits.empty(); }
-
-  /// \brief Adds a register to the set.
-  void addReg(unsigned Reg, const MCRegisterInfo &MCRI) {
-    for (MCRegUnitIterator RUnits(Reg, &MCRI); RUnits.isValid(); ++RUnits)
-      LiveUnits.insert(*RUnits);
-  }
-
-  /// \brief Removes a register from the set.
-  void removeReg(unsigned Reg, const MCRegisterInfo &MCRI) {
-    for (MCRegUnitIterator RUnits(Reg, &MCRI); RUnits.isValid(); ++RUnits)
-      LiveUnits.erase(*RUnits);
-  }
-
-  /// \brief Removes registers clobbered by the regmask operand @p Op.
-  void removeRegsInMask(const MachineOperand &Op, const MCRegisterInfo &MCRI);
-
-  /// \brief Returns true if register @p Reg (or one of its super register) is
-  /// contained in the set.
-  bool contains(unsigned Reg, const MCRegisterInfo &MCRI) const {
-    for (MCRegUnitIterator RUnits(Reg, &MCRI); RUnits.isValid(); ++RUnits) {
-      if (LiveUnits.count(*RUnits))
-        return true;
-    }
-    return false;
-  }
-
-  /// \brief Simulates liveness when stepping backwards over an
-  /// instruction(bundle): Remove Defs, add uses.
-  void stepBackward(const MachineInstr &MI, const MCRegisterInfo &MCRI);
-
-  /// \brief Simulates liveness when stepping forward over an
-  /// instruction(bundle): Remove killed-uses, add defs.
-  void stepForward(const MachineInstr &MI, const MCRegisterInfo &MCRI);
-
-  /// \brief Adds all registers in the live-in list of block @p BB.
-  void addLiveIns(const MachineBasicBlock *MBB, const MCRegisterInfo &MCRI);
-};
-
-} // namespace llvm
-
-#endif
diff --git a/include/llvm/CodeGen/LiveStackAnalysis.h b/include/llvm/CodeGen/LiveStackAnalysis.h
index 92c35f784d4c..df683983fa53 100644
--- a/include/llvm/CodeGen/LiveStackAnalysis.h
+++ b/include/llvm/CodeGen/LiveStackAnalysis.h
@@ -85,14 +85,14 @@ namespace llvm {
 
     VNInfo::Allocator& getVNInfoAllocator() { return VNInfoAllocator; }
 
-    virtual void getAnalysisUsage(AnalysisUsage &AU) const;
-    virtual void releaseMemory();
+    void getAnalysisUsage(AnalysisUsage &AU) const override;
+    void releaseMemory() override;
 
     /// runOnMachineFunction - pass entry point
-    virtual bool runOnMachineFunction(MachineFunction&);
+    bool runOnMachineFunction(MachineFunction&) override;
 
     /// print - Implement the dump method.
-    virtual void print(raw_ostream &O, const Module* = 0) const;
+    void print(raw_ostream &O, const Module* = nullptr) const override;
   };
 }
 
diff --git a/include/llvm/CodeGen/LiveVariables.h b/include/llvm/CodeGen/LiveVariables.h
index dc735f7b50b9..a4a5fcc31e12 100644
--- a/include/llvm/CodeGen/LiveVariables.h
+++ b/include/llvm/CodeGen/LiveVariables.h
@@ -177,7 +177,7 @@ private:   // Intermediate data structures
   void analyzePHINodes(const MachineFunction& Fn);
 public:
 
-  virtual bool runOnMachineFunction(MachineFunction &MF);
+  bool runOnMachineFunction(MachineFunction &MF) override;
 
   /// RegisterDefIsDead - Return true if the specified instruction defines the
   /// specified register, but that definition is dead.
@@ -258,10 +258,10 @@ public:
     (void)Removed;
     return true;
   }
-  
-  void getAnalysisUsage(AnalysisUsage &AU) const;
 
-  virtual void releaseMemory() {
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  void releaseMemory() override {
     VirtRegInfo.clear();
   }
 
diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h
index 7717809e0d9f..a08cc2eb508a 100644
--- a/include/llvm/CodeGen/MachineBasicBlock.h
+++ b/include/llvm/CodeGen/MachineBasicBlock.h
@@ -160,7 +160,7 @@ public:
     template<class OtherTy, class OtherIterTy>
     bundle_iterator(const bundle_iterator<OtherTy, OtherIterTy> &I)
       : MII(I.getInstrIterator()) {}
-    bundle_iterator() : MII(0) {}
+    bundle_iterator() : MII(nullptr) {}
 
     Ty &operator*() const { return *MII; }
     Ty *operator->() const { return &operator*(); }
@@ -219,10 +219,15 @@ public:
   unsigned size() const { return (unsigned)Insts.size(); }
   bool empty() const { return Insts.empty(); }
 
-  MachineInstr& front() { return Insts.front(); }
-  MachineInstr& back()  { return Insts.back(); }
-  const MachineInstr& front() const { return Insts.front(); }
-  const MachineInstr& back()  const { return Insts.back(); }
+  MachineInstr       &instr_front()       { return Insts.front(); }
+  MachineInstr       &instr_back()        { return Insts.back();  }
+  const MachineInstr &instr_front() const { return Insts.front(); }
+  const MachineInstr &instr_back()  const { return Insts.back();  }
+
+  MachineInstr       &front()             { return Insts.front(); }
+  MachineInstr       &back()              { return *--end();      }
+  const MachineInstr &front()       const { return Insts.front(); }
+  const MachineInstr &back()        const { return *--end();      }
 
   instr_iterator                instr_begin()       { return Insts.begin();  }
   const_instr_iterator          instr_begin() const { return Insts.begin();  }
@@ -242,6 +247,12 @@ public:
   reverse_iterator       rend  ()       { return instr_rend();   }
   const_reverse_iterator rend  () const { return instr_rend();   }
 
+  inline iterator_range<iterator> terminators() {
+    return iterator_range<iterator>(getFirstTerminator(), end());
+  }
+  inline iterator_range<const_iterator> terminators() const {
+    return iterator_range<const_iterator>(getFirstTerminator(), end());
+  }
 
   // Machine-CFG iterators
   typedef std::vector<MachineBasicBlock *>::iterator       pred_iterator;
@@ -256,7 +267,6 @@ public:
                                                          succ_reverse_iterator;
   typedef std::vector<MachineBasicBlock *>::const_reverse_iterator
                                                    const_succ_reverse_iterator;
-
   pred_iterator        pred_begin()       { return Predecessors.begin(); }
   const_pred_iterator  pred_begin() const { return Predecessors.begin(); }
   pred_iterator        pred_end()         { return Predecessors.end();   }
@@ -290,6 +300,19 @@ public:
   }
   bool                 succ_empty() const { return Successors.empty();   }
 
+  inline iterator_range<pred_iterator> predecessors() {
+    return iterator_range<pred_iterator>(pred_begin(), pred_end());
+  }
+  inline iterator_range<const_pred_iterator> predecessors() const {
+    return iterator_range<const_pred_iterator>(pred_begin(), pred_end());
+  }
+  inline iterator_range<succ_iterator> successors() {
+    return iterator_range<succ_iterator>(succ_begin(), succ_end());
+  }
+  inline iterator_range<const_succ_iterator> successors() const {
+    return iterator_range<const_succ_iterator>(succ_begin(), succ_end());
+  }
+
   // LiveIn management methods.
 
   /// addLiveIn - Add the specified register as a live in.  Note that it
@@ -363,6 +386,9 @@ public:
   ///
   void addSuccessor(MachineBasicBlock *succ, uint32_t weight = 0);
 
+  /// Set successor weight of a given iterator.
+  void setSuccWeight(succ_iterator I, uint32_t weight);
+
   /// removeSuccessor - Remove successor from the successors list of this
   /// MachineBasicBlock. The Predecessors list of succ is automatically updated.
   ///
@@ -500,7 +526,7 @@ public:
   ///
   /// If I points to a bundle of instructions, they are all erased.
   iterator erase(iterator I) {
-    return erase(I, llvm::next(I));
+    return erase(I, std::next(I));
   }
 
   /// Remove an instruction from the instruction list and delete it.
@@ -539,7 +565,7 @@ public:
   void splice(iterator Where, MachineBasicBlock *Other, iterator From) {
     // The range splice() doesn't allow noop moves, but this one does.
     if (Where != From)
-      splice(Where, Other, From, llvm::next(From));
+      splice(Where, Other, From, std::next(From));
   }
 
   /// Take a block of instructions from MBB 'Other' in the range [From, To),
@@ -594,7 +620,7 @@ public:
 
   /// computeRegisterLiveness - Return whether (physical) register \c Reg
   /// has been <def>ined and not <kill>ed as of just before \c MI.
-  /// 
+  ///
   /// Search is localised to a neighborhood of
   /// \c Neighborhood instructions before (searching for defs or kills) and
   /// Neighborhood instructions after (searching just for defs) MI.
@@ -606,7 +632,10 @@ public:
 
   // Debugging methods.
   void dump() const;
-  void print(raw_ostream &OS, SlotIndexes* = 0) const;
+  void print(raw_ostream &OS, SlotIndexes* = nullptr) const;
+
+  // Printing method used by LoopInfo.
+  void printAsOperand(raw_ostream &OS, bool PrintType = true) const;
 
   /// getNumber - MachineBasicBlocks are uniquely numbered at the function
   /// level, unless they're not in a MachineFunction yet, in which case this
@@ -655,8 +684,6 @@ private:
 
 raw_ostream& operator<<(raw_ostream &OS, const MachineBasicBlock &MBB);
 
-void WriteAsOperand(raw_ostream &, const MachineBasicBlock*, bool t);
-
 // This is useful when building IndexedMaps keyed on basic block pointers.
 struct MBB2NumberFunctor :
   public std::unary_function<const MachineBasicBlock*, unsigned> {
@@ -746,11 +773,11 @@ public:
   MachineInstrSpan(MachineBasicBlock::iterator I)
     : MBB(*I->getParent()),
       I(I),
-      B(I == MBB.begin() ? MBB.end() : llvm::prior(I)),
-      E(llvm::next(I)) {}
+      B(I == MBB.begin() ? MBB.end() : std::prev(I)),
+      E(std::next(I)) {}
 
   MachineBasicBlock::iterator begin() {
-    return B == MBB.end() ? MBB.begin() : llvm::next(B);
+    return B == MBB.end() ? MBB.begin() : std::next(B);
   }
   MachineBasicBlock::iterator end() { return E; }
   bool empty() { return begin() == end(); }
diff --git a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
index a9c7bf7dbc60..1aef689eb7a4 100644
--- a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
+++ b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h
@@ -1,4 +1,4 @@
-//====----- MachineBlockFrequencyInfo.h - MachineBlock Frequency Analysis ----====//
+//===- MachineBlockFrequencyInfo.h - MBB Frequency Analysis -*- C++ -*-----===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -22,15 +22,13 @@ namespace llvm {
 
 class MachineBasicBlock;
 class MachineBranchProbabilityInfo;
-template<class BlockT, class FunctionT, class BranchProbInfoT>
-class BlockFrequencyImpl;
+template <class BlockT> class BlockFrequencyInfoImpl;
 
-/// MachineBlockFrequencyInfo pass uses BlockFrequencyImpl implementation to estimate
-/// machine basic block frequencies.
+/// MachineBlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation
+/// to estimate machine basic block frequencies.
 class MachineBlockFrequencyInfo : public MachineFunctionPass {
-
-  BlockFrequencyImpl<MachineBasicBlock, MachineFunction,
-                     MachineBranchProbabilityInfo> *MBFI;
+  typedef BlockFrequencyInfoImpl<MachineBasicBlock> ImplType;
+  std::unique_ptr<ImplType> MBFI;
 
 public:
   static char ID;
@@ -39,9 +37,11 @@ public:
 
   ~MachineBlockFrequencyInfo();
 
-  void getAnalysisUsage(AnalysisUsage &AU) const;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  bool runOnMachineFunction(MachineFunction &F) override;
 
-  bool runOnMachineFunction(MachineFunction &F);
+  void releaseMemory() override;
 
   /// getblockFreq - Return block frequency. Return 0 if we don't have the
   /// information. Please note that initial frequency is equal to 1024. It means
@@ -49,6 +49,21 @@ public:
   /// the other block frequencies. We do this to avoid using of floating points.
   ///
   BlockFrequency getBlockFreq(const MachineBasicBlock *MBB) const;
+
+  const MachineFunction *getFunction() const;
+  void view() const;
+
+  // Print the block frequency Freq to OS using the current functions entry
+  // frequency to convert freq into a relative decimal form.
+  raw_ostream &printBlockFreq(raw_ostream &OS, const BlockFrequency Freq) const;
+
+  // Convenience method that attempts to look up the frequency associated with
+  // BB and print it to OS.
+  raw_ostream &printBlockFreq(raw_ostream &OS,
+                              const MachineBasicBlock *MBB) const;
+
+  uint64_t getEntryFreq() const;
+
 };
 
 }
diff --git a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h
index c59948fd654e..7ba749559c0f 100644
--- a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h
+++ b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h
@@ -40,7 +40,7 @@ public:
     initializeMachineBranchProbabilityInfoPass(Registry);
   }
 
-  void getAnalysisUsage(AnalysisUsage &AU) const {
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.setPreservesAll();
   }
 
@@ -60,7 +60,8 @@ public:
   uint32_t getSumForBlock(const MachineBasicBlock *MBB, uint32_t &Scale) const;
 
   // A 'Hot' edge is an edge which probability is >= 80%.
-  bool isEdgeHot(MachineBasicBlock *Src, MachineBasicBlock *Dst) const;
+  bool isEdgeHot(const MachineBasicBlock *Src,
+                 const MachineBasicBlock *Dst) const;
 
   // Return a hot successor for the block BB or null if there isn't one.
   // NB: This routine's complexity is linear on the number of successors.
@@ -72,14 +73,15 @@ public:
   // NB: This routine's complexity is linear on the number of successors of
   // Src. Querying sequentially for each successor's probability is a quadratic
   // query pattern.
-  BranchProbability getEdgeProbability(MachineBasicBlock *Src,
-                                       MachineBasicBlock *Dst) const;
+  BranchProbability getEdgeProbability(const MachineBasicBlock *Src,
+                                       const MachineBasicBlock *Dst) const;
 
   // Print value between 0 (0% probability) and 1 (100% probability),
   // however the value is never equal to 0, and can be 1 only iff SRC block
   // has only one successor.
-  raw_ostream &printEdgeProbability(raw_ostream &OS, MachineBasicBlock *Src,
-                                    MachineBasicBlock *Dst) const;
+  raw_ostream &printEdgeProbability(raw_ostream &OS,
+                                    const MachineBasicBlock *Src,
+                                    const MachineBasicBlock *Dst) const;
 };
 
 }
diff --git a/include/llvm/CodeGen/MachineCodeEmitter.h b/include/llvm/CodeGen/MachineCodeEmitter.h
index 9e41e6e9c1ee..81b0ba1e7c71 100644
--- a/include/llvm/CodeGen/MachineCodeEmitter.h
+++ b/include/llvm/CodeGen/MachineCodeEmitter.h
@@ -17,8 +17,8 @@
 #ifndef LLVM_CODEGEN_MACHINECODEEMITTER_H
 #define LLVM_CODEGEN_MACHINECODEEMITTER_H
 
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/Support/DataTypes.h"
-#include "llvm/Support/DebugLoc.h"
 #include <string>
 
 namespace llvm {
@@ -262,7 +262,7 @@ public:
     // Check for buffer overflow.
     if (Size >= (uintptr_t)(BufferEnd-CurBufferPtr)) {
       CurBufferPtr = BufferEnd;
-      Result = 0;
+      Result = nullptr;
     } else {
       // Allocate the space.
       Result = CurBufferPtr;
diff --git a/include/llvm/CodeGen/MachineCodeInfo.h b/include/llvm/CodeGen/MachineCodeInfo.h
index ba9dfab91a7e..820bc87425b9 100644
--- a/include/llvm/CodeGen/MachineCodeInfo.h
+++ b/include/llvm/CodeGen/MachineCodeInfo.h
@@ -27,7 +27,7 @@ private:
   void *Address; // The address of the function in memory
 
 public:
-  MachineCodeInfo() : Size(0), Address(0) {}
+  MachineCodeInfo() : Size(0), Address(nullptr) {}
 
   void setSize(size_t s) {
     Size = s;
diff --git a/include/llvm/CodeGen/MachineConstantPool.h b/include/llvm/CodeGen/MachineConstantPool.h
index 912ce8966268..c619afb83333 100644
--- a/include/llvm/CodeGen/MachineConstantPool.h
+++ b/include/llvm/CodeGen/MachineConstantPool.h
@@ -17,6 +17,7 @@
 #define LLVM_CODEGEN_MACHINECONSTANTPOOL_H
 
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/MC/SectionKind.h"
 #include <cassert>
 #include <climits>
 #include <vector>
@@ -119,6 +120,8 @@ public:
   ///     them.
   ///  2: This entry may have arbitrary relocations. 
   unsigned getRelocationInfo() const;
+
+  SectionKind getSectionKind(const DataLayout *DL) const;
 };
   
 /// The MachineConstantPool class keeps track of constants referenced by a
diff --git a/include/llvm/CodeGen/MachineDominanceFrontier.h b/include/llvm/CodeGen/MachineDominanceFrontier.h
new file mode 100644
index 000000000000..e099e716c63d
--- /dev/null
+++ b/include/llvm/CodeGen/MachineDominanceFrontier.h
@@ -0,0 +1,109 @@
+//===- llvm/CodeGen/MachineDominanceFrontier.h ------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINEDOMINANCEFRONTIER_H
+#define LLVM_CODEGEN_MACHINEDOMINANCEFRONTIER_H
+
+#include "llvm/Analysis/DominanceFrontier.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+
+
+namespace llvm {
+
+class MachineDominanceFrontier : public MachineFunctionPass {
+  ForwardDominanceFrontierBase<MachineBasicBlock> Base;
+public:
+  typedef DominatorTreeBase<MachineBasicBlock> DomTreeT;
+  typedef DomTreeNodeBase<MachineBasicBlock> DomTreeNodeT;
+  typedef DominanceFrontierBase<MachineBasicBlock>::DomSetType DomSetType;
+  typedef DominanceFrontierBase<MachineBasicBlock>::iterator iterator;
+  typedef DominanceFrontierBase<MachineBasicBlock>::const_iterator const_iterator;
+
+  void operator=(const MachineDominanceFrontier &) LLVM_DELETED_FUNCTION;
+  MachineDominanceFrontier(const MachineDominanceFrontier &) LLVM_DELETED_FUNCTION;
+
+  static char ID;
+
+  MachineDominanceFrontier();
+
+  DominanceFrontierBase<MachineBasicBlock> &getBase() {
+    return Base;
+  }
+
+  inline const std::vector<MachineBasicBlock*> &getRoots() const {
+    return Base.getRoots();
+  }
+
+  MachineBasicBlock *getRoot() const {
+    return Base.getRoot();
+  }
+
+  bool isPostDominator() const {
+    return Base.isPostDominator();
+  }
+
+  iterator begin() {
+    return Base.begin();
+  }
+
+  const_iterator begin() const {
+    return Base.begin();
+  }
+
+  iterator end() {
+    return Base.end();
+  }
+
+  const_iterator end() const {
+    return Base.end();
+  }
+
+  iterator find(MachineBasicBlock *B) {
+    return Base.find(B);
+  }
+
+  const_iterator find(MachineBasicBlock *B) const {
+    return Base.find(B);
+  }
+
+  iterator addBasicBlock(MachineBasicBlock *BB, const DomSetType &frontier) {
+    return Base.addBasicBlock(BB, frontier);
+  }
+
+  void removeBlock(MachineBasicBlock *BB) {
+    return Base.removeBlock(BB);
+  }
+
+  void addToFrontier(iterator I, MachineBasicBlock *Node) {
+    return Base.addToFrontier(I, Node);
+  }
+
+  void removeFromFrontier(iterator I, MachineBasicBlock *Node) {
+    return Base.removeFromFrontier(I, Node);
+  }
+
+  bool compareDomSet(DomSetType &DS1, const DomSetType &DS2) const {
+    return Base.compareDomSet(DS1, DS2);
+  }
+
+  bool compare(DominanceFrontierBase<MachineBasicBlock> &Other) const {
+    return Base.compare(Other);
+  }
+
+  bool runOnMachineFunction(MachineFunction &F) override;
+
+  void releaseMemory() override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+};
+
+}
+
+#endif
diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h
index e41d206da65c..f1ae0bf5f9cf 100644
--- a/include/llvm/CodeGen/MachineDominators.h
+++ b/include/llvm/CodeGen/MachineDominators.h
@@ -15,11 +15,11 @@
 #ifndef LLVM_CODEGEN_MACHINEDOMINATORS_H
 #define LLVM_CODEGEN_MACHINEDOMINATORS_H
 
-#include "llvm/Analysis/DominatorInternals.h"
-#include "llvm/Analysis/Dominators.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/Support/GenericDomTree.h"
+#include "llvm/Support/GenericDomTreeConstruction.h"
 
 namespace llvm {
 
@@ -48,7 +48,7 @@ public:
 
   DominatorTreeBase<MachineBasicBlock>& getBase() { return *DT; }
 
-  virtual void getAnalysisUsage(AnalysisUsage &AU) const;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
 
   /// getRoots -  Return the root blocks of the current CFG.  This may include
   /// multiple blocks if we are computing post dominators.  For forward
@@ -66,7 +66,7 @@ public:
     return DT->getRootNode();
   }
 
-  virtual bool runOnMachineFunction(MachineFunction &F);
+  bool runOnMachineFunction(MachineFunction &F) override;
 
   inline bool dominates(const MachineDomTreeNode* A,
                         const MachineDomTreeNode* B) const {
@@ -166,9 +166,9 @@ public:
     return DT->isReachableFromEntry(A);
   }
 
-  virtual void releaseMemory();
+  void releaseMemory() override;
 
-  virtual void print(raw_ostream &OS, const Module*) const;
+  void print(raw_ostream &OS, const Module*) const override;
 };
 
 //===-------------------------------------
diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h
index 022634df87cf..c51f8fe03bbf 100644
--- a/include/llvm/CodeGen/MachineFrameInfo.h
+++ b/include/llvm/CodeGen/MachineFrameInfo.h
@@ -101,11 +101,6 @@ class MachineFrameInfo {
     // cannot alias any other memory objects.
     bool isSpillSlot;
 
-    // MayNeedSP - If true the stack object triggered the creation of the stack
-    // protector. We should allocate this object right after the stack
-    // protector.
-    bool MayNeedSP;
-
     /// Alloca - If this stack object is originated from an Alloca instruction
     /// this value saves the original IR allocation. Can be NULL.
     const AllocaInst *Alloca;
@@ -115,9 +110,9 @@ class MachineFrameInfo {
     bool PreAllocated;
 
     StackObject(uint64_t Sz, unsigned Al, int64_t SP, bool IM,
-                bool isSS, bool NSP, const AllocaInst *Val)
+                bool isSS, const AllocaInst *Val)
       : SPOffset(SP), Size(Sz), Alignment(Al), isImmutable(IM),
-        isSpillSlot(isSS), MayNeedSP(NSP), Alloca(Val), PreAllocated(false) {}
+        isSpillSlot(isSS), Alloca(Val), PreAllocated(false) {}
   };
 
   const TargetMachine &TM;
@@ -145,6 +140,14 @@ class MachineFrameInfo {
   /// to builtin \@llvm.returnaddress.
   bool ReturnAddressTaken;
 
+  /// HasStackMap - This boolean keeps track of whether there is a call
+  /// to builtin \@llvm.experimental.stackmap.
+  bool HasStackMap;
+
+  /// HasPatchPoint - This boolean keeps track of whether there is a call
+  /// to builtin \@llvm.experimental.patchpoint.
+  bool HasPatchPoint;
+
   /// StackSize - The prolog/epilog code inserter calculates the final stack
   /// offsets for all of the fixed size objects, updating the Objects list
   /// above.  It then updates StackSize to contain the number of bytes that need
@@ -223,6 +226,10 @@ class MachineFrameInfo {
   /// Whether the "realign-stack" option is on.
   bool RealignOption;
 
+  /// True if the function includes inline assembly that adjusts the stack
+  /// pointer.
+  bool HasInlineAsmWithSPAdjust;
+
   const TargetFrameLowering *getFrameLowering() const;
 public:
     explicit MachineFrameInfo(const TargetMachine &TM, bool RealignOpt)
@@ -231,6 +238,8 @@ public:
     HasVarSizedObjects = false;
     FrameAddressTaken = false;
     ReturnAddressTaken = false;
+    HasStackMap = false;
+    HasPatchPoint = false;
     AdjustsStack = false;
     HasCalls = false;
     StackProtectorIdx = -1;
@@ -240,6 +249,7 @@ public:
     LocalFrameSize = 0;
     LocalFrameMaxAlign = 0;
     UseLocalStackAllocationBlock = false;
+    HasInlineAsmWithSPAdjust = false;
   }
 
   /// hasStackObjects - Return true if there are any stack objects in this
@@ -276,6 +286,18 @@ public:
   bool isReturnAddressTaken() const { return ReturnAddressTaken; }
   void setReturnAddressIsTaken(bool s) { ReturnAddressTaken = s; }
 
+  /// hasStackMap - This method may be called any time after instruction
+  /// selection is complete to determine if there is a call to builtin
+  /// \@llvm.experimental.stackmap.
+  bool hasStackMap() const { return HasStackMap; }
+  void setHasStackMap(bool s = true) { HasStackMap = s; }
+
+  /// hasPatchPoint - This method may be called any time after instruction
+  /// selection is complete to determine if there is a call to builtin
+  /// \@llvm.experimental.patchpoint.
+  bool hasPatchPoint() const { return HasPatchPoint; }
+  void setHasPatchPoint(bool s = true) { HasPatchPoint = s; }
+
   /// getObjectIndexBegin - Return the minimum frame object index.
   ///
   int getObjectIndexBegin() const { return -NumFixedObjects; }
@@ -380,14 +402,6 @@ public:
     return Objects[ObjectIdx+NumFixedObjects].Alloca;
   }
 
-  /// NeedsStackProtector - Returns true if the object may need stack
-  /// protectors.
-  bool MayNeedStackProtector(int ObjectIdx) const {
-    assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() &&
-           "Invalid Object Idx!");
-    return Objects[ObjectIdx+NumFixedObjects].MayNeedSP;
-  }
-
   /// getObjectOffset - Return the assigned stack offset of the specified object
   /// from the incoming stack pointer.
   ///
@@ -451,6 +465,10 @@ public:
   bool hasCalls() const { return HasCalls; }
   void setHasCalls(bool V) { HasCalls = V; }
 
+  /// Returns true if the function contains any stack-adjusting inline assembly.
+  bool hasInlineAsmWithSPAdjust() const { return HasInlineAsmWithSPAdjust; }
+  void setHasInlineAsmWithSPAdjust(bool B) { HasInlineAsmWithSPAdjust = B; }
+
   /// getMaxCallFrameSize - Return the maximum size of a call frame that must be
   /// allocated for an outgoing function call.  This is only available if
   /// CallFrameSetup/Destroy pseudo instructions are used by the target, and
@@ -466,6 +484,9 @@ public:
   ///
   int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool Immutable);
 
+  /// CreateFixedSpillStackObject - Create a spill slot at a fixed location
+  /// on the stack.  Returns an index with a negative value.
+  int CreateFixedSpillStackObject(uint64_t Size, int64_t SPOffset);
 
   /// isFixedObjectIndex - Returns true if the specified index corresponds to a
   /// fixed stack object.
@@ -501,7 +522,7 @@ public:
   /// a nonnegative identifier to represent it.
   ///
   int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSS,
-                        bool MayNeedSP = false, const AllocaInst *Alloca = 0);
+                        const AllocaInst *Alloca = nullptr);
 
   /// CreateSpillStackObject - Create a new statically sized stack object that
   /// represents a spill slot, returning a nonnegative identifier to represent
@@ -521,7 +542,7 @@ public:
   /// variable sized object is created, whether or not the index returned is
   /// actually used.
   ///
-  int CreateVariableSizedObject(unsigned Alignment);
+  int CreateVariableSizedObject(unsigned Alignment, const AllocaInst *Alloca);
 
   /// getCalleeSavedInfo - Returns a reference to call saved info vector for the
   /// current function.
diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h
index c886e256e044..042c62b4a887 100644
--- a/include/llvm/CodeGen/MachineFunction.h
+++ b/include/llvm/CodeGen/MachineFunction.h
@@ -20,9 +20,9 @@
 
 #include "llvm/ADT/ilist.h"
 #include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/ArrayRecycler.h"
-#include "llvm/Support/DebugLoc.h"
 #include "llvm/Support/Recycler.h"
 
 namespace llvm {
@@ -131,8 +131,8 @@ class MachineFunction {
   /// about the control flow of such functions.
   bool ExposesReturnsTwice;
 
-  /// True if the function includes MS-style inline assembly.
-  bool HasMSInlineAsm;
+  /// True if the function includes any inline assembly.
+  bool HasInlineAsm;
 
   MachineFunction(const MachineFunction &) LLVM_DELETED_FUNCTION;
   void operator=(const MachineFunction&) LLVM_DELETED_FUNCTION;
@@ -218,29 +218,23 @@ public:
     ExposesReturnsTwice = B;
   }
 
-  /// Returns true if the function contains any MS-style inline assembly.
-  bool hasMSInlineAsm() const {
-    return HasMSInlineAsm;
+  /// Returns true if the function contains any inline assembly.
+  bool hasInlineAsm() const {
+    return HasInlineAsm;
   }
 
-  /// Set a flag that indicates that the function contains MS-style inline
-  /// assembly.
-  void setHasMSInlineAsm(bool B) {
-    HasMSInlineAsm = B;
+  /// Set a flag that indicates that the function contains inline assembly.
+  void setHasInlineAsm(bool B) {
+    HasInlineAsm = B;
   }
-  
+
   /// getInfo - Keep track of various per-function pieces of information for
   /// backends that would like to do so.
   ///
   template<typename Ty>
   Ty *getInfo() {
-    if (!MFInfo) {
-        // This should be just `new (Allocator.Allocate<Ty>()) Ty(*this)', but
-        // that apparently breaks GCC 3.3.
-        Ty *Loc = static_cast<Ty*>(Allocator.Allocate(sizeof(Ty),
-                                                      AlignOf<Ty>::Alignment));
-        MFInfo = new (Loc) Ty(*this);
-    }
+    if (!MFInfo)
+      MFInfo = new (Allocator.Allocate<Ty>()) Ty(*this);
     return static_cast<Ty*>(MFInfo);
   }
 
@@ -260,6 +254,9 @@ public:
     return MBBNumbering[N];
   }
 
+  /// Should we be emitting segmented stack stuff for the function
+  bool shouldSplitStack();
+
   /// getNumBlockIDs - Return the number of MBB ID's allocated.
   ///
   unsigned getNumBlockIDs() const { return (unsigned)MBBNumbering.size(); }
@@ -269,12 +266,12 @@ public:
   /// dense, and match the ordering of the blocks within the function.  If a
   /// specific MachineBasicBlock is specified, only that block and those after
   /// it are renumbered.
-  void RenumberBlocks(MachineBasicBlock *MBBFrom = 0);
+  void RenumberBlocks(MachineBasicBlock *MBBFrom = nullptr);
   
   /// print - Print out the MachineFunction in a format suitable for debugging
   /// to the specified stream.
   ///
-  void print(raw_ostream &OS, SlotIndexes* = 0) const;
+  void print(raw_ostream &OS, SlotIndexes* = nullptr) const;
 
   /// viewCFG - This function is meant for use from the debugger.  You can just
   /// say 'call F->viewCFG()' and a ghostview window should pop up from the
@@ -297,7 +294,7 @@ public:
 
   /// verify - Run the current MachineFunction through the machine code
   /// verifier, useful for debugger use.
-  void verify(Pass *p = NULL, const char *Banner = NULL) const;
+  void verify(Pass *p = nullptr, const char *Banner = nullptr) const;
 
   // Provide accessors for the MachineBasicBlock list...
   typedef BasicBlockListType::iterator iterator;
@@ -365,7 +362,7 @@ public:
   /// implementation.
   void removeFromMBBNumbering(unsigned N) {
     assert(N < MBBNumbering.size() && "Illegal basic block #");
-    MBBNumbering[N] = 0;
+    MBBNumbering[N] = nullptr;
   }
 
   /// CreateMachineInstr - Allocate a new MachineInstr. Use this instead
@@ -390,7 +387,7 @@ public:
   /// CreateMachineBasicBlock - Allocate a new MachineBasicBlock. Use this
   /// instead of `new MachineBasicBlock'.
   ///
-  MachineBasicBlock *CreateMachineBasicBlock(const BasicBlock *bb = 0);
+  MachineBasicBlock *CreateMachineBasicBlock(const BasicBlock *bb = nullptr);
 
   /// DeleteMachineBasicBlock - Delete the given MachineBasicBlock.
   ///
@@ -402,8 +399,8 @@ public:
   MachineMemOperand *getMachineMemOperand(MachinePointerInfo PtrInfo,
                                           unsigned f, uint64_t s,
                                           unsigned base_alignment,
-                                          const MDNode *TBAAInfo = 0,
-                                          const MDNode *Ranges = 0);
+                                          const MDNode *TBAAInfo = nullptr,
+                                          const MDNode *Ranges = nullptr);
   
   /// getMachineMemOperand - Allocate a new MachineMemOperand by copying
   /// an existing one, adjusting by an offset and using the given size.
@@ -427,6 +424,15 @@ public:
     OperandRecycler.deallocate(Cap, Array);
   }
 
+  /// \brief Allocate and initialize a register mask with @p NumRegister bits.
+  uint32_t *allocateRegisterMask(unsigned NumRegister) {
+    unsigned Size = (NumRegister + 31) / 32;
+    uint32_t *Mask = Allocator.Allocate<uint32_t>(Size);
+    for (unsigned i = 0; i != Size; ++i)
+      Mask[i] = 0;
+    return Mask;
+  }
+
   /// allocateMemRefsArray - Allocate an array to hold MachineMemOperand
   /// pointers.  This array is owned by the MachineFunction.
   MachineInstr::mmo_iterator allocateMemRefsArray(unsigned long Num);
diff --git a/include/llvm/CodeGen/MachineFunctionAnalysis.h b/include/llvm/CodeGen/MachineFunctionAnalysis.h
index 112f07ea50d7..36f1c6627096 100644
--- a/include/llvm/CodeGen/MachineFunctionAnalysis.h
+++ b/include/llvm/CodeGen/MachineFunctionAnalysis.h
@@ -34,16 +34,16 @@ public:
   ~MachineFunctionAnalysis();
 
   MachineFunction &getMF() const { return *MF; }
-  
-  virtual const char* getPassName() const {
+
+  const char* getPassName() const override {
     return "Machine Function Analysis";
   }
 
 private:
-  virtual bool doInitialization(Module &M);
-  virtual bool runOnFunction(Function &F);
-  virtual void releaseMemory();
-  virtual void getAnalysisUsage(AnalysisUsage &AU) const;
+  bool doInitialization(Module &M) override;
+  bool runOnFunction(Function &F) override;
+  void releaseMemory() override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
 };
 
 } // End llvm namespace
diff --git a/include/llvm/CodeGen/MachineFunctionPass.h b/include/llvm/CodeGen/MachineFunctionPass.h
index 04881e52ca7f..50a1f6e96217 100644
--- a/include/llvm/CodeGen/MachineFunctionPass.h
+++ b/include/llvm/CodeGen/MachineFunctionPass.h
@@ -44,14 +44,14 @@ protected:
   /// For MachineFunctionPasses, calling AU.preservesCFG() indicates that
   /// the pass does not modify the MachineBasicBlock CFG.
   ///
-  virtual void getAnalysisUsage(AnalysisUsage &AU) const;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
 
 private:
   /// createPrinterPass - Get a machine function printer pass.
-  virtual Pass *createPrinterPass(raw_ostream &O,
-                                  const std::string &Banner) const;
+  Pass *createPrinterPass(raw_ostream &O,
+                          const std::string &Banner) const override;
 
-  virtual bool runOnFunction(Function &F);
+  bool runOnFunction(Function &F) override;
 };
 
 } // End llvm namespace
diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h
index cccab81efbb2..3c828116411e 100644
--- a/include/llvm/CodeGen/MachineInstr.h
+++ b/include/llvm/CodeGen/MachineInstr.h
@@ -22,13 +22,14 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/IR/DebugInfo.h"
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/InlineAsm.h"
 #include "llvm/MC/MCInstrDesc.h"
 #include "llvm/Support/ArrayRecycler.h"
-#include "llvm/Support/DebugLoc.h"
 #include "llvm/Target/TargetOpcodes.h"
-#include <vector>
 
 namespace llvm {
 
@@ -243,6 +244,14 @@ public:
   ///
   DebugLoc getDebugLoc() const { return debugLoc; }
 
+  /// getDebugVariable() - Return the debug variable referenced by
+  /// this DBG_VALUE instruction.
+  DIVariable getDebugVariable() const {
+    assert(isDebugValue() && "not a DBG_VALUE");
+    const MDNode *Var = getOperand(getNumOperands() - 1).getMetadata();
+    return DIVariable(Var);
+  }
+
   /// emitError - Emit an error referring to the source location of this
   /// instruction. This should only be used for inline assembly that is somehow
   /// impossible to compile. Other errors should have been handled much
@@ -287,11 +296,57 @@ public:
   const_mop_iterator operands_begin() const { return Operands; }
   const_mop_iterator operands_end() const { return Operands + NumOperands; }
 
+  iterator_range<mop_iterator> operands() {
+    return iterator_range<mop_iterator>(operands_begin(), operands_end());
+  }
+  iterator_range<const_mop_iterator> operands() const {
+    return iterator_range<const_mop_iterator>(operands_begin(), operands_end());
+  }
+  iterator_range<mop_iterator> explicit_operands() {
+    return iterator_range<mop_iterator>(
+        operands_begin(), operands_begin() + getNumExplicitOperands());
+  }
+  iterator_range<const_mop_iterator> explicit_operands() const {
+    return iterator_range<const_mop_iterator>(
+        operands_begin(), operands_begin() + getNumExplicitOperands());
+  }
+  iterator_range<mop_iterator> implicit_operands() {
+    return iterator_range<mop_iterator>(explicit_operands().end(),
+                                        operands_end());
+  }
+  iterator_range<const_mop_iterator> implicit_operands() const {
+    return iterator_range<const_mop_iterator>(explicit_operands().end(),
+                                              operands_end());
+  }
+  iterator_range<mop_iterator> defs() {
+    return iterator_range<mop_iterator>(
+        operands_begin(), operands_begin() + getDesc().getNumDefs());
+  }
+  iterator_range<const_mop_iterator> defs() const {
+    return iterator_range<const_mop_iterator>(
+        operands_begin(), operands_begin() + getDesc().getNumDefs());
+  }
+  iterator_range<mop_iterator> uses() {
+    return iterator_range<mop_iterator>(
+        operands_begin() + getDesc().getNumDefs(), operands_end());
+  }
+  iterator_range<const_mop_iterator> uses() const {
+    return iterator_range<const_mop_iterator>(
+        operands_begin() + getDesc().getNumDefs(), operands_end());
+  }
+
   /// Access to memory operands of the instruction
   mmo_iterator memoperands_begin() const { return MemRefs; }
   mmo_iterator memoperands_end() const { return MemRefs + NumMemRefs; }
   bool memoperands_empty() const { return NumMemRefs == 0; }
 
+  iterator_range<mmo_iterator>  memoperands() {
+    return iterator_range<mmo_iterator>(memoperands_begin(), memoperands_end());
+  }
+  iterator_range<mmo_iterator> memoperands() const {
+    return iterator_range<mmo_iterator>(memoperands_begin(), memoperands_end());
+  }
+
   /// hasOneMemOperand - Return true if this instruction has exactly one
   /// MachineMemOperand.
   bool hasOneMemOperand() const {
@@ -623,19 +678,19 @@ public:
   /// bundle remain bundled.
   void eraseFromBundle();
 
+  bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; }
+  bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; }
+
   /// isLabel - Returns true if the MachineInstr represents a label.
   ///
-  bool isLabel() const {
-    return getOpcode() == TargetOpcode::PROLOG_LABEL ||
-           getOpcode() == TargetOpcode::EH_LABEL ||
-           getOpcode() == TargetOpcode::GC_LABEL;
+  bool isLabel() const { return isEHLabel() || isGCLabel(); }
+  bool isCFIInstruction() const {
+    return getOpcode() == TargetOpcode::CFI_INSTRUCTION;
   }
 
-  bool isPrologLabel() const {
-    return getOpcode() == TargetOpcode::PROLOG_LABEL;
-  }
-  bool isEHLabel() const { return getOpcode() == TargetOpcode::EH_LABEL; }
-  bool isGCLabel() const { return getOpcode() == TargetOpcode::GC_LABEL; }
+  // True if the instruction represents a position in the function.
+  bool isPosition() const { return isLabel() || isCFIInstruction(); }
+
   bool isDebugValue() const { return getOpcode() == TargetOpcode::DBG_VALUE; }
   /// A DBG_VALUE is indirect iff the first operand is a register and
   /// the second operand is an immediate.
@@ -672,6 +727,9 @@ public:
   bool isFullCopy() const {
     return isCopy() && !getOperand(0).getSubReg() && !getOperand(1).getSubReg();
   }
+  bool isExtractSubreg() const {
+    return getOpcode() == TargetOpcode::EXTRACT_SUBREG;
+  }
 
   /// isCopyLike - Return true if the instruction behaves like a copy.
   /// This does not include native copy instructions.
@@ -701,7 +759,7 @@ public:
     // Pseudo-instructions that don't produce any real output.
     case TargetOpcode::IMPLICIT_DEF:
     case TargetOpcode::KILL:
-    case TargetOpcode::PROLOG_LABEL:
+    case TargetOpcode::CFI_INSTRUCTION:
     case TargetOpcode::EH_LABEL:
     case TargetOpcode::GC_LABEL:
     case TargetOpcode::DBG_VALUE:
@@ -721,7 +779,8 @@ public:
   /// is a read of a super-register.
   /// This does not count partial redefines of virtual registers as reads:
   ///   %reg1024:6 = OP.
-  bool readsRegister(unsigned Reg, const TargetRegisterInfo *TRI = NULL) const {
+  bool readsRegister(unsigned Reg,
+                     const TargetRegisterInfo *TRI = nullptr) const {
     return findRegisterUseOperandIdx(Reg, false, TRI) != -1;
   }
 
@@ -737,12 +796,13 @@ public:
   /// partial defines.
   /// If Ops is not null, all operand indices for Reg are added.
   std::pair<bool,bool> readsWritesVirtualRegister(unsigned Reg,
-                                      SmallVectorImpl<unsigned> *Ops = 0) const;
+                                SmallVectorImpl<unsigned> *Ops = nullptr) const;
 
   /// killsRegister - Return true if the MachineInstr kills the specified
   /// register. If TargetRegisterInfo is passed, then it also checks if there is
   /// a kill of a super-register.
-  bool killsRegister(unsigned Reg, const TargetRegisterInfo *TRI = NULL) const {
+  bool killsRegister(unsigned Reg,
+                     const TargetRegisterInfo *TRI = nullptr) const {
     return findRegisterUseOperandIdx(Reg, true, TRI) != -1;
   }
 
@@ -750,7 +810,8 @@ public:
   /// specified register. If TargetRegisterInfo is passed, then it also checks
   /// if there is a def of a super-register.
   /// NOTE: It's ignoring subreg indices on virtual registers.
-  bool definesRegister(unsigned Reg, const TargetRegisterInfo *TRI=NULL) const {
+  bool definesRegister(unsigned Reg,
+                       const TargetRegisterInfo *TRI = nullptr) const {
     return findRegisterDefOperandIdx(Reg, false, false, TRI) != -1;
   }
 
@@ -765,7 +826,7 @@ public:
   /// instruction. If TargetRegisterInfo is passed, then it also checks
   /// if there is a dead def of a super-register.
   bool registerDefIsDead(unsigned Reg,
-                         const TargetRegisterInfo *TRI = NULL) const {
+                         const TargetRegisterInfo *TRI = nullptr) const {
     return findRegisterDefOperandIdx(Reg, true, false, TRI) != -1;
   }
 
@@ -773,14 +834,14 @@ public:
   /// the specific register or -1 if it is not found. It further tightens
   /// the search criteria to a use that kills the register if isKill is true.
   int findRegisterUseOperandIdx(unsigned Reg, bool isKill = false,
-                                const TargetRegisterInfo *TRI = NULL) const;
+                                const TargetRegisterInfo *TRI = nullptr) const;
 
   /// findRegisterUseOperand - Wrapper for findRegisterUseOperandIdx, it returns
   /// a pointer to the MachineOperand rather than an index.
   MachineOperand *findRegisterUseOperand(unsigned Reg, bool isKill = false,
-                                         const TargetRegisterInfo *TRI = NULL) {
+                                      const TargetRegisterInfo *TRI = nullptr) {
     int Idx = findRegisterUseOperandIdx(Reg, isKill, TRI);
-    return (Idx == -1) ? NULL : &getOperand(Idx);
+    return (Idx == -1) ? nullptr : &getOperand(Idx);
   }
 
   /// findRegisterDefOperandIdx() - Returns the operand index that is a def of
@@ -791,14 +852,14 @@ public:
   /// This may also return a register mask operand when Overlap is true.
   int findRegisterDefOperandIdx(unsigned Reg,
                                 bool isDead = false, bool Overlap = false,
-                                const TargetRegisterInfo *TRI = NULL) const;
+                                const TargetRegisterInfo *TRI = nullptr) const;
 
   /// findRegisterDefOperand - Wrapper for findRegisterDefOperandIdx, it returns
   /// a pointer to the MachineOperand rather than an index.
   MachineOperand *findRegisterDefOperand(unsigned Reg, bool isDead = false,
-                                         const TargetRegisterInfo *TRI = NULL) {
+                                      const TargetRegisterInfo *TRI = nullptr) {
     int Idx = findRegisterDefOperandIdx(Reg, isDead, false, TRI);
-    return (Idx == -1) ? NULL : &getOperand(Idx);
+    return (Idx == -1) ? nullptr : &getOperand(Idx);
   }
 
   /// findFirstPredOperandIdx() - Find the index of the first operand in the
@@ -816,7 +877,7 @@ public:
   /// The flag operand is an immediate that can be decoded with methods like
   /// InlineAsm::hasRegClassConstraint().
   ///
-  int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = 0) const;
+  int findInlineAsmFlagIdx(unsigned OpIdx, unsigned *GroupNo = nullptr) const;
 
   /// getRegClassConstraint - Compute the static register class constraint for
   /// operand OpIdx.  For normal instructions, this is derived from the
@@ -830,6 +891,37 @@ public:
                         const TargetInstrInfo *TII,
                         const TargetRegisterInfo *TRI) const;
 
+  /// \brief Applies the constraints (def/use) implied by this MI on \p Reg to
+  /// the given \p CurRC.
+  /// If \p ExploreBundle is set and MI is part of a bundle, all the
+  /// instructions inside the bundle will be taken into account. In other words,
+  /// this method accumulates all the constrains of the operand of this MI and
+  /// the related bundle if MI is a bundle or inside a bundle.
+  ///
+  /// Returns the register class that statisfies both \p CurRC and the
+  /// constraints set by MI. Returns NULL if such a register class does not
+  /// exist.
+  ///
+  /// \pre CurRC must not be NULL.
+  const TargetRegisterClass *getRegClassConstraintEffectForVReg(
+      unsigned Reg, const TargetRegisterClass *CurRC,
+      const TargetInstrInfo *TII, const TargetRegisterInfo *TRI,
+      bool ExploreBundle = false) const;
+
+  /// \brief Applies the constraints (def/use) implied by the \p OpIdx operand
+  /// to the given \p CurRC.
+  ///
+  /// Returns the register class that statisfies both \p CurRC and the
+  /// constraints set by \p OpIdx MI. Returns NULL if such a register class
+  /// does not exist.
+  ///
+  /// \pre CurRC must not be NULL.
+  /// \pre The operand at \p OpIdx must be a register.
+  const TargetRegisterClass *
+  getRegClassConstraintEffect(unsigned OpIdx, const TargetRegisterClass *CurRC,
+                              const TargetInstrInfo *TII,
+                              const TargetRegisterInfo *TRI) const;
+
   /// tieOperands - Add a tie between the register operands at DefIdx and
   /// UseIdx. The tie will cause the register allocator to ensure that the two
   /// operands are assigned the same physical register.
@@ -847,7 +939,8 @@ public:
   /// check if the register def is tied to a source operand, due to either
   /// two-address elimination or inline assembly constraints. Returns the
   /// first tied use operand index by reference if UseOpIdx is not null.
-  bool isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx = 0) const {
+  bool isRegTiedToUseOperand(unsigned DefOpIdx,
+                             unsigned *UseOpIdx = nullptr) const {
     const MachineOperand &MO = getOperand(DefOpIdx);
     if (!MO.isReg() || !MO.isDef() || !MO.isTied())
       return false;
@@ -857,9 +950,10 @@ public:
   }
 
   /// isRegTiedToDefOperand - Return true if the use operand of the specified
-  /// index is tied to an def operand. It also returns the def operand index by
+  /// index is tied to a def operand. It also returns the def operand index by
   /// reference if DefOpIdx is not null.
-  bool isRegTiedToDefOperand(unsigned UseOpIdx, unsigned *DefOpIdx = 0) const {
+  bool isRegTiedToDefOperand(unsigned UseOpIdx,
+                             unsigned *DefOpIdx = nullptr) const {
     const MachineOperand &MO = getOperand(UseOpIdx);
     if (!MO.isReg() || !MO.isUse() || !MO.isTied())
       return false;
@@ -898,7 +992,8 @@ public:
 
   /// addRegisterDefined - We have determined MI defines a register. Make sure
   /// there is an operand defining Reg.
-  void addRegisterDefined(unsigned Reg, const TargetRegisterInfo *RegInfo = 0);
+  void addRegisterDefined(unsigned Reg,
+                          const TargetRegisterInfo *RegInfo = nullptr);
 
   /// setPhysRegsDeadExcept - Mark every physreg used by this instruction as
   /// dead except those in the UsedRegs list.
@@ -952,7 +1047,7 @@ public:
   //
   // Debugging support
   //
-  void print(raw_ostream &OS, const TargetMachine *TM = 0,
+  void print(raw_ostream &OS, const TargetMachine *TM = nullptr,
              bool SkipOpers = false) const;
   void dump() const;
 
@@ -1038,6 +1133,13 @@ private:
   /// hasPropertyInBundle - Slow path for hasProperty when we're dealing with a
   /// bundle.
   bool hasPropertyInBundle(unsigned Mask, QueryType Type) const;
+
+  /// \brief Implements the logic of getRegClassConstraintEffectForVReg for the
+  /// this MI and the given operand index \p OpIdx.
+  /// If the related operand does not constrained Reg, this returns CurRC.
+  const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl(
+      unsigned OpIdx, unsigned Reg, const TargetRegisterClass *CurRC,
+      const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const;
 };
 
 /// MachineInstrExpressionTrait - Special DenseMapInfo traits to compare
@@ -1046,7 +1148,7 @@ private:
 /// useful for CSE, etc.
 struct MachineInstrExpressionTrait : DenseMapInfo<MachineInstr*> {
   static inline MachineInstr *getEmptyKey() {
-    return 0;
+    return nullptr;
   }
 
   static inline MachineInstr *getTombstoneKey() {
diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h
index df01371a47ec..21a482cdbd4c 100644
--- a/include/llvm/CodeGen/MachineInstrBuilder.h
+++ b/include/llvm/CodeGen/MachineInstrBuilder.h
@@ -46,7 +46,7 @@ class MachineInstrBuilder {
   MachineFunction *MF;
   MachineInstr *MI;
 public:
-  MachineInstrBuilder() : MF(0), MI(0) {}
+  MachineInstrBuilder() : MF(nullptr), MI(nullptr) {}
 
   /// Create a MachineInstrBuilder for manipulating an existing instruction.
   /// F must be the machine function  that was used to allocate I.
@@ -172,7 +172,12 @@ public:
     MI->addOperand(*MF, MachineOperand::CreateMetadata(MD));
     return *this;
   }
-  
+
+  const MachineInstrBuilder &addCFIIndex(unsigned CFIIndex) const {
+    MI->addOperand(*MF, MachineOperand::CreateCFIIndex(CFIIndex));
+    return *this;
+  }
+
   const MachineInstrBuilder &addSym(MCSymbol *Sym) const {
     MI->addOperand(*MF, MachineOperand::CreateMCSymbol(Sym));
     return *this;
diff --git a/include/llvm/CodeGen/MachineInstrBundle.h b/include/llvm/CodeGen/MachineInstrBundle.h
index 9519edb3ebae..122022486345 100644
--- a/include/llvm/CodeGen/MachineInstrBundle.h
+++ b/include/llvm/CodeGen/MachineInstrBundle.h
@@ -196,7 +196,7 @@ public:
   ///            each operand referring to Reg.
   /// @returns A filled-in RegInfo struct.
   VirtRegInfo analyzeVirtReg(unsigned Reg,
-                 SmallVectorImpl<std::pair<MachineInstr*, unsigned> > *Ops = 0);
+           SmallVectorImpl<std::pair<MachineInstr*, unsigned> > *Ops = nullptr);
 
   /// analyzePhysReg - Analyze how the current instruction or bundle uses a
   /// physical register.  This function should not be called after operator++(),
diff --git a/include/llvm/CodeGen/MachineLoopInfo.h b/include/llvm/CodeGen/MachineLoopInfo.h
index b058ecb4c279..4fbd46b6fb57 100644
--- a/include/llvm/CodeGen/MachineLoopInfo.h
+++ b/include/llvm/CodeGen/MachineLoopInfo.h
@@ -31,6 +31,7 @@
 #define LLVM_CODEGEN_MACHINELOOPINFO_H
 
 #include "llvm/Analysis/LoopInfo.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 
 namespace llvm {
@@ -119,11 +120,11 @@ public:
 
   /// runOnFunction - Calculate the natural loop information.
   ///
-  virtual bool runOnMachineFunction(MachineFunction &F);
+  bool runOnMachineFunction(MachineFunction &F) override;
 
-  virtual void releaseMemory() { LI.releaseMemory(); }
+  void releaseMemory() override { LI.releaseMemory(); }
 
-  virtual void getAnalysisUsage(AnalysisUsage &AU) const;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
 
   /// removeLoop - This removes the specified top-level loop from this loop info
   /// object.  The loop is not deleted, as it will presumably be inserted into
diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h
index 00a55b57f334..2532c16271f0 100644
--- a/include/llvm/CodeGen/MachineMemOperand.h
+++ b/include/llvm/CodeGen/MachineMemOperand.h
@@ -16,11 +16,13 @@
 #ifndef LLVM_CODEGEN_MACHINEMEMOPERAND_H
 #define LLVM_CODEGEN_MACHINEMEMOPERAND_H
 
+#include "llvm/ADT/PointerUnion.h"
+#include "llvm/CodeGen/PseudoSourceValue.h"
+#include "llvm/IR/Value.h"  // PointerLikeTypeTraits<Value*>
 #include "llvm/Support/DataTypes.h"
 
 namespace llvm {
 
-class Value;
 class FoldingSetNodeID;
 class MDNode;
 class raw_ostream;
@@ -33,17 +35,23 @@ struct MachinePointerInfo {
   /// V - This is the IR pointer value for the access, or it is null if unknown.
   /// If this is null, then the access is to a pointer in the default address
   /// space.
-  const Value *V;
+  PointerUnion<const Value *, const PseudoSourceValue *> V;
 
   /// Offset - This is an offset from the base Value*.
   int64_t Offset;
 
-  explicit MachinePointerInfo(const Value *v = 0, int64_t offset = 0)
+  explicit MachinePointerInfo(const Value *v = nullptr, int64_t offset = 0)
+    : V(v), Offset(offset) {}
+
+  explicit MachinePointerInfo(const PseudoSourceValue *v,
+                              int64_t offset = 0)
     : V(v), Offset(offset) {}
 
   MachinePointerInfo getWithOffset(int64_t O) const {
-    if (V == 0) return MachinePointerInfo(0, 0);
-    return MachinePointerInfo(V, Offset+O);
+    if (V.isNull()) return MachinePointerInfo();
+    if (V.is<const Value*>())
+      return MachinePointerInfo(V.get<const Value*>(), Offset+O);
+    return MachinePointerInfo(V.get<const PseudoSourceValue*>(), Offset+O);
   }
 
   /// getAddrSpace - Return the LLVM IR address space number that this pointer
@@ -109,8 +117,8 @@ public:
   /// MachineMemOperand - Construct an MachineMemOperand object with the
   /// specified PtrInfo, flags, size, and base alignment.
   MachineMemOperand(MachinePointerInfo PtrInfo, unsigned flags, uint64_t s,
-                    unsigned base_alignment, const MDNode *TBAAInfo = 0,
-                    const MDNode *Ranges = 0);
+                    unsigned base_alignment, const MDNode *TBAAInfo = nullptr,
+                    const MDNode *Ranges = nullptr);
 
   const MachinePointerInfo &getPointerInfo() const { return PtrInfo; }
 
@@ -121,7 +129,13 @@ public:
   /// other PseudoSourceValue member functions which return objects which stand
   /// for frame/stack pointer relative references and other special references
   /// which are not representable in the high-level IR.
-  const Value *getValue() const { return PtrInfo.V; }
+  const Value *getValue() const { return PtrInfo.V.dyn_cast<const Value*>(); }
+
+  const PseudoSourceValue *getPseudoValue() const {
+    return PtrInfo.V.dyn_cast<const PseudoSourceValue*>();
+  }
+
+  const void *getOpaqueValue() const { return PtrInfo.V.getOpaqueValue(); }
 
   /// getFlags - Return the raw flags of the source value, \see MemOperandFlags.
   unsigned int getFlags() const { return Flags & ((1 << MOMaxBits) - 1); }
@@ -134,6 +148,8 @@ public:
   /// number.
   int64_t getOffset() const { return PtrInfo.Offset; }
 
+  unsigned getAddrSpace() const { return PtrInfo.getAddrSpace(); }
+
   /// getSize - Return the size in bytes of the memory reference.
   uint64_t getSize() const { return Size; }
 
@@ -175,6 +191,7 @@ public:
   /// should only be used when an object is being relocated and all references
   /// to it are being updated.
   void setValue(const Value *NewSV) { PtrInfo.V = NewSV; }
+  void setValue(const PseudoSourceValue *NewSV) { PtrInfo.V = NewSV; }
   void setOffset(int64_t NewOffset) { PtrInfo.Offset = NewOffset; }
 
   /// Profile - Gather unique data for the object.
diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h
index 460c08c8ca7e..6d8d05684c56 100644
--- a/include/llvm/CodeGen/MachineModuleInfo.h
+++ b/include/llvm/CodeGen/MachineModuleInfo.h
@@ -35,14 +35,14 @@
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Metadata.h"
+#include "llvm/IR/ValueHandle.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MachineLocation.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/DataTypes.h"
-#include "llvm/Support/DebugLoc.h"
 #include "llvm/Support/Dwarf.h"
-#include "llvm/Support/ValueHandle.h"
 
 namespace llvm {
 
@@ -71,7 +71,7 @@ struct LandingPadInfo {
   std::vector<int> TypeIds;              // List of type ids (filters negative)
 
   explicit LandingPadInfo(MachineBasicBlock *MBB)
-    : LandingPadBlock(MBB), LandingPadLabel(0), Personality(0) {}
+    : LandingPadBlock(MBB), LandingPadLabel(nullptr), Personality(nullptr) {}
 };
 
 //===----------------------------------------------------------------------===//
@@ -168,10 +168,13 @@ class MachineModuleInfo : public ImmutablePass {
 public:
   static char ID; // Pass identification, replacement for typeid
 
-  typedef std::pair<unsigned, DebugLoc> UnsignedDebugLocPair;
-  typedef SmallVector<std::pair<TrackingVH<MDNode>, UnsignedDebugLocPair>, 4>
-    VariableDbgInfoMapTy;
-  VariableDbgInfoMapTy VariableDbgInfo;
+  struct VariableDbgInfo {
+    TrackingVH<MDNode> Var;
+    unsigned Slot;
+    DebugLoc Loc;
+  };
+  typedef SmallVector<VariableDbgInfo, 4> VariableDbgInfoMapTy;
+  VariableDbgInfoMapTy VariableDbgInfos;
 
   MachineModuleInfo();  // DUMMY CONSTRUCTOR, DO NOT CALL.
   // Real constructor.
@@ -180,8 +183,8 @@ public:
   ~MachineModuleInfo();
 
   // Initialization and Finalization
-  virtual bool doInitialization(Module &);
-  virtual bool doFinalization(Module &);
+  bool doInitialization(Module &) override;
+  bool doFinalization(Module &) override;
 
   /// EndFunction - Discard function meta information.
   ///
@@ -198,7 +201,7 @@ public:
   ///
   template<typename Ty>
   Ty &getObjFileInfo() {
-    if (ObjFileMMI == 0)
+    if (ObjFileMMI == nullptr)
       ObjFileMMI = new Ty(*this);
     return *static_cast<Ty*>(ObjFileMMI);
   }
@@ -238,8 +241,10 @@ public:
     return FrameInstructions;
   }
 
-  void addFrameInst(const MCCFIInstruction &Inst) {
+  unsigned LLVM_ATTRIBUTE_UNUSED_RESULT
+  addFrameInst(const MCCFIInstruction &Inst) {
     FrameInstructions.push_back(Inst);
+    return FrameInstructions.size() - 1;
   }
 
   /// getCompactUnwindEncoding - Returns the compact unwind encoding for a
@@ -329,7 +334,7 @@ public:
 
   /// TidyLandingPads - Remap landing pad labels and remove any deleted landing
   /// pads.
-  void TidyLandingPads(DenseMap<MCSymbol*, uintptr_t> *LPMap = 0);
+  void TidyLandingPads(DenseMap<MCSymbol*, uintptr_t> *LPMap = nullptr);
 
   /// getLandingPads - Return a reference to the landing pad info for the
   /// current function.
@@ -399,10 +404,11 @@ public:
   /// setVariableDbgInfo - Collect information used to emit debugging
   /// information of a variable.
   void setVariableDbgInfo(MDNode *N, unsigned Slot, DebugLoc Loc) {
-    VariableDbgInfo.push_back(std::make_pair(N, std::make_pair(Slot, Loc)));
+    VariableDbgInfo Info = { N, Slot, Loc };
+    VariableDbgInfos.push_back(std::move(Info));
   }
 
-  VariableDbgInfoMapTy &getVariableDbgInfo() { return VariableDbgInfo; }
+  VariableDbgInfoMapTy &getVariableDbgInfo() { return VariableDbgInfos; }
 
 }; // End class MachineModuleInfo
 
diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h
index 40f3580bfdb4..22969bc80776 100644
--- a/include/llvm/CodeGen/MachineOperand.h
+++ b/include/llvm/CodeGen/MachineOperand.h
@@ -42,28 +42,30 @@ class MCSymbol;
 ///
 class MachineOperand {
 public:
-  enum MachineOperandType {
-    MO_Register,               ///< Register operand.
-    MO_Immediate,              ///< Immediate operand
-    MO_CImmediate,             ///< Immediate >64bit operand
-    MO_FPImmediate,            ///< Floating-point immediate operand
-    MO_MachineBasicBlock,      ///< MachineBasicBlock reference
-    MO_FrameIndex,             ///< Abstract Stack Frame Index
-    MO_ConstantPoolIndex,      ///< Address of indexed Constant in Constant Pool
-    MO_TargetIndex,            ///< Target-dependent index+offset operand.
-    MO_JumpTableIndex,         ///< Address of indexed Jump Table for switch
-    MO_ExternalSymbol,         ///< Name of external global symbol
-    MO_GlobalAddress,          ///< Address of a global value
-    MO_BlockAddress,           ///< Address of a basic block
-    MO_RegisterMask,           ///< Mask of preserved registers.
-    MO_Metadata,               ///< Metadata reference (for debug info)
-    MO_MCSymbol                ///< MCSymbol reference (for debug/eh info)
+  enum MachineOperandType : unsigned char {
+    MO_Register,          ///< Register operand.
+    MO_Immediate,         ///< Immediate operand
+    MO_CImmediate,        ///< Immediate >64bit operand
+    MO_FPImmediate,       ///< Floating-point immediate operand
+    MO_MachineBasicBlock, ///< MachineBasicBlock reference
+    MO_FrameIndex,        ///< Abstract Stack Frame Index
+    MO_ConstantPoolIndex, ///< Address of indexed Constant in Constant Pool
+    MO_TargetIndex,       ///< Target-dependent index+offset operand.
+    MO_JumpTableIndex,    ///< Address of indexed Jump Table for switch
+    MO_ExternalSymbol,    ///< Name of external global symbol
+    MO_GlobalAddress,     ///< Address of a global value
+    MO_BlockAddress,      ///< Address of a basic block
+    MO_RegisterMask,      ///< Mask of preserved registers.
+    MO_RegisterLiveOut,   ///< Mask of live-out registers.
+    MO_Metadata,          ///< Metadata reference (for debug info)
+    MO_MCSymbol,          ///< MCSymbol reference (for debug/eh info)
+    MO_CFIIndex           ///< MCCFIInstruction index.
   };
 
 private:
   /// OpKind - Specify what kind of operand this is.  This discriminates the
   /// union.
-  unsigned char OpKind; // MachineOperandType
+  MachineOperandType OpKind;
 
   /// Subregister number for MO_Register.  A value of 0 indicates the
   /// MO_Register has no subReg.
@@ -149,13 +151,14 @@ private:
 
   /// Contents union - This contains the payload for the various operand types.
   union {
-    MachineBasicBlock *MBB;   // For MO_MachineBasicBlock.
-    const ConstantFP *CFP;    // For MO_FPImmediate.
-    const ConstantInt *CI;    // For MO_CImmediate. Integers > 64bit.
-    int64_t ImmVal;           // For MO_Immediate.
-    const uint32_t *RegMask;  // For MO_RegisterMask.
-    const MDNode *MD;         // For MO_Metadata.
-    MCSymbol *Sym;            // For MO_MCSymbol
+    MachineBasicBlock *MBB;  // For MO_MachineBasicBlock.
+    const ConstantFP *CFP;   // For MO_FPImmediate.
+    const ConstantInt *CI;   // For MO_CImmediate. Integers > 64bit.
+    int64_t ImmVal;          // For MO_Immediate.
+    const uint32_t *RegMask; // For MO_RegisterMask and MO_RegisterLiveOut.
+    const MDNode *MD;        // For MO_Metadata.
+    MCSymbol *Sym;           // For MO_MCSymbol.
+    unsigned CFIIndex;       // For MO_CFI.
 
     struct {                  // For MO_Register.
       // Register number is in SmallContents.RegNo.
@@ -178,7 +181,7 @@ private:
   } Contents;
 
   explicit MachineOperand(MachineOperandType K)
-    : OpKind(K), SubReg_TargetFlags(0), ParentMI(0) {}
+    : OpKind(K), SubReg_TargetFlags(0), ParentMI(nullptr) {}
 public:
   /// getType - Returns the MachineOperandType for this operand.
   ///
@@ -212,9 +215,9 @@ public:
   ///
   /// Never call clearParent() on an operand in a MachineInstr.
   ///
-  void clearParent() { ParentMI = 0; }
+  void clearParent() { ParentMI = nullptr; }
 
-  void print(raw_ostream &os, const TargetMachine *TM = 0) const;
+  void print(raw_ostream &os, const TargetMachine *TM = nullptr) const;
 
   //===--------------------------------------------------------------------===//
   // Accessors that tell you what kind of MachineOperand you're looking at.
@@ -224,7 +227,7 @@ public:
   bool isReg() const { return OpKind == MO_Register; }
   /// isImm - Tests if this is a MO_Immediate operand.
   bool isImm() const { return OpKind == MO_Immediate; }
-  /// isCImm - Test if t his is a MO_CImmediate operand.
+  /// isCImm - Test if this is a MO_CImmediate operand.
   bool isCImm() const { return OpKind == MO_CImmediate; }
   /// isFPImm - Tests if this is a MO_FPImmediate operand.
   bool isFPImm() const { return OpKind == MO_FPImmediate; }
@@ -246,10 +249,12 @@ public:
   bool isBlockAddress() const { return OpKind == MO_BlockAddress; }
   /// isRegMask - Tests if this is a MO_RegisterMask operand.
   bool isRegMask() const { return OpKind == MO_RegisterMask; }
+  /// isRegLiveOut - Tests if this is a MO_RegisterLiveOut operand.
+  bool isRegLiveOut() const { return OpKind == MO_RegisterLiveOut; }
   /// isMetadata - Tests if this is a MO_Metadata operand.
   bool isMetadata() const { return OpKind == MO_Metadata; }
   bool isMCSymbol() const { return OpKind == MO_MCSymbol; }
-
+  bool isCFIIndex() const { return OpKind == MO_CFIIndex; }
 
   //===--------------------------------------------------------------------===//
   // Accessors for Register Operands
@@ -440,6 +445,11 @@ public:
     return Contents.Sym;
   }
 
+  unsigned getCFIIndex() const {
+    assert(isCFIIndex() && "Wrong MachineOperand accessor");
+    return Contents.CFIIndex;
+  }
+
   /// getOffset - Return the offset from the symbol in this operand. This always
   /// returns 0 for ExternalSymbol operands.
   int64_t getOffset() const {
@@ -476,6 +486,12 @@ public:
     return Contents.RegMask;
   }
 
+  /// getRegLiveOut - Returns a bit mask of live-out registers.
+  const uint32_t *getRegLiveOut() const {
+    assert(isRegLiveOut() && "Wrong MachineOperand accessor");
+    return Contents.RegMask;
+  }
+
   const MDNode *getMetadata() const {
     assert(isMetadata() && "Wrong MachineOperand accessor");
     return Contents.MD;
@@ -577,8 +593,8 @@ public:
     Op.TiedTo = 0;
     Op.IsDebug = isDebug;
     Op.SmallContents.RegNo = Reg;
-    Op.Contents.Reg.Prev = 0;
-    Op.Contents.Reg.Next = 0;
+    Op.Contents.Reg.Prev = nullptr;
+    Op.Contents.Reg.Next = nullptr;
     Op.setSubReg(SubReg);
     return Op;
   }
@@ -659,6 +675,12 @@ public:
     Op.Contents.RegMask = Mask;
     return Op;
   }
+  static MachineOperand CreateRegLiveOut(const uint32_t *Mask) {
+    assert(Mask && "Missing live-out register mask");
+    MachineOperand Op(MachineOperand::MO_RegisterLiveOut);
+    Op.Contents.RegMask = Mask;
+    return Op;
+  }
   static MachineOperand CreateMetadata(const MDNode *Meta) {
     MachineOperand Op(MachineOperand::MO_Metadata);
     Op.Contents.MD = Meta;
@@ -671,6 +693,12 @@ public:
     return Op;
   }
 
+  static MachineOperand CreateCFIIndex(unsigned CFIIndex) {
+    MachineOperand Op(MachineOperand::MO_CFIIndex);
+    Op.Contents.CFIIndex = CFIIndex;
+    return Op;
+  }
+
   friend class MachineInstr;
   friend class MachineRegisterInfo;
 private:
@@ -683,12 +711,12 @@ private:
   /// part of a machine instruction.
   bool isOnRegUseList() const {
     assert(isReg() && "Can only add reg operand to use lists");
-    return Contents.Reg.Prev != 0;
+    return Contents.Reg.Prev != nullptr;
   }
 };
 
 inline raw_ostream &operator<<(raw_ostream &OS, const MachineOperand& MO) {
-  MO.print(OS, 0);
+  MO.print(OS, nullptr);
   return OS;
 }
 
diff --git a/include/llvm/CodeGen/MachinePassRegistry.h b/include/llvm/CodeGen/MachinePassRegistry.h
index 90ee7f4bb9e7..c962e687243e 100644
--- a/include/llvm/CodeGen/MachinePassRegistry.h
+++ b/include/llvm/CodeGen/MachinePassRegistry.h
@@ -59,7 +59,7 @@ private:
 public:
 
   MachinePassRegistryNode(const char *N, const char *D, MachinePassCtor C)
-  : Next(NULL)
+  : Next(nullptr)
   , Name(N)
   , Description(D)
   , Ctor(C)
@@ -123,7 +123,7 @@ class RegisterPassParser : public MachinePassRegistryListener,
                    public cl::parser<typename RegistryClass::FunctionPassCtor> {
 public:
   RegisterPassParser() {}
-  ~RegisterPassParser() { RegistryClass::setListener(NULL); }
+  ~RegisterPassParser() { RegistryClass::setListener(nullptr); }
 
   void initialize(cl::Option &O) {
     cl::parser<typename RegistryClass::FunctionPassCtor>::initialize(O);
@@ -142,12 +142,10 @@ public:
 
   // Implement the MachinePassRegistryListener callbacks.
   //
-  virtual void NotifyAdd(const char *N,
-                         MachinePassCtor C,
-                         const char *D) {
+  void NotifyAdd(const char *N, MachinePassCtor C, const char *D) override {
     this->addLiteralOption(N, (typename RegistryClass::FunctionPassCtor)C, D);
   }
-  virtual void NotifyRemove(const char *N) {
+  void NotifyRemove(const char *N) override {
     this->removeLiteralOption(N);
   }
 };
diff --git a/include/llvm/CodeGen/MachinePostDominators.h b/include/llvm/CodeGen/MachinePostDominators.h
index ca09aef82616..beb2c4f0c5c0 100644
--- a/include/llvm/CodeGen/MachinePostDominators.h
+++ b/include/llvm/CodeGen/MachinePostDominators.h
@@ -15,7 +15,6 @@
 #ifndef LLVM_CODEGEN_MACHINEPOSTDOMINATORS_H
 #define LLVM_CODEGEN_MACHINEPOSTDOMINATORS_H
 
-#include "llvm/Analysis/Dominators.h"
 #include "llvm/CodeGen/MachineDominators.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 
@@ -78,9 +77,9 @@ public:
     return DT->findNearestCommonDominator(A, B);
   }
 
-  virtual bool runOnMachineFunction(MachineFunction &MF);
-  virtual void getAnalysisUsage(AnalysisUsage &AU) const;
-  virtual void print(llvm::raw_ostream &OS, const Module *M = 0) const;
+  bool runOnMachineFunction(MachineFunction &MF) override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  void print(llvm::raw_ostream &OS, const Module *M = nullptr) const override;
 };
 } //end of namespace llvm
 
diff --git a/include/llvm/CodeGen/MachineRegionInfo.h b/include/llvm/CodeGen/MachineRegionInfo.h
new file mode 100644
index 000000000000..43499dba71c0
--- /dev/null
+++ b/include/llvm/CodeGen/MachineRegionInfo.h
@@ -0,0 +1,183 @@
+//===- llvm/CodeGen/MachineRegionInfo.h -------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINEREGIONINFO_H
+#define LLVM_CODEGEN_MACHINEREGIONINFO_H
+
+#include "llvm/Analysis/RegionInfo.h"
+#include "llvm/Analysis/RegionIterator.h"
+#include "llvm/CodeGen/MachineDominanceFrontier.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+
+
+namespace llvm {
+
+class MachineDominatorTree;
+struct MachinePostDominatorTree;
+class MachineRegion;
+class MachineRegionNode;
+class MachineRegionInfo;
+
+template<>
+struct RegionTraits<MachineFunction> {
+  typedef MachineFunction FuncT;
+  typedef MachineBasicBlock BlockT;
+  typedef MachineRegion RegionT;
+  typedef MachineRegionNode RegionNodeT;
+  typedef MachineRegionInfo RegionInfoT;
+  typedef MachineDominatorTree DomTreeT;
+  typedef MachineDomTreeNode DomTreeNodeT;
+  typedef MachinePostDominatorTree PostDomTreeT;
+  typedef MachineDominanceFrontier DomFrontierT;
+  typedef MachineInstr InstT;
+  typedef MachineLoop LoopT;
+  typedef MachineLoopInfo LoopInfoT;
+
+  static unsigned getNumSuccessors(MachineBasicBlock *BB) {
+    return BB->succ_size();
+  }
+};
+
+
+class MachineRegionNode : public RegionNodeBase<RegionTraits<MachineFunction>> {
+public:
+  inline MachineRegionNode(MachineRegion *Parent,
+                           MachineBasicBlock *Entry,
+                           bool isSubRegion = false)
+    : RegionNodeBase<RegionTraits<MachineFunction>>(Parent, Entry, isSubRegion) {
+
+  }
+
+  ~MachineRegionNode() { }
+
+  bool operator==(const MachineRegion &RN) const {
+    return this == reinterpret_cast<const MachineRegionNode*>(&RN);
+  }
+};
+
+class MachineRegion : public RegionBase<RegionTraits<MachineFunction>> {
+public:
+  MachineRegion(MachineBasicBlock *Entry, MachineBasicBlock *Exit,
+                MachineRegionInfo* RI,
+                MachineDominatorTree *DT, MachineRegion *Parent = nullptr);
+  ~MachineRegion();
+
+  bool operator==(const MachineRegionNode &RN) const {
+    return &RN == reinterpret_cast<const MachineRegionNode*>(this);
+  }
+};
+
+class MachineRegionInfo : public RegionInfoBase<RegionTraits<MachineFunction>> {
+public:
+  explicit MachineRegionInfo();
+
+  virtual ~MachineRegionInfo();
+
+  // updateStatistics - Update statistic about created regions.
+  void updateStatistics(MachineRegion *R) final;
+
+  void recalculate(MachineFunction &F,
+                   MachineDominatorTree *DT,
+                   MachinePostDominatorTree *PDT,
+                   MachineDominanceFrontier *DF);
+};
+
+class MachineRegionInfoPass : public MachineFunctionPass {
+  MachineRegionInfo RI;
+
+public:
+  static char ID;
+  explicit MachineRegionInfoPass();
+
+  ~MachineRegionInfoPass();
+
+  MachineRegionInfo &getRegionInfo() {
+    return RI;
+  }
+
+  const MachineRegionInfo &getRegionInfo() const {
+    return RI;
+  }
+
+  /// @name MachineFunctionPass interface
+  //@{
+  bool runOnMachineFunction(MachineFunction &F) override;
+  void releaseMemory() override;
+  void verifyAnalysis() const override;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+  void print(raw_ostream &OS, const Module *) const override;
+  void dump() const;
+  //@}
+};
+
+
+template <>
+template <>
+inline MachineBasicBlock* RegionNodeBase<RegionTraits<MachineFunction>>::getNodeAs<MachineBasicBlock>() const {
+  assert(!isSubRegion() && "This is not a MachineBasicBlock RegionNode!");
+  return getEntry();
+}
+
+template<>
+template<>
+inline MachineRegion* RegionNodeBase<RegionTraits<MachineFunction>>::getNodeAs<MachineRegion>() const {
+  assert(isSubRegion() && "This is not a subregion RegionNode!");
+  auto Unconst = const_cast<RegionNodeBase<RegionTraits<MachineFunction>>*>(this);
+  return reinterpret_cast<MachineRegion*>(Unconst);
+}
+
+
+RegionNodeGraphTraits(MachineRegionNode, MachineBasicBlock, MachineRegion);
+RegionNodeGraphTraits(const MachineRegionNode, MachineBasicBlock, MachineRegion);
+
+RegionGraphTraits(MachineRegion, MachineRegionNode);
+RegionGraphTraits(const MachineRegion, const MachineRegionNode);
+
+template <> struct GraphTraits<MachineRegionInfo*>
+  : public GraphTraits<FlatIt<MachineRegionNode*> > {
+  typedef df_iterator<NodeType*, SmallPtrSet<NodeType*, 8>, false,
+                      GraphTraits<FlatIt<NodeType*> > > nodes_iterator;
+
+  static NodeType *getEntryNode(MachineRegionInfo *RI) {
+    return GraphTraits<FlatIt<MachineRegion*> >::getEntryNode(RI->getTopLevelRegion());
+  }
+  static nodes_iterator nodes_begin(MachineRegionInfo* RI) {
+    return nodes_iterator::begin(getEntryNode(RI));
+  }
+  static nodes_iterator nodes_end(MachineRegionInfo *RI) {
+    return nodes_iterator::end(getEntryNode(RI));
+  }
+};
+
+template <> struct GraphTraits<MachineRegionInfoPass*>
+  : public GraphTraits<MachineRegionInfo *> {
+  typedef df_iterator<NodeType*, SmallPtrSet<NodeType*, 8>, false,
+                      GraphTraits<FlatIt<NodeType*> > > nodes_iterator;
+
+  static NodeType *getEntryNode(MachineRegionInfoPass *RI) {
+    return GraphTraits<MachineRegionInfo*>::getEntryNode(&RI->getRegionInfo());
+  }
+  static nodes_iterator nodes_begin(MachineRegionInfoPass* RI) {
+    return GraphTraits<MachineRegionInfo*>::nodes_begin(&RI->getRegionInfo());
+  }
+  static nodes_iterator nodes_end(MachineRegionInfoPass *RI) {
+    return GraphTraits<MachineRegionInfo*>::nodes_end(&RI->getRegionInfo());
+  }
+};
+
+EXTERN_TEMPLATE_INSTANTIATION(class RegionBase<RegionTraits<MachineFunction>>);
+EXTERN_TEMPLATE_INSTANTIATION(class RegionNodeBase<RegionTraits<MachineFunction>>);
+EXTERN_TEMPLATE_INSTANTIATION(class RegionInfoBase<RegionTraits<MachineFunction>>);
+
+}
+
+#endif
diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h
index 58ca907316ac..51139f72ba22 100644
--- a/include/llvm/CodeGen/MachineRegisterInfo.h
+++ b/include/llvm/CodeGen/MachineRegisterInfo.h
@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/IndexedMap.h"
+#include "llvm/ADT/iterator_range.h"
 #include "llvm/CodeGen/MachineInstrBundle.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetRegisterInfo.h"
@@ -134,7 +135,7 @@ public:
     // notifications, we will need to change to using a list.
     assert(TheDelegate == delegate &&
            "Only the current delegate can perform reset!");
-    TheDelegate = 0;
+    TheDelegate = nullptr;
   }
 
   void setDelegate(Delegate *delegate) {
@@ -200,19 +201,66 @@ public:
   /// reg_begin/reg_end - Provide iteration support to walk over all definitions
   /// and uses of a register within the MachineFunction that corresponds to this
   /// MachineRegisterInfo object.
-  template<bool Uses, bool Defs, bool SkipDebug>
+  template<bool Uses, bool Defs, bool SkipDebug,
+           bool ByOperand, bool ByInstr, bool ByBundle>
   class defusechain_iterator;
+  template<bool Uses, bool Defs, bool SkipDebug,
+           bool ByOperand, bool ByInstr, bool ByBundle>
+  class defusechain_instr_iterator;
 
   // Make it a friend so it can access getNextOperandForReg().
-  template<bool, bool, bool> friend class defusechain_iterator;
+  template<bool, bool, bool, bool, bool, bool>
+    friend class defusechain_iterator;
+  template<bool, bool, bool, bool, bool, bool>
+    friend class defusechain_instr_iterator;
+
+
 
   /// reg_iterator/reg_begin/reg_end - Walk all defs and uses of the specified
   /// register.
-  typedef defusechain_iterator<true,true,false> reg_iterator;
+  typedef defusechain_iterator<true,true,false,true,false,false>
+          reg_iterator;
   reg_iterator reg_begin(unsigned RegNo) const {
     return reg_iterator(getRegUseDefListHead(RegNo));
   }
-  static reg_iterator reg_end() { return reg_iterator(0); }
+  static reg_iterator reg_end() { return reg_iterator(nullptr); }
+
+  inline iterator_range<reg_iterator>  reg_operands(unsigned Reg) const {
+    return iterator_range<reg_iterator>(reg_begin(Reg), reg_end());
+  }
+
+  /// reg_instr_iterator/reg_instr_begin/reg_instr_end - Walk all defs and uses
+  /// of the specified register, stepping by MachineInstr.
+  typedef defusechain_instr_iterator<true,true,false,false,true,false>
+          reg_instr_iterator;
+  reg_instr_iterator reg_instr_begin(unsigned RegNo) const {
+    return reg_instr_iterator(getRegUseDefListHead(RegNo));
+  }
+  static reg_instr_iterator reg_instr_end() {
+    return reg_instr_iterator(nullptr);
+  }
+
+  inline iterator_range<reg_instr_iterator>
+  reg_instructions(unsigned Reg) const {
+    return iterator_range<reg_instr_iterator>(reg_instr_begin(Reg),
+                                              reg_instr_end());
+  }
+
+  /// reg_bundle_iterator/reg_bundle_begin/reg_bundle_end - Walk all defs and uses
+  /// of the specified register, stepping by bundle.
+  typedef defusechain_instr_iterator<true,true,false,false,false,true>
+          reg_bundle_iterator;
+  reg_bundle_iterator reg_bundle_begin(unsigned RegNo) const {
+    return reg_bundle_iterator(getRegUseDefListHead(RegNo));
+  }
+  static reg_bundle_iterator reg_bundle_end() {
+    return reg_bundle_iterator(nullptr);
+  }
+
+  inline iterator_range<reg_bundle_iterator> reg_bundles(unsigned Reg) const {
+    return iterator_range<reg_bundle_iterator>(reg_bundle_begin(Reg),
+                                               reg_bundle_end());
+  }
 
   /// reg_empty - Return true if there are no instructions using or defining the
   /// specified register (it may be live-in).
@@ -220,11 +268,56 @@ public:
 
   /// reg_nodbg_iterator/reg_nodbg_begin/reg_nodbg_end - Walk all defs and uses
   /// of the specified register, skipping those marked as Debug.
-  typedef defusechain_iterator<true,true,true> reg_nodbg_iterator;
+  typedef defusechain_iterator<true,true,true,true,false,false>
+          reg_nodbg_iterator;
   reg_nodbg_iterator reg_nodbg_begin(unsigned RegNo) const {
     return reg_nodbg_iterator(getRegUseDefListHead(RegNo));
   }
-  static reg_nodbg_iterator reg_nodbg_end() { return reg_nodbg_iterator(0); }
+  static reg_nodbg_iterator reg_nodbg_end() {
+    return reg_nodbg_iterator(nullptr);
+  }
+
+  inline iterator_range<reg_nodbg_iterator>
+  reg_nodbg_operands(unsigned Reg) const {
+    return iterator_range<reg_nodbg_iterator>(reg_nodbg_begin(Reg),
+                                              reg_nodbg_end());
+  }
+
+  /// reg_instr_nodbg_iterator/reg_instr_nodbg_begin/reg_instr_nodbg_end - Walk
+  /// all defs and uses of the specified register, stepping by MachineInstr,
+  /// skipping those marked as Debug.
+  typedef defusechain_instr_iterator<true,true,true,false,true,false>
+          reg_instr_nodbg_iterator;
+  reg_instr_nodbg_iterator reg_instr_nodbg_begin(unsigned RegNo) const {
+    return reg_instr_nodbg_iterator(getRegUseDefListHead(RegNo));
+  }
+  static reg_instr_nodbg_iterator reg_instr_nodbg_end() {
+    return reg_instr_nodbg_iterator(nullptr);
+  }
+
+  inline iterator_range<reg_instr_nodbg_iterator>
+  reg_nodbg_instructions(unsigned Reg) const {
+    return iterator_range<reg_instr_nodbg_iterator>(reg_instr_nodbg_begin(Reg),
+                                                    reg_instr_nodbg_end());
+  }
+
+  /// reg_bundle_nodbg_iterator/reg_bundle_nodbg_begin/reg_bundle_nodbg_end - Walk
+  /// all defs and uses of the specified register, stepping by bundle,
+  /// skipping those marked as Debug.
+  typedef defusechain_instr_iterator<true,true,true,false,false,true>
+          reg_bundle_nodbg_iterator;
+  reg_bundle_nodbg_iterator reg_bundle_nodbg_begin(unsigned RegNo) const {
+    return reg_bundle_nodbg_iterator(getRegUseDefListHead(RegNo));
+  }
+  static reg_bundle_nodbg_iterator reg_bundle_nodbg_end() {
+    return reg_bundle_nodbg_iterator(nullptr);
+  }
+
+  inline iterator_range<reg_bundle_nodbg_iterator> 
+  reg_nodbg_bundles(unsigned Reg) const {
+    return iterator_range<reg_bundle_nodbg_iterator>(reg_bundle_nodbg_begin(Reg),
+                                                     reg_bundle_nodbg_end());
+  }
 
   /// reg_nodbg_empty - Return true if the only instructions using or defining
   /// Reg are Debug instructions.
@@ -233,11 +326,49 @@ public:
   }
 
   /// def_iterator/def_begin/def_end - Walk all defs of the specified register.
-  typedef defusechain_iterator<false,true,false> def_iterator;
+  typedef defusechain_iterator<false,true,false,true,false,false>
+          def_iterator;
   def_iterator def_begin(unsigned RegNo) const {
     return def_iterator(getRegUseDefListHead(RegNo));
   }
-  static def_iterator def_end() { return def_iterator(0); }
+  static def_iterator def_end() { return def_iterator(nullptr); }
+
+  inline iterator_range<def_iterator> def_operands(unsigned Reg) const {
+    return iterator_range<def_iterator>(def_begin(Reg), def_end());
+  }
+
+  /// def_instr_iterator/def_instr_begin/def_instr_end - Walk all defs of the
+  /// specified register, stepping by MachineInst.
+  typedef defusechain_instr_iterator<false,true,false,false,true,false>
+          def_instr_iterator;
+  def_instr_iterator def_instr_begin(unsigned RegNo) const {
+    return def_instr_iterator(getRegUseDefListHead(RegNo));
+  }
+  static def_instr_iterator def_instr_end() {
+    return def_instr_iterator(nullptr);
+  }
+
+  inline iterator_range<def_instr_iterator>
+  def_instructions(unsigned Reg) const {
+    return iterator_range<def_instr_iterator>(def_instr_begin(Reg),
+                                              def_instr_end());
+  }
+
+  /// def_bundle_iterator/def_bundle_begin/def_bundle_end - Walk all defs of the
+  /// specified register, stepping by bundle.
+  typedef defusechain_instr_iterator<false,true,false,false,false,true>
+          def_bundle_iterator;
+  def_bundle_iterator def_bundle_begin(unsigned RegNo) const {
+    return def_bundle_iterator(getRegUseDefListHead(RegNo));
+  }
+  static def_bundle_iterator def_bundle_end() {
+    return def_bundle_iterator(nullptr);
+  }
+
+  inline iterator_range<def_bundle_iterator> def_bundles(unsigned Reg) const {
+    return iterator_range<def_bundle_iterator>(def_bundle_begin(Reg),
+                                               def_bundle_end());
+  }
 
   /// def_empty - Return true if there are no instructions defining the
   /// specified register (it may be live-in).
@@ -253,11 +384,49 @@ public:
   }
 
   /// use_iterator/use_begin/use_end - Walk all uses of the specified register.
-  typedef defusechain_iterator<true,false,false> use_iterator;
+  typedef defusechain_iterator<true,false,false,true,false,false>
+          use_iterator;
   use_iterator use_begin(unsigned RegNo) const {
     return use_iterator(getRegUseDefListHead(RegNo));
   }
-  static use_iterator use_end() { return use_iterator(0); }
+  static use_iterator use_end() { return use_iterator(nullptr); }
+
+  inline iterator_range<use_iterator> use_operands(unsigned Reg) const {
+    return iterator_range<use_iterator>(use_begin(Reg), use_end());
+  }
+
+  /// use_instr_iterator/use_instr_begin/use_instr_end - Walk all uses of the
+  /// specified register, stepping by MachineInstr.
+  typedef defusechain_instr_iterator<true,false,false,false,true,false>
+          use_instr_iterator;
+  use_instr_iterator use_instr_begin(unsigned RegNo) const {
+    return use_instr_iterator(getRegUseDefListHead(RegNo));
+  }
+  static use_instr_iterator use_instr_end() {
+    return use_instr_iterator(nullptr);
+  }
+
+  inline iterator_range<use_instr_iterator>
+  use_instructions(unsigned Reg) const {
+    return iterator_range<use_instr_iterator>(use_instr_begin(Reg),
+                                              use_instr_end());
+  }
+
+  /// use_bundle_iterator/use_bundle_begin/use_bundle_end - Walk all uses of the
+  /// specified register, stepping by bundle.
+  typedef defusechain_instr_iterator<true,false,false,false,false,true>
+          use_bundle_iterator;
+  use_bundle_iterator use_bundle_begin(unsigned RegNo) const {
+    return use_bundle_iterator(getRegUseDefListHead(RegNo));
+  }
+  static use_bundle_iterator use_bundle_end() {
+    return use_bundle_iterator(nullptr);
+  }
+
+  inline iterator_range<use_bundle_iterator> use_bundles(unsigned Reg) const {
+    return iterator_range<use_bundle_iterator>(use_bundle_begin(Reg),
+                                               use_bundle_end());
+  }
 
   /// use_empty - Return true if there are no instructions using the specified
   /// register.
@@ -274,11 +443,56 @@ public:
 
   /// use_nodbg_iterator/use_nodbg_begin/use_nodbg_end - Walk all uses of the
   /// specified register, skipping those marked as Debug.
-  typedef defusechain_iterator<true,false,true> use_nodbg_iterator;
+  typedef defusechain_iterator<true,false,true,true,false,false>
+          use_nodbg_iterator;
   use_nodbg_iterator use_nodbg_begin(unsigned RegNo) const {
     return use_nodbg_iterator(getRegUseDefListHead(RegNo));
   }
-  static use_nodbg_iterator use_nodbg_end() { return use_nodbg_iterator(0); }
+  static use_nodbg_iterator use_nodbg_end() {
+    return use_nodbg_iterator(nullptr);
+  }
+
+  inline iterator_range<use_nodbg_iterator>
+  use_nodbg_operands(unsigned Reg) const {
+    return iterator_range<use_nodbg_iterator>(use_nodbg_begin(Reg),
+                                              use_nodbg_end());
+  }
+
+  /// use_instr_nodbg_iterator/use_instr_nodbg_begin/use_instr_nodbg_end - Walk
+  /// all uses of the specified register, stepping by MachineInstr, skipping
+  /// those marked as Debug.
+  typedef defusechain_instr_iterator<true,false,true,false,true,false>
+          use_instr_nodbg_iterator;
+  use_instr_nodbg_iterator use_instr_nodbg_begin(unsigned RegNo) const {
+    return use_instr_nodbg_iterator(getRegUseDefListHead(RegNo));
+  }
+  static use_instr_nodbg_iterator use_instr_nodbg_end() {
+    return use_instr_nodbg_iterator(nullptr);
+  }
+
+  inline iterator_range<use_instr_nodbg_iterator>
+  use_nodbg_instructions(unsigned Reg) const {
+    return iterator_range<use_instr_nodbg_iterator>(use_instr_nodbg_begin(Reg),
+                                                    use_instr_nodbg_end());
+  }
+
+  /// use_bundle_nodbg_iterator/use_bundle_nodbg_begin/use_bundle_nodbg_end - Walk
+  /// all uses of the specified register, stepping by bundle, skipping
+  /// those marked as Debug.
+  typedef defusechain_instr_iterator<true,false,true,false,false,true>
+          use_bundle_nodbg_iterator;
+  use_bundle_nodbg_iterator use_bundle_nodbg_begin(unsigned RegNo) const {
+    return use_bundle_nodbg_iterator(getRegUseDefListHead(RegNo));
+  }
+  static use_bundle_nodbg_iterator use_bundle_nodbg_end() {
+    return use_bundle_nodbg_iterator(nullptr);
+  }
+
+  inline iterator_range<use_bundle_nodbg_iterator>
+  use_nodbg_bundles(unsigned Reg) const {
+    return iterator_range<use_bundle_nodbg_iterator>(use_bundle_nodbg_begin(Reg),
+                                                     use_bundle_nodbg_end());
+  }
 
   /// use_nodbg_empty - Return true if there are no non-Debug instructions
   /// using the specified register.
@@ -401,6 +615,10 @@ public:
     return Hint.first ? 0 : Hint.second;
   }
 
+  /// markUsesInDebugValueAsUndef - Mark every DBG_VALUE referencing the
+  /// specified register as undefined which causes the DBG_VALUE to be
+  /// deleted during LiveDebugVariables analysis.
+  void markUsesInDebugValueAsUndef(unsigned Reg) const;
 
   //===--------------------------------------------------------------------===//
   // Physical Register Use Info
@@ -552,7 +770,8 @@ public:
   /// returns defs.  If neither are true then you are silly and it always
   /// returns end().  If SkipDebug is true it skips uses marked Debug
   /// when incrementing.
-  template<bool ReturnUses, bool ReturnDefs, bool SkipDebug>
+  template<bool ReturnUses, bool ReturnDefs, bool SkipDebug,
+           bool ByOperand, bool ByInstr, bool ByBundle>
   class defusechain_iterator
     : public std::iterator<std::forward_iterator_tag, MachineInstr, ptrdiff_t> {
     MachineOperand *Op;
@@ -563,10 +782,30 @@ public:
         if ((!ReturnUses && op->isUse()) ||
             (!ReturnDefs && op->isDef()) ||
             (SkipDebug && op->isDebug()))
-          ++*this;
+          advance();
       }
     }
     friend class MachineRegisterInfo;
+
+    void advance() {
+      assert(Op && "Cannot increment end iterator!");
+      Op = getNextOperandForReg(Op);
+
+      // All defs come before the uses, so stop def_iterator early.
+      if (!ReturnUses) {
+        if (Op) {
+          if (Op->isUse())
+            Op = nullptr;
+          else
+            assert(!Op->isDebug() && "Can't have debug defs");
+        }
+      } else {
+        // If this is an operand we don't care about, skip it.
+        while (Op && ((!ReturnDefs && Op->isDef()) ||
+                      (SkipDebug && Op->isDebug())))
+          Op = getNextOperandForReg(Op);
+      }
+    }
   public:
     typedef std::iterator<std::forward_iterator_tag,
                           MachineInstr, ptrdiff_t>::reference reference;
@@ -574,7 +813,7 @@ public:
                           MachineInstr, ptrdiff_t>::pointer pointer;
 
     defusechain_iterator(const defusechain_iterator &I) : Op(I.Op) {}
-    defusechain_iterator() : Op(0) {}
+    defusechain_iterator() : Op(nullptr) {}
 
     bool operator==(const defusechain_iterator &x) const {
       return Op == x.Op;
@@ -584,18 +823,82 @@ public:
     }
 
     /// atEnd - return true if this iterator is equal to reg_end() on the value.
-    bool atEnd() const { return Op == 0; }
+    bool atEnd() const { return Op == nullptr; }
 
     // Iterator traversal: forward iteration only
     defusechain_iterator &operator++() {          // Preincrement
       assert(Op && "Cannot increment end iterator!");
+      if (ByOperand)
+        advance();
+      else if (ByInstr) {
+        MachineInstr *P = Op->getParent();
+        do {
+          advance();
+        } while (Op && Op->getParent() == P);
+      } else if (ByBundle) {
+        MachineInstr *P = getBundleStart(Op->getParent());
+        do {
+          advance();
+        } while (Op && getBundleStart(Op->getParent()) == P);
+      }
+
+      return *this;
+    }
+    defusechain_iterator operator++(int) {        // Postincrement
+      defusechain_iterator tmp = *this; ++*this; return tmp;
+    }
+
+    /// getOperandNo - Return the operand # of this MachineOperand in its
+    /// MachineInstr.
+    unsigned getOperandNo() const {
+      assert(Op && "Cannot dereference end iterator!");
+      return Op - &Op->getParent()->getOperand(0);
+    }
+
+    // Retrieve a reference to the current operand.
+    MachineOperand &operator*() const {
+      assert(Op && "Cannot dereference end iterator!");
+      return *Op;
+    }
+
+    MachineOperand *operator->() const {
+      assert(Op && "Cannot dereference end iterator!");
+      return Op;
+    }
+  };
+
+  /// defusechain_iterator - This class provides iterator support for machine
+  /// operands in the function that use or define a specific register.  If
+  /// ReturnUses is true it returns uses of registers, if ReturnDefs is true it
+  /// returns defs.  If neither are true then you are silly and it always
+  /// returns end().  If SkipDebug is true it skips uses marked Debug
+  /// when incrementing.
+  template<bool ReturnUses, bool ReturnDefs, bool SkipDebug,
+           bool ByOperand, bool ByInstr, bool ByBundle>
+  class defusechain_instr_iterator
+    : public std::iterator<std::forward_iterator_tag, MachineInstr, ptrdiff_t> {
+    MachineOperand *Op;
+    explicit defusechain_instr_iterator(MachineOperand *op) : Op(op) {
+      // If the first node isn't one we're interested in, advance to one that
+      // we are interested in.
+      if (op) {
+        if ((!ReturnUses && op->isUse()) ||
+            (!ReturnDefs && op->isDef()) ||
+            (SkipDebug && op->isDebug()))
+          advance();
+      }
+    }
+    friend class MachineRegisterInfo;
+
+    void advance() {
+      assert(Op && "Cannot increment end iterator!");
       Op = getNextOperandForReg(Op);
 
       // All defs come before the uses, so stop def_iterator early.
       if (!ReturnUses) {
         if (Op) {
           if (Op->isUse())
-            Op = 0;
+            Op = nullptr;
           else
             assert(!Op->isDebug() && "Can't have debug defs");
         }
@@ -605,52 +908,59 @@ public:
                       (SkipDebug && Op->isDebug())))
           Op = getNextOperandForReg(Op);
       }
+    }
+  public:
+    typedef std::iterator<std::forward_iterator_tag,
+                          MachineInstr, ptrdiff_t>::reference reference;
+    typedef std::iterator<std::forward_iterator_tag,
+                          MachineInstr, ptrdiff_t>::pointer pointer;
 
-      return *this;
+    defusechain_instr_iterator(const defusechain_instr_iterator &I) : Op(I.Op){}
+    defusechain_instr_iterator() : Op(nullptr) {}
+
+    bool operator==(const defusechain_instr_iterator &x) const {
+      return Op == x.Op;
     }
-    defusechain_iterator operator++(int) {        // Postincrement
-      defusechain_iterator tmp = *this; ++*this; return tmp;
+    bool operator!=(const defusechain_instr_iterator &x) const {
+      return !operator==(x);
     }
 
-    /// skipInstruction - move forward until reaching a different instruction.
-    /// Return the skipped instruction that is no longer pointed to, or NULL if
-    /// already pointing to end().
-    MachineInstr *skipInstruction() {
-      if (!Op) return 0;
-      MachineInstr *MI = Op->getParent();
-      do ++*this;
-      while (Op && Op->getParent() == MI);
-      return MI;
-    }
+    /// atEnd - return true if this iterator is equal to reg_end() on the value.
+    bool atEnd() const { return Op == nullptr; }
 
-    MachineInstr *skipBundle() {
-      if (!Op) return 0;
-      MachineInstr *MI = getBundleStart(Op->getParent());
-      do ++*this;
-      while (Op && getBundleStart(Op->getParent()) == MI);
-      return MI;
-    }
+    // Iterator traversal: forward iteration only
+    defusechain_instr_iterator &operator++() {          // Preincrement
+      assert(Op && "Cannot increment end iterator!");
+      if (ByOperand)
+        advance();
+      else if (ByInstr) {
+        MachineInstr *P = Op->getParent();
+        do {
+          advance();
+        } while (Op && Op->getParent() == P);
+      } else if (ByBundle) {
+        MachineInstr *P = getBundleStart(Op->getParent());
+        do {
+          advance();
+        } while (Op && getBundleStart(Op->getParent()) == P);
+      }
 
-    MachineOperand &getOperand() const {
-      assert(Op && "Cannot dereference end iterator!");
-      return *Op;
+      return *this;
     }
-
-    /// getOperandNo - Return the operand # of this MachineOperand in its
-    /// MachineInstr.
-    unsigned getOperandNo() const {
-      assert(Op && "Cannot dereference end iterator!");
-      return Op - &Op->getParent()->getOperand(0);
+    defusechain_instr_iterator operator++(int) {        // Postincrement
+      defusechain_instr_iterator tmp = *this; ++*this; return tmp;
     }
 
     // Retrieve a reference to the current operand.
     MachineInstr &operator*() const {
       assert(Op && "Cannot dereference end iterator!");
+      if (ByBundle) return *(getBundleStart(Op->getParent()));
       return *Op->getParent();
     }
 
     MachineInstr *operator->() const {
       assert(Op && "Cannot dereference end iterator!");
+      if (ByBundle) return getBundleStart(Op->getParent());
       return Op->getParent();
     }
   };
@@ -663,7 +973,7 @@ class PSetIterator {
   const int *PSet;
   unsigned Weight;
 public:
-  PSetIterator(): PSet(0), Weight(0) {}
+  PSetIterator(): PSet(nullptr), Weight(0) {}
   PSetIterator(unsigned RegUnit, const MachineRegisterInfo *MRI) {
     const TargetRegisterInfo *TRI = MRI->getTargetRegisterInfo();
     if (TargetRegisterInfo::isVirtualRegister(RegUnit)) {
@@ -676,7 +986,7 @@ public:
       Weight = TRI->getRegUnitWeight(RegUnit);
     }
     if (*PSet == -1)
-      PSet = 0;
+      PSet = nullptr;
   }
   bool isValid() const { return PSet; }
 
@@ -688,7 +998,7 @@ public:
     assert(isValid() && "Invalid PSetIterator.");
     ++PSet;
     if (*PSet == -1)
-      PSet = 0;
+      PSet = nullptr;
   }
 };
 
diff --git a/include/llvm/CodeGen/MachineSSAUpdater.h b/include/llvm/CodeGen/MachineSSAUpdater.h
index edf93d13bd1d..486a26e92714 100644
--- a/include/llvm/CodeGen/MachineSSAUpdater.h
+++ b/include/llvm/CodeGen/MachineSSAUpdater.h
@@ -14,6 +14,7 @@
 #ifndef LLVM_CODEGEN_MACHINESSAUPDATER_H
 #define LLVM_CODEGEN_MACHINESSAUPDATER_H
 
+#include "llvm/Support/Allocator.h"
 #include "llvm/Support/Compiler.h"
 
 namespace llvm {
@@ -26,7 +27,6 @@ namespace llvm {
   class TargetRegisterClass;
   template<typename T> class SmallVectorImpl;
   template<typename T> class SSAUpdaterTraits;
-  class BumpPtrAllocator;
 
 /// MachineSSAUpdater - This class updates SSA form for a set of virtual
 /// registers defined in multiple blocks.  This is used when code duplication
@@ -57,7 +57,7 @@ public:
   /// MachineSSAUpdater constructor.  If InsertedPHIs is specified, it will be
   /// filled in with all PHI Nodes created by rewriting.
   explicit MachineSSAUpdater(MachineFunction &MF,
-                             SmallVectorImpl<MachineInstr*> *InsertedPHIs = 0);
+                        SmallVectorImpl<MachineInstr*> *InsertedPHIs = nullptr);
   ~MachineSSAUpdater();
 
   /// Initialize - Reset this object to get ready for a new set of SSA
@@ -105,7 +105,6 @@ public:
   void RewriteUse(MachineOperand &U);
 
 private:
-  void ReplaceRegWith(unsigned OldReg, unsigned NewReg);
   unsigned GetValueAtEndOfBlockInternal(MachineBasicBlock *BB);
 
   void operator=(const MachineSSAUpdater&) LLVM_DELETED_FUNCTION;
diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h
index 77828953347c..7d85432101b5 100644
--- a/include/llvm/CodeGen/MachineScheduler.h
+++ b/include/llvm/CodeGen/MachineScheduler.h
@@ -23,7 +23,7 @@
 //   return new CustomMachineScheduler(C);
 // }
 //
-// The default scheduler, ScheduleDAGMI, builds the DAG and drives list
+// The default scheduler, ScheduleDAGMILive, builds the DAG and drives list
 // scheduling while updating the instruction stream, register pressure, and live
 // intervals. Most targets don't need to override the DAG builder and list
 // schedulier, but subtargets that require custom scheduling heuristics may
@@ -81,6 +81,8 @@
 #include "llvm/CodeGen/RegisterPressure.h"
 #include "llvm/CodeGen/ScheduleDAGInstrs.h"
 
+#include <memory>
+
 namespace llvm {
 
 extern cl::opt<bool> ForceTopDown;
@@ -93,6 +95,7 @@ class MachineLoopInfo;
 class RegisterClassInfo;
 class ScheduleDAGInstrs;
 class SchedDFSResult;
+class ScheduleHazardRecognizer;
 
 /// MachineSchedContext provides enough context from the MachineScheduler pass
 /// for the target to instantiate a scheduler.
@@ -154,8 +157,8 @@ struct MachineSchedPolicy {
   bool OnlyTopDown;
   bool OnlyBottomUp;
 
-  MachineSchedPolicy():
-    ShouldTrackPressure(false), OnlyTopDown(false), OnlyBottomUp(false) {}
+  MachineSchedPolicy(): ShouldTrackPressure(false), OnlyTopDown(false),
+    OnlyBottomUp(false) {}
 };
 
 /// MachineSchedStrategy - Interface to the scheduling algorithm used by
@@ -204,63 +207,6 @@ public:
   virtual void releaseBottomNode(SUnit *SU) = 0;
 };
 
-/// ReadyQueue encapsulates vector of "ready" SUnits with basic convenience
-/// methods for pushing and removing nodes. ReadyQueue's are uniquely identified
-/// by an ID. SUnit::NodeQueueId is a mask of the ReadyQueues the SUnit is in.
-///
-/// This is a convenience class that may be used by implementations of
-/// MachineSchedStrategy.
-class ReadyQueue {
-  unsigned ID;
-  std::string Name;
-  std::vector<SUnit*> Queue;
-
-public:
-  ReadyQueue(unsigned id, const Twine &name): ID(id), Name(name.str()) {}
-
-  unsigned getID() const { return ID; }
-
-  StringRef getName() const { return Name; }
-
-  // SU is in this queue if it's NodeQueueID is a superset of this ID.
-  bool isInQueue(SUnit *SU) const { return (SU->NodeQueueId & ID); }
-
-  bool empty() const { return Queue.empty(); }
-
-  void clear() { Queue.clear(); }
-
-  unsigned size() const { return Queue.size(); }
-
-  typedef std::vector<SUnit*>::iterator iterator;
-
-  iterator begin() { return Queue.begin(); }
-
-  iterator end() { return Queue.end(); }
-
-  ArrayRef<SUnit*> elements() { return Queue; }
-
-  iterator find(SUnit *SU) {
-    return std::find(Queue.begin(), Queue.end(), SU);
-  }
-
-  void push(SUnit *SU) {
-    Queue.push_back(SU);
-    SU->NodeQueueId |= ID;
-  }
-
-  iterator remove(iterator I) {
-    (*I)->NodeQueueId &= ~ID;
-    *I = Queue.back();
-    unsigned idx = I - Queue.begin();
-    Queue.pop_back();
-    return Queue.begin() + idx;
-  }
-
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
-  void dump();
-#endif
-};
-
 /// Mutate the DAG as a postpass after normal DAG building.
 class ScheduleDAGMutation {
   virtual void anchor();
@@ -270,52 +216,27 @@ public:
   virtual void apply(ScheduleDAGMI *DAG) = 0;
 };
 
-/// ScheduleDAGMI is an implementation of ScheduleDAGInstrs that schedules
-/// machine instructions while updating LiveIntervals and tracking regpressure.
+/// ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply
+/// schedules machine instructions according to the given MachineSchedStrategy
+/// without much extra book-keeping. This is the common functionality between
+/// PreRA and PostRA MachineScheduler.
 class ScheduleDAGMI : public ScheduleDAGInstrs {
 protected:
   AliasAnalysis *AA;
-  RegisterClassInfo *RegClassInfo;
-  MachineSchedStrategy *SchedImpl;
-
-  /// Information about DAG subtrees. If DFSResult is NULL, then SchedulerTrees
-  /// will be empty.
-  SchedDFSResult *DFSResult;
-  BitVector ScheduledTrees;
+  std::unique_ptr<MachineSchedStrategy> SchedImpl;
 
   /// Topo - A topological ordering for SUnits which permits fast IsReachable
   /// and similar queries.
   ScheduleDAGTopologicalSort Topo;
 
   /// Ordered list of DAG postprocessing steps.
-  std::vector<ScheduleDAGMutation*> Mutations;
-
-  MachineBasicBlock::iterator LiveRegionEnd;
-
-  // Map each SU to its summary of pressure changes. This array is updated for
-  // liveness during bottom-up scheduling. Top-down scheduling may proceed but
-  // has no affect on the pressure diffs.
-  PressureDiffs SUPressureDiffs;
-
-  /// Register pressure in this region computed by initRegPressure.
-  bool ShouldTrackPressure;
-  IntervalPressure RegPressure;
-  RegPressureTracker RPTracker;
-
-  /// List of pressure sets that exceed the target's pressure limit before
-  /// scheduling, listed in increasing set ID order. Each pressure set is paired
-  /// with its max pressure in the currently scheduled regions.
-  std::vector<PressureChange> RegionCriticalPSets;
+  std::vector<std::unique_ptr<ScheduleDAGMutation>> Mutations;
 
   /// The top of the unscheduled zone.
   MachineBasicBlock::iterator CurrentTop;
-  IntervalPressure TopPressure;
-  RegPressureTracker TopRPTracker;
 
   /// The bottom of the unscheduled zone.
   MachineBasicBlock::iterator CurrentBottom;
-  IntervalPressure BotPressure;
-  RegPressureTracker BotRPTracker;
 
   /// Record the next node in a scheduled cluster.
   const SUnit *NextClusterPred;
@@ -326,32 +247,31 @@ protected:
   /// scheduler at the point determined by misched-cutoff.
   unsigned NumInstrsScheduled;
 #endif
-
 public:
-  ScheduleDAGMI(MachineSchedContext *C, MachineSchedStrategy *S):
-    ScheduleDAGInstrs(*C->MF, *C->MLI, *C->MDT, /*IsPostRA=*/false, C->LIS),
-    AA(C->AA), RegClassInfo(C->RegClassInfo), SchedImpl(S), DFSResult(0),
-    Topo(SUnits, &ExitSU), ShouldTrackPressure(false),
-    RPTracker(RegPressure), CurrentTop(), TopRPTracker(TopPressure),
-    CurrentBottom(), BotRPTracker(BotPressure),
-    NextClusterPred(NULL), NextClusterSucc(NULL) {
+  ScheduleDAGMI(MachineSchedContext *C, std::unique_ptr<MachineSchedStrategy> S,
+                bool IsPostRA)
+      : ScheduleDAGInstrs(*C->MF, *C->MLI, *C->MDT, IsPostRA,
+                          /*RemoveKillFlags=*/IsPostRA, C->LIS),
+        AA(C->AA), SchedImpl(std::move(S)), Topo(SUnits, &ExitSU), CurrentTop(),
+        CurrentBottom(), NextClusterPred(nullptr), NextClusterSucc(nullptr) {
 #ifndef NDEBUG
     NumInstrsScheduled = 0;
 #endif
   }
 
-  virtual ~ScheduleDAGMI();
+  // Provide a vtable anchor
+  ~ScheduleDAGMI() override;
 
-  /// \brief Return true if register pressure tracking is enabled.
-  bool isTrackingPressure() const { return ShouldTrackPressure; }
+  /// Return true if this DAG supports VReg liveness and RegPressure.
+  virtual bool hasVRegLiveness() const { return false; }
 
   /// Add a postprocessing step to the DAG builder.
   /// Mutations are applied in the order that they are added after normal DAG
   /// building and before MachineSchedStrategy initialization.
   ///
   /// ScheduleDAGMI takes ownership of the Mutation object.
-  void addMutation(ScheduleDAGMutation *Mutation) {
-    Mutations.push_back(Mutation);
+  void addMutation(std::unique_ptr<ScheduleDAGMutation> Mutation) {
+    Mutations.push_back(std::move(Mutation));
   }
 
   /// \brief True if an edge can be added from PredSU to SuccSU without creating
@@ -374,16 +294,106 @@ public:
   void enterRegion(MachineBasicBlock *bb,
                    MachineBasicBlock::iterator begin,
                    MachineBasicBlock::iterator end,
-                   unsigned regioninstrs) LLVM_OVERRIDE;
+                   unsigned regioninstrs) override;
 
   /// Implement ScheduleDAGInstrs interface for scheduling a sequence of
   /// reorderable instructions.
-  virtual void schedule();
+  void schedule() override;
 
   /// Change the position of an instruction within the basic block and update
   /// live ranges and region boundary iterators.
   void moveInstruction(MachineInstr *MI, MachineBasicBlock::iterator InsertPos);
 
+  const SUnit *getNextClusterPred() const { return NextClusterPred; }
+
+  const SUnit *getNextClusterSucc() const { return NextClusterSucc; }
+
+  void viewGraph(const Twine &Name, const Twine &Title) override;
+  void viewGraph() override;
+
+protected:
+  // Top-Level entry points for the schedule() driver...
+
+  /// Apply each ScheduleDAGMutation step in order. This allows different
+  /// instances of ScheduleDAGMI to perform custom DAG postprocessing.
+  void postprocessDAG();
+
+  /// Release ExitSU predecessors and setup scheduler queues.
+  void initQueues(ArrayRef<SUnit*> TopRoots, ArrayRef<SUnit*> BotRoots);
+
+  /// Update scheduler DAG and queues after scheduling an instruction.
+  void updateQueues(SUnit *SU, bool IsTopNode);
+
+  /// Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
+  void placeDebugValues();
+
+  /// \brief dump the scheduled Sequence.
+  void dumpSchedule() const;
+
+  // Lesser helpers...
+  bool checkSchedLimit();
+
+  void findRootsAndBiasEdges(SmallVectorImpl<SUnit*> &TopRoots,
+                             SmallVectorImpl<SUnit*> &BotRoots);
+
+  void releaseSucc(SUnit *SU, SDep *SuccEdge);
+  void releaseSuccessors(SUnit *SU);
+  void releasePred(SUnit *SU, SDep *PredEdge);
+  void releasePredecessors(SUnit *SU);
+};
+
+/// ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules
+/// machine instructions while updating LiveIntervals and tracking regpressure.
+class ScheduleDAGMILive : public ScheduleDAGMI {
+protected:
+  RegisterClassInfo *RegClassInfo;
+
+  /// Information about DAG subtrees. If DFSResult is NULL, then SchedulerTrees
+  /// will be empty.
+  SchedDFSResult *DFSResult;
+  BitVector ScheduledTrees;
+
+  MachineBasicBlock::iterator LiveRegionEnd;
+
+  // Map each SU to its summary of pressure changes. This array is updated for
+  // liveness during bottom-up scheduling. Top-down scheduling may proceed but
+  // has no affect on the pressure diffs.
+  PressureDiffs SUPressureDiffs;
+
+  /// Register pressure in this region computed by initRegPressure.
+  bool ShouldTrackPressure;
+  IntervalPressure RegPressure;
+  RegPressureTracker RPTracker;
+
+  /// List of pressure sets that exceed the target's pressure limit before
+  /// scheduling, listed in increasing set ID order. Each pressure set is paired
+  /// with its max pressure in the currently scheduled regions.
+  std::vector<PressureChange> RegionCriticalPSets;
+
+  /// The top of the unscheduled zone.
+  IntervalPressure TopPressure;
+  RegPressureTracker TopRPTracker;
+
+  /// The bottom of the unscheduled zone.
+  IntervalPressure BotPressure;
+  RegPressureTracker BotRPTracker;
+
+public:
+  ScheduleDAGMILive(MachineSchedContext *C,
+                    std::unique_ptr<MachineSchedStrategy> S)
+      : ScheduleDAGMI(C, std::move(S), /*IsPostRA=*/false),
+        RegClassInfo(C->RegClassInfo), DFSResult(nullptr),
+        ShouldTrackPressure(false), RPTracker(RegPressure),
+        TopRPTracker(TopPressure), BotRPTracker(BotPressure) {}
+
+  virtual ~ScheduleDAGMILive();
+
+  /// Return true if this DAG supports VReg liveness and RegPressure.
+  bool hasVRegLiveness() const override { return true; }
+
+  /// \brief Return true if register pressure tracking is enabled.
+  bool isTrackingPressure() const { return ShouldTrackPressure; }
+
   /// Get current register pressure for the top scheduled instructions.
   const IntervalPressure &getTopPressure() const { return TopPressure; }
   const RegPressureTracker &getTopRPTracker() const { return TopRPTracker; }
@@ -403,10 +413,6 @@ public:
     return SUPressureDiffs[SU->NodeNum];
   }
 
-  const SUnit *getNextClusterPred() const { return NextClusterPred; }
-
-  const SUnit *getNextClusterSucc() const { return NextClusterSucc; }
-
   /// Compute a DFSResult after DAG building is complete, and before any
   /// queue comparisons.
   void computeDFSResult();
@@ -416,12 +422,21 @@ public:
 
   BitVector &getScheduledTrees() { return ScheduledTrees; }
 
+  /// Implement the ScheduleDAGInstrs interface for handling the next scheduling
+  /// region. This covers all instructions in a block, while schedule() may only
+  /// cover a subset.
+  void enterRegion(MachineBasicBlock *bb,
+                   MachineBasicBlock::iterator begin,
+                   MachineBasicBlock::iterator end,
+                   unsigned regioninstrs) override;
+
+  /// Implement ScheduleDAGInstrs interface for scheduling a sequence of
+  /// reorderable instructions.
+  void schedule() override;
+
   /// Compute the cyclic critical path through the DAG.
   unsigned computeCyclicCriticalPath();
 
-  void viewGraph(const Twine &Name, const Twine &Title) LLVM_OVERRIDE;
-  void viewGraph() LLVM_OVERRIDE;
-
 protected:
   // Top-Level entry points for the schedule() driver...
 
@@ -431,25 +446,9 @@ protected:
   /// bottom of the DAG region without covereing any unscheduled instruction.
   void buildDAGWithRegPressure();
 
-  /// Apply each ScheduleDAGMutation step in order. This allows different
-  /// instances of ScheduleDAGMI to perform custom DAG postprocessing.
-  void postprocessDAG();
-
-  /// Release ExitSU predecessors and setup scheduler queues.
-  void initQueues(ArrayRef<SUnit*> TopRoots, ArrayRef<SUnit*> BotRoots);
-
   /// Move an instruction and update register pressure.
   void scheduleMI(SUnit *SU, bool IsTopNode);
 
-  /// Update scheduler DAG and queues after scheduling an instruction.
-  void updateQueues(SUnit *SU, bool IsTopNode);
-
-  /// Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues.
-  void placeDebugValues();
-
-  /// \brief dump the scheduled Sequence.
-  void dumpSchedule() const;
-
   // Lesser helpers...
 
   void initRegPressure();
@@ -458,16 +457,495 @@ protected:
 
   void updateScheduledPressure(const SUnit *SU,
                                const std::vector<unsigned> &NewMaxPressure);
+};
 
-  bool checkSchedLimit();
+//===----------------------------------------------------------------------===//
+///
+/// Helpers for implementing custom MachineSchedStrategy classes. These take
+/// care of the book-keeping associated with list scheduling heuristics.
+///
+//===----------------------------------------------------------------------===//
 
-  void findRootsAndBiasEdges(SmallVectorImpl<SUnit*> &TopRoots,
-                             SmallVectorImpl<SUnit*> &BotRoots);
+/// ReadyQueue encapsulates vector of "ready" SUnits with basic convenience
+/// methods for pushing and removing nodes. ReadyQueue's are uniquely identified
+/// by an ID. SUnit::NodeQueueId is a mask of the ReadyQueues the SUnit is in.
+///
+/// This is a convenience class that may be used by implementations of
+/// MachineSchedStrategy.
+class ReadyQueue {
+  unsigned ID;
+  std::string Name;
+  std::vector<SUnit*> Queue;
 
-  void releaseSucc(SUnit *SU, SDep *SuccEdge);
-  void releaseSuccessors(SUnit *SU);
-  void releasePred(SUnit *SU, SDep *PredEdge);
-  void releasePredecessors(SUnit *SU);
+public:
+  ReadyQueue(unsigned id, const Twine &name): ID(id), Name(name.str()) {}
+
+  unsigned getID() const { return ID; }
+
+  StringRef getName() const { return Name; }
+
+  // SU is in this queue if it's NodeQueueID is a superset of this ID.
+  bool isInQueue(SUnit *SU) const { return (SU->NodeQueueId & ID); }
+
+  bool empty() const { return Queue.empty(); }
+
+  void clear() { Queue.clear(); }
+
+  unsigned size() const { return Queue.size(); }
+
+  typedef std::vector<SUnit*>::iterator iterator;
+
+  iterator begin() { return Queue.begin(); }
+
+  iterator end() { return Queue.end(); }
+
+  ArrayRef<SUnit*> elements() { return Queue; }
+
+  iterator find(SUnit *SU) {
+    return std::find(Queue.begin(), Queue.end(), SU);
+  }
+
+  void push(SUnit *SU) {
+    Queue.push_back(SU);
+    SU->NodeQueueId |= ID;
+  }
+
+  iterator remove(iterator I) {
+    (*I)->NodeQueueId &= ~ID;
+    *I = Queue.back();
+    unsigned idx = I - Queue.begin();
+    Queue.pop_back();
+    return Queue.begin() + idx;
+  }
+
+  void dump();
+};
+
+/// Summarize the unscheduled region.
+struct SchedRemainder {
+  // Critical path through the DAG in expected latency.
+  unsigned CriticalPath;
+  unsigned CyclicCritPath;
+
+  // Scaled count of micro-ops left to schedule.
+  unsigned RemIssueCount;
+
+  bool IsAcyclicLatencyLimited;
+
+  // Unscheduled resources
+  SmallVector<unsigned, 16> RemainingCounts;
+
+  void reset() {
+    CriticalPath = 0;
+    CyclicCritPath = 0;
+    RemIssueCount = 0;
+    IsAcyclicLatencyLimited = false;
+    RemainingCounts.clear();
+  }
+
+  SchedRemainder() { reset(); }
+
+  void init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel);
+};
+
+/// Each Scheduling boundary is associated with ready queues. It tracks the
+/// current cycle in the direction of movement, and maintains the state
+/// of "hazards" and other interlocks at the current cycle.
+class SchedBoundary {
+public:
+  /// SUnit::NodeQueueId: 0 (none), 1 (top), 2 (bot), 3 (both)
+  enum {
+    TopQID = 1,
+    BotQID = 2,
+    LogMaxQID = 2
+  };
+
+  ScheduleDAGMI *DAG;
+  const TargetSchedModel *SchedModel;
+  SchedRemainder *Rem;
+
+  ReadyQueue Available;
+  ReadyQueue Pending;
+
+  ScheduleHazardRecognizer *HazardRec;
+
+private:
+  /// True if the pending Q should be checked/updated before scheduling another
+  /// instruction.
+  bool CheckPending;
+
+  // For heuristics, keep a list of the nodes that immediately depend on the
+  // most recently scheduled node.
+  SmallPtrSet<const SUnit*, 8> NextSUs;
+
+  /// Number of cycles it takes to issue the instructions scheduled in this
+  /// zone. It is defined as: scheduled-micro-ops / issue-width + stalls.
+  /// See getStalls().
+  unsigned CurrCycle;
+
+  /// Micro-ops issued in the current cycle
+  unsigned CurrMOps;
+
+  /// MinReadyCycle - Cycle of the soonest available instruction.
+  unsigned MinReadyCycle;
+
+  // The expected latency of the critical path in this scheduled zone.
+  unsigned ExpectedLatency;
+
+  // The latency of dependence chains leading into this zone.
+  // For each node scheduled bottom-up: DLat = max DLat, N.Depth.
+  // For each cycle scheduled: DLat -= 1.
+  unsigned DependentLatency;
+
+  /// Count the scheduled (issued) micro-ops that can be retired by
+  /// time=CurrCycle assuming the first scheduled instr is retired at time=0.
+  unsigned RetiredMOps;
+
+  // Count scheduled resources that have been executed. Resources are
+  // considered executed if they become ready in the time that it takes to
+  // saturate any resource including the one in question. Counts are scaled
+  // for direct comparison with other resources. Counts can be compared with
+  // MOps * getMicroOpFactor and Latency * getLatencyFactor.
+  SmallVector<unsigned, 16> ExecutedResCounts;
+
+  /// Cache the max count for a single resource.
+  unsigned MaxExecutedResCount;
+
+  // Cache the critical resources ID in this scheduled zone.
+  unsigned ZoneCritResIdx;
+
+  // Is the scheduled region resource limited vs. latency limited.
+  bool IsResourceLimited;
+
+  // Record the highest cycle at which each resource has been reserved by a
+  // scheduled instruction.
+  SmallVector<unsigned, 16> ReservedCycles;
+
+#ifndef NDEBUG
+  // Remember the greatest possible stall as an upper bound on the number of
+  // times we should retry the pending queue because of a hazard.
+  unsigned MaxObservedStall;
+#endif
+
+public:
+  /// Pending queues extend the ready queues with the same ID and the
+  /// PendingFlag set.
+  SchedBoundary(unsigned ID, const Twine &Name):
+    DAG(nullptr), SchedModel(nullptr), Rem(nullptr), Available(ID, Name+".A"),
+    Pending(ID << LogMaxQID, Name+".P"),
+    HazardRec(nullptr) {
+    reset();
+  }
+
+  ~SchedBoundary();
+
+  void reset();
+
+  void init(ScheduleDAGMI *dag, const TargetSchedModel *smodel,
+            SchedRemainder *rem);
+
+  bool isTop() const {
+    return Available.getID() == TopQID;
+  }
+
+  /// Number of cycles to issue the instructions scheduled in this zone.
+  unsigned getCurrCycle() const { return CurrCycle; }
+
+  /// Micro-ops issued in the current cycle
+  unsigned getCurrMOps() const { return CurrMOps; }
+
+  /// Return true if the given SU is used by the most recently scheduled
+  /// instruction.
+  bool isNextSU(const SUnit *SU) const { return NextSUs.count(SU); }
+
+  // The latency of dependence chains leading into this zone.
+  unsigned getDependentLatency() const { return DependentLatency; }
+
+  /// Get the number of latency cycles "covered" by the scheduled
+  /// instructions. This is the larger of the critical path within the zone
+  /// and the number of cycles required to issue the instructions.
+  unsigned getScheduledLatency() const {
+    return std::max(ExpectedLatency, CurrCycle);
+  }
+
+  unsigned getUnscheduledLatency(SUnit *SU) const {
+    return isTop() ? SU->getHeight() : SU->getDepth();
+  }
+
+  unsigned getResourceCount(unsigned ResIdx) const {
+    return ExecutedResCounts[ResIdx];
+  }
+
+  /// Get the scaled count of scheduled micro-ops and resources, including
+  /// executed resources.
+  unsigned getCriticalCount() const {
+    if (!ZoneCritResIdx)
+      return RetiredMOps * SchedModel->getMicroOpFactor();
+    return getResourceCount(ZoneCritResIdx);
+  }
+
+  /// Get a scaled count for the minimum execution time of the scheduled
+  /// micro-ops that are ready to execute by getExecutedCount. Notice the
+  /// feedback loop.
+  unsigned getExecutedCount() const {
+    return std::max(CurrCycle * SchedModel->getLatencyFactor(),
+                    MaxExecutedResCount);
+  }
+
+  unsigned getZoneCritResIdx() const { return ZoneCritResIdx; }
+
+  // Is the scheduled region resource limited vs. latency limited.
+  bool isResourceLimited() const { return IsResourceLimited; }
+
+  /// Get the difference between the given SUnit's ready time and the current
+  /// cycle.
+  unsigned getLatencyStallCycles(SUnit *SU);
+
+  unsigned getNextResourceCycle(unsigned PIdx, unsigned Cycles);
+
+  bool checkHazard(SUnit *SU);
+
+  unsigned findMaxLatency(ArrayRef<SUnit*> ReadySUs);
+
+  unsigned getOtherResourceCount(unsigned &OtherCritIdx);
+
+  void releaseNode(SUnit *SU, unsigned ReadyCycle);
+
+  void releaseTopNode(SUnit *SU);
+
+  void releaseBottomNode(SUnit *SU);
+
+  void bumpCycle(unsigned NextCycle);
+
+  void incExecutedResources(unsigned PIdx, unsigned Count);
+
+  unsigned countResource(unsigned PIdx, unsigned Cycles, unsigned ReadyCycle);
+
+  void bumpNode(SUnit *SU);
+
+  void releasePending();
+
+  void removeReady(SUnit *SU);
+
+  /// Call this before applying any other heuristics to the Available queue.
+  /// Updates the Available/Pending Q's if necessary and returns the single
+  /// available instruction, or NULL if there are multiple candidates.
+  SUnit *pickOnlyChoice();
+
+#ifndef NDEBUG
+  void dumpScheduledState();
+#endif
+};
+
+/// Base class for GenericScheduler. This class maintains information about
+/// scheduling candidates based on TargetSchedModel making it easy to implement
+/// heuristics for either preRA or postRA scheduling.
+class GenericSchedulerBase : public MachineSchedStrategy {
+public:
+  /// Represent the type of SchedCandidate found within a single queue.
+  /// pickNodeBidirectional depends on these listed by decreasing priority.
+  enum CandReason {
+    NoCand, PhysRegCopy, RegExcess, RegCritical, Stall, Cluster, Weak, RegMax,
+    ResourceReduce, ResourceDemand, BotHeightReduce, BotPathReduce,
+    TopDepthReduce, TopPathReduce, NextDefUse, NodeOrder};
+
+#ifndef NDEBUG
+  static const char *getReasonStr(GenericSchedulerBase::CandReason Reason);
+#endif
+
+  /// Policy for scheduling the next instruction in the candidate's zone.
+  struct CandPolicy {
+    bool ReduceLatency;
+    unsigned ReduceResIdx;
+    unsigned DemandResIdx;
+
+    CandPolicy(): ReduceLatency(false), ReduceResIdx(0), DemandResIdx(0) {}
+  };
+
+  /// Status of an instruction's critical resource consumption.
+  struct SchedResourceDelta {
+    // Count critical resources in the scheduled region required by SU.
+    unsigned CritResources;
+
+    // Count critical resources from another region consumed by SU.
+    unsigned DemandedResources;
+
+    SchedResourceDelta(): CritResources(0), DemandedResources(0) {}
+
+    bool operator==(const SchedResourceDelta &RHS) const {
+      return CritResources == RHS.CritResources
+        && DemandedResources == RHS.DemandedResources;
+    }
+    bool operator!=(const SchedResourceDelta &RHS) const {
+      return !operator==(RHS);
+    }
+  };
+
+  /// Store the state used by GenericScheduler heuristics, required for the
+  /// lifetime of one invocation of pickNode().
+  struct SchedCandidate {
+    CandPolicy Policy;
+
+    // The best SUnit candidate.
+    SUnit *SU;
+
+    // The reason for this candidate.
+    CandReason Reason;
+
+    // Set of reasons that apply to multiple candidates.
+    uint32_t RepeatReasonSet;
+
+    // Register pressure values for the best candidate.
+    RegPressureDelta RPDelta;
+
+    // Critical resource consumption of the best candidate.
+    SchedResourceDelta ResDelta;
+
+    SchedCandidate(const CandPolicy &policy)
+      : Policy(policy), SU(nullptr), Reason(NoCand), RepeatReasonSet(0) {}
+
+    bool isValid() const { return SU; }
+
+    // Copy the status of another candidate without changing policy.
+    void setBest(SchedCandidate &Best) {
+      assert(Best.Reason != NoCand && "uninitialized Sched candidate");
+      SU = Best.SU;
+      Reason = Best.Reason;
+      RPDelta = Best.RPDelta;
+      ResDelta = Best.ResDelta;
+    }
+
+    bool isRepeat(CandReason R) { return RepeatReasonSet & (1 << R); }
+    void setRepeat(CandReason R) { RepeatReasonSet |= (1 << R); }
+
+    void initResourceDelta(const ScheduleDAGMI *DAG,
+                           const TargetSchedModel *SchedModel);
+  };
+
+protected:
+  const MachineSchedContext *Context;
+  const TargetSchedModel *SchedModel;
+  const TargetRegisterInfo *TRI;
+
+  SchedRemainder Rem;
+protected:
+  GenericSchedulerBase(const MachineSchedContext *C):
+    Context(C), SchedModel(nullptr), TRI(nullptr) {}
+
+  void setPolicy(CandPolicy &Policy, bool IsPostRA, SchedBoundary &CurrZone,
+                 SchedBoundary *OtherZone);
+
+#ifndef NDEBUG
+  void traceCandidate(const SchedCandidate &Cand);
+#endif
+};
+
+/// GenericScheduler shrinks the unscheduled zone using heuristics to balance
+/// the schedule.
+class GenericScheduler : public GenericSchedulerBase {
+  ScheduleDAGMILive *DAG;
+
+  // State of the top and bottom scheduled instruction boundaries.
+  SchedBoundary Top;
+  SchedBoundary Bot;
+
+  MachineSchedPolicy RegionPolicy;
+public:
+  GenericScheduler(const MachineSchedContext *C):
+    GenericSchedulerBase(C), DAG(nullptr), Top(SchedBoundary::TopQID, "TopQ"),
+    Bot(SchedBoundary::BotQID, "BotQ") {}
+
+  void initPolicy(MachineBasicBlock::iterator Begin,
+                  MachineBasicBlock::iterator End,
+                  unsigned NumRegionInstrs) override;
+
+  bool shouldTrackPressure() const override {
+    return RegionPolicy.ShouldTrackPressure;
+  }
+
+  void initialize(ScheduleDAGMI *dag) override;
+
+  SUnit *pickNode(bool &IsTopNode) override;
+
+  void schedNode(SUnit *SU, bool IsTopNode) override;
+
+  void releaseTopNode(SUnit *SU) override {
+    Top.releaseTopNode(SU);
+  }
+
+  void releaseBottomNode(SUnit *SU) override {
+    Bot.releaseBottomNode(SU);
+  }
+
+  void registerRoots() override;
+
+protected:
+  void checkAcyclicLatency();
+
+  void tryCandidate(SchedCandidate &Cand,
+                    SchedCandidate &TryCand,
+                    SchedBoundary &Zone,
+                    const RegPressureTracker &RPTracker,
+                    RegPressureTracker &TempTracker);
+
+  SUnit *pickNodeBidirectional(bool &IsTopNode);
+
+  void pickNodeFromQueue(SchedBoundary &Zone,
+                         const RegPressureTracker &RPTracker,
+                         SchedCandidate &Candidate);
+
+  void reschedulePhysRegCopies(SUnit *SU, bool isTop);
+};
+
+/// PostGenericScheduler - Interface to the scheduling algorithm used by
+/// ScheduleDAGMI.
+///
+/// Callbacks from ScheduleDAGMI:
+///   initPolicy -> initialize(DAG) -> registerRoots -> pickNode ...
+class PostGenericScheduler : public GenericSchedulerBase {
+  ScheduleDAGMI *DAG;
+  SchedBoundary Top;
+  SmallVector<SUnit*, 8> BotRoots;
+public:
+  PostGenericScheduler(const MachineSchedContext *C):
+    GenericSchedulerBase(C), Top(SchedBoundary::TopQID, "TopQ") {}
+
+  virtual ~PostGenericScheduler() {}
+
+  void initPolicy(MachineBasicBlock::iterator Begin,
+                  MachineBasicBlock::iterator End,
+                  unsigned NumRegionInstrs) override {
+    /* no configurable policy */
+  };
+
+  /// PostRA scheduling does not track pressure.
+  bool shouldTrackPressure() const override { return false; }
+
+  void initialize(ScheduleDAGMI *Dag) override;
+
+  void registerRoots() override;
+
+  SUnit *pickNode(bool &IsTopNode) override;
+
+  void scheduleTree(unsigned SubtreeID) override {
+    llvm_unreachable("PostRA scheduler does not support subtree analysis.");
+  }
+
+  void schedNode(SUnit *SU, bool IsTopNode) override;
+
+  void releaseTopNode(SUnit *SU) override {
+    Top.releaseTopNode(SU);
+  }
+
+  // Only called for roots.
+  void releaseBottomNode(SUnit *SU) override {
+    BotRoots.push_back(SU);
+  }
+
+protected:
+  void tryCandidate(SchedCandidate &Cand, SchedCandidate &TryCand);
+
+  void pickNodeFromQueue(SchedCandidate &Cand);
 };
 
 } // namespace llvm
diff --git a/include/llvm/CodeGen/MachineTraceMetrics.h b/include/llvm/CodeGen/MachineTraceMetrics.h
index 9794707e3544..323b694f3933 100644
--- a/include/llvm/CodeGen/MachineTraceMetrics.h
+++ b/include/llvm/CodeGen/MachineTraceMetrics.h
@@ -77,10 +77,10 @@ public:
   class Trace;
   static char ID;
   MachineTraceMetrics();
-  void getAnalysisUsage(AnalysisUsage&) const;
-  bool runOnMachineFunction(MachineFunction&);
-  void releaseMemory();
-  void verifyAnalysis() const;
+  void getAnalysisUsage(AnalysisUsage&) const override;
+  bool runOnMachineFunction(MachineFunction&) override;
+  void releaseMemory() override;
+  void verifyAnalysis() const override;
 
   friend class Ensemble;
   friend class Trace;
@@ -154,7 +154,7 @@ public:
     unsigned InstrHeight;
 
     TraceBlockInfo() :
-      Pred(0), Succ(0),
+      Pred(nullptr), Succ(nullptr),
       InstrDepth(~0u), InstrHeight(~0u),
       HasValidInstrDepths(false), HasValidInstrHeights(false) {}
 
diff --git a/include/llvm/CodeGen/MachineValueType.h b/include/llvm/CodeGen/MachineValueType.h
new file mode 100644
index 000000000000..ad215ec09843
--- /dev/null
+++ b/include/llvm/CodeGen/MachineValueType.h
@@ -0,0 +1,579 @@
+//===- CodeGen/MachineValueType.h - Machine-Level types ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the set of machine-level target independent types which
+// legal values in the code generator use.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_MACHINEVALUETYPE_H
+#define LLVM_CODEGEN_MACHINEVALUETYPE_H
+
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MathExtras.h"
+
+namespace llvm {
+
+  class Type;
+
+  /// MVT - Machine Value Type. Every type that is supported natively by some
+  /// processor targeted by LLVM occurs here. This means that any legal value
+  /// type can be represented by an MVT.
+  class MVT {
+  public:
+    enum SimpleValueType {
+      // INVALID_SIMPLE_VALUE_TYPE - Simple value types less than zero are
+      // considered extended value types.
+      INVALID_SIMPLE_VALUE_TYPE = -1,
+
+      // If you change this numbering, you must change the values in
+      // ValueTypes.td as well!
+      Other          =   0,   // This is a non-standard value
+      i1             =   1,   // This is a 1 bit integer value
+      i8             =   2,   // This is an 8 bit integer value
+      i16            =   3,   // This is a 16 bit integer value
+      i32            =   4,   // This is a 32 bit integer value
+      i64            =   5,   // This is a 64 bit integer value
+      i128           =   6,   // This is a 128 bit integer value
+
+      FIRST_INTEGER_VALUETYPE = i1,
+      LAST_INTEGER_VALUETYPE  = i128,
+
+      f16            =   7,   // This is a 16 bit floating point value
+      f32            =   8,   // This is a 32 bit floating point value
+      f64            =   9,   // This is a 64 bit floating point value
+      f80            =  10,   // This is a 80 bit floating point value
+      f128           =  11,   // This is a 128 bit floating point value
+      ppcf128        =  12,   // This is a PPC 128-bit floating point value
+
+      FIRST_FP_VALUETYPE = f16,
+      LAST_FP_VALUETYPE  = ppcf128,
+
+      v2i1           =  13,   //  2 x i1
+      v4i1           =  14,   //  4 x i1
+      v8i1           =  15,   //  8 x i1
+      v16i1          =  16,   // 16 x i1
+      v32i1          =  17,   // 32 x i1
+      v64i1          =  18,   // 64 x i1
+
+      v1i8           =  19,   //  1 x i8
+      v2i8           =  20,   //  2 x i8
+      v4i8           =  21,   //  4 x i8
+      v8i8           =  22,   //  8 x i8
+      v16i8          =  23,   // 16 x i8
+      v32i8          =  24,   // 32 x i8
+      v64i8          =  25,   // 64 x i8
+      v1i16          =  26,   //  1 x i16
+      v2i16          =  27,   //  2 x i16
+      v4i16          =  28,   //  4 x i16
+      v8i16          =  29,   //  8 x i16
+      v16i16         =  30,   // 16 x i16
+      v32i16         =  31,   // 32 x i16
+      v1i32          =  32,   //  1 x i32
+      v2i32          =  33,   //  2 x i32
+      v4i32          =  34,   //  4 x i32
+      v8i32          =  35,   //  8 x i32
+      v16i32         =  36,   // 16 x i32
+      v1i64          =  37,   //  1 x i64
+      v2i64          =  38,   //  2 x i64
+      v4i64          =  39,   //  4 x i64
+      v8i64          =  40,   //  8 x i64
+      v16i64         =  41,   // 16 x i64
+
+      FIRST_INTEGER_VECTOR_VALUETYPE = v2i1,
+      LAST_INTEGER_VECTOR_VALUETYPE = v16i64,
+
+      v2f16          =  42,   //  2 x f16
+      v4f16          =  43,   //  4 x f16
+      v8f16          =  44,   //  8 x f16
+      v1f32          =  45,   //  1 x f32
+      v2f32          =  46,   //  2 x f32
+      v4f32          =  47,   //  4 x f32
+      v8f32          =  48,   //  8 x f32
+      v16f32         =  49,   // 16 x f32
+      v1f64          =  50,   //  1 x f64
+      v2f64          =  51,   //  2 x f64
+      v4f64          =  52,   //  4 x f64
+      v8f64          =  53,   //  8 x f64
+
+      FIRST_FP_VECTOR_VALUETYPE = v2f16,
+      LAST_FP_VECTOR_VALUETYPE = v8f64,
+
+      FIRST_VECTOR_VALUETYPE = v2i1,
+      LAST_VECTOR_VALUETYPE  = v8f64,
+
+      x86mmx         =  54,   // This is an X86 MMX value
+
+      Glue           =  55,   // This glues nodes together during pre-RA sched
+
+      isVoid         =  56,   // This has no value
+
+      Untyped        =  57,   // This value takes a register, but has
+                              // unspecified type.  The register class
+                              // will be determined by the opcode.
+
+      LAST_VALUETYPE =  58,   // This always remains at the end of the list.
+
+      // This is the current maximum for LAST_VALUETYPE.
+      // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
+      // This value must be a multiple of 32.
+      MAX_ALLOWED_VALUETYPE = 64,
+
+      // Metadata - This is MDNode or MDString.
+      Metadata       = 250,
+
+      // iPTRAny - An int value the size of the pointer of the current
+      // target to any address space. This must only be used internal to
+      // tblgen. Other than for overloading, we treat iPTRAny the same as iPTR.
+      iPTRAny        = 251,
+
+      // vAny - A vector with any length and element size. This is used
+      // for intrinsics that have overloadings based on vector types.
+      // This is only for tblgen's consumption!
+      vAny           = 252,
+
+      // fAny - Any floating-point or vector floating-point value. This is used
+      // for intrinsics that have overloadings based on floating-point types.
+      // This is only for tblgen's consumption!
+      fAny           = 253,
+
+      // iAny - An integer or vector integer value of any bit width. This is
+      // used for intrinsics that have overloadings based on integer bit widths.
+      // This is only for tblgen's consumption!
+      iAny           = 254,
+
+      // iPTR - An int value the size of the pointer of the current
+      // target.  This should only be used internal to tblgen!
+      iPTR           = 255
+    };
+
+    SimpleValueType SimpleTy;
+
+    MVT() : SimpleTy((SimpleValueType)(INVALID_SIMPLE_VALUE_TYPE)) {}
+    MVT(SimpleValueType SVT) : SimpleTy(SVT) { }
+
+    bool operator>(const MVT& S)  const { return SimpleTy >  S.SimpleTy; }
+    bool operator<(const MVT& S)  const { return SimpleTy <  S.SimpleTy; }
+    bool operator==(const MVT& S) const { return SimpleTy == S.SimpleTy; }
+    bool operator!=(const MVT& S) const { return SimpleTy != S.SimpleTy; }
+    bool operator>=(const MVT& S) const { return SimpleTy >= S.SimpleTy; }
+    bool operator<=(const MVT& S) const { return SimpleTy <= S.SimpleTy; }
+
+    /// isFloatingPoint - Return true if this is a FP, or a vector FP type.
+    bool isFloatingPoint() const {
+      return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE &&
+               SimpleTy <= MVT::LAST_FP_VALUETYPE) ||
+              (SimpleTy >= MVT::FIRST_FP_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_FP_VECTOR_VALUETYPE));
+    }
+
+    /// isInteger - Return true if this is an integer, or a vector integer type.
+    bool isInteger() const {
+      return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
+               SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) ||
+              (SimpleTy >= MVT::FIRST_INTEGER_VECTOR_VALUETYPE &&
+               SimpleTy <= MVT::LAST_INTEGER_VECTOR_VALUETYPE));
+    }
+
+    /// isVector - Return true if this is a vector value type.
+    bool isVector() const {
+      return (SimpleTy >= MVT::FIRST_VECTOR_VALUETYPE &&
+              SimpleTy <= MVT::LAST_VECTOR_VALUETYPE);
+    }
+
+    /// is16BitVector - Return true if this is a 16-bit vector type.
+    bool is16BitVector() const {
+      return (SimpleTy == MVT::v2i8  || SimpleTy == MVT::v1i16 ||
+              SimpleTy == MVT::v16i1);
+    }
+
+    /// is32BitVector - Return true if this is a 32-bit vector type.
+    bool is32BitVector() const {
+      return (SimpleTy == MVT::v4i8  || SimpleTy == MVT::v2i16 ||
+              SimpleTy == MVT::v1i32);
+    }
+
+    /// is64BitVector - Return true if this is a 64-bit vector type.
+    bool is64BitVector() const {
+      return (SimpleTy == MVT::v8i8  || SimpleTy == MVT::v4i16 ||
+              SimpleTy == MVT::v2i32 || SimpleTy == MVT::v1i64 ||
+              SimpleTy == MVT::v1f64 || SimpleTy == MVT::v2f32);
+    }
+
+    /// is128BitVector - Return true if this is a 128-bit vector type.
+    bool is128BitVector() const {
+      return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 ||
+              SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 ||
+              SimpleTy == MVT::v4f32 || SimpleTy == MVT::v2f64);
+    }
+
+    /// is256BitVector - Return true if this is a 256-bit vector type.
+    bool is256BitVector() const {
+      return (SimpleTy == MVT::v8f32 || SimpleTy == MVT::v4f64  ||
+              SimpleTy == MVT::v32i8 || SimpleTy == MVT::v16i16 ||
+              SimpleTy == MVT::v8i32 || SimpleTy == MVT::v4i64);
+    }
+
+    /// is512BitVector - Return true if this is a 512-bit vector type.
+    bool is512BitVector() const {
+      return (SimpleTy == MVT::v8f64 || SimpleTy == MVT::v16f32 ||
+              SimpleTy == MVT::v64i8 || SimpleTy == MVT::v32i16 ||
+              SimpleTy == MVT::v8i64 || SimpleTy == MVT::v16i32);
+    }
+
+    /// is1024BitVector - Return true if this is a 1024-bit vector type.
+    bool is1024BitVector() const {
+      return (SimpleTy == MVT::v16i64);
+    }
+
+    /// isOverloaded - Return true if this is an overloaded type for TableGen.
+    bool isOverloaded() const {
+      return (SimpleTy==MVT::iAny || SimpleTy==MVT::fAny ||
+              SimpleTy==MVT::vAny || SimpleTy==MVT::iPTRAny);
+    }
+
+    /// isPow2VectorType - Returns true if the given vector is a power of 2.
+    bool isPow2VectorType() const {
+      unsigned NElts = getVectorNumElements();
+      return !(NElts & (NElts - 1));
+    }
+
+    /// getPow2VectorType - Widens the length of the given vector MVT up to
+    /// the nearest power of 2 and returns that type.
+    MVT getPow2VectorType() const {
+      if (isPow2VectorType())
+        return *this;
+
+      unsigned NElts = getVectorNumElements();
+      unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts);
+      return MVT::getVectorVT(getVectorElementType(), Pow2NElts);
+    }
+
+    /// getScalarType - If this is a vector type, return the element type,
+    /// otherwise return this.
+    MVT getScalarType() const {
+      return isVector() ? getVectorElementType() : *this;
+    }
+
+    MVT getVectorElementType() const {
+      switch (SimpleTy) {
+      default:
+        llvm_unreachable("Not a vector MVT!");
+      case v2i1 :
+      case v4i1 :
+      case v8i1 :
+      case v16i1 :
+      case v32i1 :
+      case v64i1: return i1;
+      case v1i8 :
+      case v2i8 :
+      case v4i8 :
+      case v8i8 :
+      case v16i8:
+      case v32i8:
+      case v64i8: return i8;
+      case v1i16:
+      case v2i16:
+      case v4i16:
+      case v8i16:
+      case v16i16:
+      case v32i16: return i16;
+      case v1i32:
+      case v2i32:
+      case v4i32:
+      case v8i32:
+      case v16i32: return i32;
+      case v1i64:
+      case v2i64:
+      case v4i64:
+      case v8i64:
+      case v16i64: return i64;
+      case v2f16:
+      case v4f16:
+      case v8f16: return f16;
+      case v1f32:
+      case v2f32:
+      case v4f32:
+      case v8f32:
+      case v16f32: return f32;
+      case v1f64:
+      case v2f64:
+      case v4f64:
+      case v8f64: return f64;
+      }
+    }
+
+    unsigned getVectorNumElements() const {
+      switch (SimpleTy) {
+      default:
+        llvm_unreachable("Not a vector MVT!");
+      case v32i1:
+      case v32i8:
+      case v32i16: return 32;
+      case v64i1:
+      case v64i8: return 64;
+      case v16i1:
+      case v16i8:
+      case v16i16:
+      case v16i32:
+      case v16i64:
+      case v16f32: return 16;
+      case v8i1 :
+      case v8i8 :
+      case v8i16:
+      case v8i32:
+      case v8i64:
+      case v8f16:
+      case v8f32:
+      case v8f64: return 8;
+      case v4i1:
+      case v4i8:
+      case v4i16:
+      case v4i32:
+      case v4i64:
+      case v4f16:
+      case v4f32:
+      case v4f64: return 4;
+      case v2i1:
+      case v2i8:
+      case v2i16:
+      case v2i32:
+      case v2i64:
+      case v2f16:
+      case v2f32:
+      case v2f64: return 2;
+      case v1i8:
+      case v1i16:
+      case v1i32:
+      case v1i64:
+      case v1f32:
+      case v1f64: return 1;
+      }
+    }
+
+    unsigned getSizeInBits() const {
+      switch (SimpleTy) {
+      default:
+        llvm_unreachable("getSizeInBits called on extended MVT.");
+      case Other:
+        llvm_unreachable("Value type is non-standard value, Other.");
+      case iPTR:
+        llvm_unreachable("Value type size is target-dependent. Ask TLI.");
+      case iPTRAny:
+      case iAny:
+      case fAny:
+      case vAny:
+        llvm_unreachable("Value type is overloaded.");
+      case Metadata:
+        llvm_unreachable("Value type is metadata.");
+      case i1  :  return 1;
+      case v2i1:  return 2;
+      case v4i1:  return 4;
+      case i8  :
+      case v1i8:
+      case v8i1: return 8;
+      case i16 :
+      case f16:
+      case v16i1:
+      case v2i8:
+      case v1i16: return 16;
+      case f32 :
+      case i32 :
+      case v32i1:
+      case v4i8:
+      case v2i16:
+      case v2f16:
+      case v1f32:
+      case v1i32: return 32;
+      case x86mmx:
+      case f64 :
+      case i64 :
+      case v64i1:
+      case v8i8:
+      case v4i16:
+      case v2i32:
+      case v1i64:
+      case v4f16:
+      case v2f32:
+      case v1f64: return 64;
+      case f80 :  return 80;
+      case f128:
+      case ppcf128:
+      case i128:
+      case v16i8:
+      case v8i16:
+      case v4i32:
+      case v2i64:
+      case v8f16:
+      case v4f32:
+      case v2f64: return 128;
+      case v32i8:
+      case v16i16:
+      case v8i32:
+      case v4i64:
+      case v8f32:
+      case v4f64: return 256;
+      case v64i8:
+      case v32i16:
+      case v16i32:
+      case v8i64:
+      case v16f32:
+      case v8f64: return 512;
+      case v16i64:return 1024;
+      }
+    }
+
+    unsigned getScalarSizeInBits() const {
+      return getScalarType().getSizeInBits();
+    }
+
+    /// getStoreSize - Return the number of bytes overwritten by a store
+    /// of the specified value type.
+    unsigned getStoreSize() const {
+      return (getSizeInBits() + 7) / 8;
+    }
+
+    /// getStoreSizeInBits - Return the number of bits overwritten by a store
+    /// of the specified value type.
+    unsigned getStoreSizeInBits() const {
+      return getStoreSize() * 8;
+    }
+
+    /// Return true if this has more bits than VT.
+    bool bitsGT(MVT VT) const {
+      return getSizeInBits() > VT.getSizeInBits();
+    }
+
+    /// Return true if this has no less bits than VT.
+    bool bitsGE(MVT VT) const {
+      return getSizeInBits() >= VT.getSizeInBits();
+    }
+
+    /// Return true if this has less bits than VT.
+    bool bitsLT(MVT VT) const {
+      return getSizeInBits() < VT.getSizeInBits();
+    }
+
+    /// Return true if this has no more bits than VT.
+    bool bitsLE(MVT VT) const {
+      return getSizeInBits() <= VT.getSizeInBits();
+    }
+
+
+    static MVT getFloatingPointVT(unsigned BitWidth) {
+      switch (BitWidth) {
+      default:
+        llvm_unreachable("Bad bit width!");
+      case 16:
+        return MVT::f16;
+      case 32:
+        return MVT::f32;
+      case 64:
+        return MVT::f64;
+      case 80:
+        return MVT::f80;
+      case 128:
+        return MVT::f128;
+      }
+    }
+
+    static MVT getIntegerVT(unsigned BitWidth) {
+      switch (BitWidth) {
+      default:
+        return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
+      case 1:
+        return MVT::i1;
+      case 8:
+        return MVT::i8;
+      case 16:
+        return MVT::i16;
+      case 32:
+        return MVT::i32;
+      case 64:
+        return MVT::i64;
+      case 128:
+        return MVT::i128;
+      }
+    }
+
+    static MVT getVectorVT(MVT VT, unsigned NumElements) {
+      switch (VT.SimpleTy) {
+      default:
+        break;
+      case MVT::i1:
+        if (NumElements == 2)  return MVT::v2i1;
+        if (NumElements == 4)  return MVT::v4i1;
+        if (NumElements == 8)  return MVT::v8i1;
+        if (NumElements == 16) return MVT::v16i1;
+        if (NumElements == 32) return MVT::v32i1;
+        if (NumElements == 64) return MVT::v64i1;
+        break;
+      case MVT::i8:
+        if (NumElements == 1)  return MVT::v1i8;
+        if (NumElements == 2)  return MVT::v2i8;
+        if (NumElements == 4)  return MVT::v4i8;
+        if (NumElements == 8)  return MVT::v8i8;
+        if (NumElements == 16) return MVT::v16i8;
+        if (NumElements == 32) return MVT::v32i8;
+        if (NumElements == 64) return MVT::v64i8;
+        break;
+      case MVT::i16:
+        if (NumElements == 1)  return MVT::v1i16;
+        if (NumElements == 2)  return MVT::v2i16;
+        if (NumElements == 4)  return MVT::v4i16;
+        if (NumElements == 8)  return MVT::v8i16;
+        if (NumElements == 16) return MVT::v16i16;
+        if (NumElements == 32) return MVT::v32i16;
+        break;
+      case MVT::i32:
+        if (NumElements == 1)  return MVT::v1i32;
+        if (NumElements == 2)  return MVT::v2i32;
+        if (NumElements == 4)  return MVT::v4i32;
+        if (NumElements == 8)  return MVT::v8i32;
+        if (NumElements == 16) return MVT::v16i32;
+        break;
+      case MVT::i64:
+        if (NumElements == 1)  return MVT::v1i64;
+        if (NumElements == 2)  return MVT::v2i64;
+        if (NumElements == 4)  return MVT::v4i64;
+        if (NumElements == 8)  return MVT::v8i64;
+        if (NumElements == 16) return MVT::v16i64;
+        break;
+      case MVT::f16:
+        if (NumElements == 2)  return MVT::v2f16;
+        if (NumElements == 4)  return MVT::v4f16;
+        if (NumElements == 8)  return MVT::v8f16;
+        break;
+      case MVT::f32:
+        if (NumElements == 1)  return MVT::v1f32;
+        if (NumElements == 2)  return MVT::v2f32;
+        if (NumElements == 4)  return MVT::v4f32;
+        if (NumElements == 8)  return MVT::v8f32;
+        if (NumElements == 16) return MVT::v16f32;
+        break;
+      case MVT::f64:
+        if (NumElements == 1)  return MVT::v1f64;
+        if (NumElements == 2)  return MVT::v2f64;
+        if (NumElements == 4)  return MVT::v4f64;
+        if (NumElements == 8)  return MVT::v8f64;
+        break;
+      }
+      return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
+    }
+
+    /// Return the value type corresponding to the specified type.  This returns
+    /// all pointers as iPTR.  If HandleUnknown is true, unknown types are
+    /// returned as Other, otherwise they are invalid.
+    static MVT getVT(Type *Ty, bool HandleUnknown = false);
+
+  };
+
+} // End llvm namespace
+
+#endif
diff --git a/include/llvm/CodeGen/PBQP/CostAllocator.h b/include/llvm/CodeGen/PBQP/CostAllocator.h
new file mode 100644
index 000000000000..ff62c0959344
--- /dev/null
+++ b/include/llvm/CodeGen/PBQP/CostAllocator.h
@@ -0,0 +1,147 @@
+//===---------- CostAllocator.h - PBQP Cost Allocator -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines classes conforming to the PBQP cost value manager concept.
+//
+// Cost value managers are memory managers for PBQP cost values (vectors and
+// matrices). Since PBQP graphs can grow very large (E.g. hundreds of thousands
+// of edges on the largest function in SPEC2006).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_COSTALLOCATOR_H
+#define LLVM_COSTALLOCATOR_H
+
+#include <set>
+#include <type_traits>
+
+namespace PBQP {
+
+template <typename CostT,
+          typename CostKeyTComparator>
+class CostPool {
+public:
+
+  class PoolEntry {
+  public:
+    template <typename CostKeyT>
+    PoolEntry(CostPool &pool, CostKeyT cost)
+      : pool(pool), cost(std::move(cost)), refCount(0) {}
+    ~PoolEntry() { pool.removeEntry(this); }
+    void incRef() { ++refCount; }
+    bool decRef() { --refCount; return (refCount == 0); }
+    CostT& getCost() { return cost; }
+    const CostT& getCost() const { return cost; }
+  private:
+    CostPool &pool;
+    CostT cost;
+    std::size_t refCount;
+  };
+
+  class PoolRef {
+  public:
+    PoolRef(PoolEntry *entry) : entry(entry) {
+      this->entry->incRef();
+    }
+    PoolRef(const PoolRef &r) {
+      entry = r.entry;
+      entry->incRef();
+    }
+    PoolRef& operator=(const PoolRef &r) {
+      assert(entry != nullptr && "entry should not be null.");
+      PoolEntry *temp = r.entry;
+      temp->incRef();
+      entry->decRef();
+      entry = temp;
+      return *this;
+    }
+
+    ~PoolRef() {
+      if (entry->decRef())
+        delete entry;
+    }
+    void reset(PoolEntry *entry) {
+      entry->incRef();
+      this->entry->decRef();
+      this->entry = entry;
+    }
+    CostT& operator*() { return entry->getCost(); }
+    const CostT& operator*() const { return entry->getCost(); }
+    CostT* operator->() { return &entry->getCost(); }
+    const CostT* operator->() const { return &entry->getCost(); }
+  private:
+    PoolEntry *entry;
+  };
+
+private:
+  class EntryComparator {
+  public:
+    template <typename CostKeyT>
+    typename std::enable_if<
+               !std::is_same<PoolEntry*,
+                             typename std::remove_const<CostKeyT>::type>::value,
+               bool>::type
+    operator()(const PoolEntry* a, const CostKeyT &b) {
+      return compare(a->getCost(), b);
+    }
+    bool operator()(const PoolEntry* a, const PoolEntry* b) {
+      return compare(a->getCost(), b->getCost());
+    }
+  private:
+    CostKeyTComparator compare;
+  };
+
+  typedef std::set<PoolEntry*, EntryComparator> EntrySet;
+
+  EntrySet entrySet;
+
+  void removeEntry(PoolEntry *p) { entrySet.erase(p); }
+
+public:
+
+  template <typename CostKeyT>
+  PoolRef getCost(CostKeyT costKey) {
+    typename EntrySet::iterator itr =
+      std::lower_bound(entrySet.begin(), entrySet.end(), costKey,
+                       EntryComparator());
+
+    if (itr != entrySet.end() && costKey == (*itr)->getCost())
+      return PoolRef(*itr);
+
+    PoolEntry *p = new PoolEntry(*this, std::move(costKey));
+    entrySet.insert(itr, p);
+    return PoolRef(p);
+  }
+};
+
+template <typename VectorT, typename VectorTComparator,
+          typename MatrixT, typename MatrixTComparator>
+class PoolCostAllocator {
+private:
+  typedef CostPool<VectorT, VectorTComparator> VectorCostPool;
+  typedef CostPool<MatrixT, MatrixTComparator> MatrixCostPool;
+public:
+  typedef VectorT Vector;
+  typedef MatrixT Matrix;
+  typedef typename VectorCostPool::PoolRef VectorPtr;
+  typedef typename MatrixCostPool::PoolRef MatrixPtr;
+
+  template <typename VectorKeyT>
+  VectorPtr getVector(VectorKeyT v) { return vectorPool.getCost(std::move(v)); }
+
+  template <typename MatrixKeyT>
+  MatrixPtr getMatrix(MatrixKeyT m) { return matrixPool.getCost(std::move(m)); }
+private:
+  VectorCostPool vectorPool;
+  MatrixCostPool matrixPool;
+};
+
+}
+
+#endif // LLVM_COSTALLOCATOR_H
diff --git a/include/llvm/CodeGen/PBQP/Graph.h b/include/llvm/CodeGen/PBQP/Graph.h
index aca0a9130342..a55f0ea96c0a 100644
--- a/include/llvm/CodeGen/PBQP/Graph.h
+++ b/include/llvm/CodeGen/PBQP/Graph.h
@@ -15,464 +15,628 @@
 #ifndef LLVM_CODEGEN_PBQP_GRAPH_H
 #define LLVM_CODEGEN_PBQP_GRAPH_H
 
-#include "Math.h"
 #include "llvm/ADT/ilist.h"
 #include "llvm/ADT/ilist_node.h"
+#include "llvm/Support/Compiler.h"
 #include <list>
 #include <map>
 #include <set>
 
 namespace PBQP {
 
-  /// PBQP Graph class.
-  /// Instances of this class describe PBQP problems.
-  class Graph {
+  class GraphBase {
   public:
-
     typedef unsigned NodeId;
     typedef unsigned EdgeId;
 
-  private:
+    /// \brief Returns a value representing an invalid (non-existent) node.
+    static NodeId invalidNodeId() {
+      return std::numeric_limits<NodeId>::max();
+    }
 
-    typedef std::set<NodeId> AdjEdgeList;
+    /// \brief Returns a value representing an invalid (non-existent) edge.
+    static EdgeId invalidEdgeId() {
+      return std::numeric_limits<EdgeId>::max();
+    }
+  };
 
+  /// PBQP Graph class.
+  /// Instances of this class describe PBQP problems.
+  ///
+  template <typename SolverT>
+  class Graph : public GraphBase {
+  private:
+    typedef typename SolverT::CostAllocator CostAllocator;
   public:
-
-    typedef AdjEdgeList::iterator AdjEdgeItr;
+    typedef typename SolverT::RawVector RawVector;
+    typedef typename SolverT::RawMatrix RawMatrix;
+    typedef typename SolverT::Vector Vector;
+    typedef typename SolverT::Matrix Matrix;
+    typedef typename CostAllocator::VectorPtr VectorPtr;
+    typedef typename CostAllocator::MatrixPtr MatrixPtr;
+    typedef typename SolverT::NodeMetadata NodeMetadata;
+    typedef typename SolverT::EdgeMetadata EdgeMetadata;
 
   private:
 
     class NodeEntry {
-    private:
-      Vector costs;
-      AdjEdgeList adjEdges;
-      void *data;
-      NodeEntry() : costs(0, 0) {}
     public:
-      NodeEntry(const Vector &costs) : costs(costs), data(0) {}
-      Vector& getCosts() { return costs; }
-      const Vector& getCosts() const { return costs; }
-      unsigned getDegree() const { return adjEdges.size(); }
-      AdjEdgeItr edgesBegin() { return adjEdges.begin(); }
-      AdjEdgeItr edgesEnd() { return adjEdges.end(); }
-      AdjEdgeItr addEdge(EdgeId e) {
-        return adjEdges.insert(adjEdges.end(), e);
+      typedef std::vector<EdgeId> AdjEdgeList;
+      typedef AdjEdgeList::size_type AdjEdgeIdx;
+      typedef AdjEdgeList::const_iterator AdjEdgeItr;
+
+      static AdjEdgeIdx getInvalidAdjEdgeIdx() {
+        return std::numeric_limits<AdjEdgeIdx>::max();
       }
-      void removeEdge(AdjEdgeItr ae) {
-        adjEdges.erase(ae);
+
+      NodeEntry(VectorPtr Costs) : Costs(Costs) {}
+
+      AdjEdgeIdx addAdjEdgeId(EdgeId EId) {
+        AdjEdgeIdx Idx = AdjEdgeIds.size();
+        AdjEdgeIds.push_back(EId);
+        return Idx;
+      }
+
+      void removeAdjEdgeId(Graph &G, NodeId ThisNId, AdjEdgeIdx Idx) {
+        // Swap-and-pop for fast removal.
+        //   1) Update the adj index of the edge currently at back().
+        //   2) Move last Edge down to Idx.
+        //   3) pop_back()
+        // If Idx == size() - 1 then the setAdjEdgeIdx and swap are
+        // redundant, but both operations are cheap.
+        G.getEdge(AdjEdgeIds.back()).setAdjEdgeIdx(ThisNId, Idx);
+        AdjEdgeIds[Idx] = AdjEdgeIds.back();
+        AdjEdgeIds.pop_back();
       }
-      void setData(void *data) { this->data = data; }
-      void* getData() { return data; }
+
+      const AdjEdgeList& getAdjEdgeIds() const { return AdjEdgeIds; }
+
+      VectorPtr Costs;
+      NodeMetadata Metadata;
+    private:
+      AdjEdgeList AdjEdgeIds;
     };
 
     class EdgeEntry {
-    private:
-      NodeId node1, node2;
-      Matrix costs;
-      AdjEdgeItr node1AEItr, node2AEItr;
-      void *data;
-      EdgeEntry() : costs(0, 0, 0), data(0) {}
     public:
-      EdgeEntry(NodeId node1, NodeId node2, const Matrix &costs)
-        : node1(node1), node2(node2), costs(costs) {}
-      NodeId getNode1() const { return node1; }
-      NodeId getNode2() const { return node2; }
-      Matrix& getCosts() { return costs; }
-      const Matrix& getCosts() const { return costs; }
-      void setNode1AEItr(AdjEdgeItr ae) { node1AEItr = ae; }
-      AdjEdgeItr getNode1AEItr() { return node1AEItr; }
-      void setNode2AEItr(AdjEdgeItr ae) { node2AEItr = ae; }
-      AdjEdgeItr getNode2AEItr() { return node2AEItr; }
-      void setData(void *data) { this->data = data; }
-      void *getData() { return data; }
+      EdgeEntry(NodeId N1Id, NodeId N2Id, MatrixPtr Costs)
+        : Costs(Costs) {
+        NIds[0] = N1Id;
+        NIds[1] = N2Id;
+        ThisEdgeAdjIdxs[0] = NodeEntry::getInvalidAdjEdgeIdx();
+        ThisEdgeAdjIdxs[1] = NodeEntry::getInvalidAdjEdgeIdx();
+      }
+
+      void invalidate() {
+        NIds[0] = NIds[1] = Graph::invalidNodeId();
+        ThisEdgeAdjIdxs[0] = ThisEdgeAdjIdxs[1] =
+          NodeEntry::getInvalidAdjEdgeIdx();
+        Costs = nullptr;
+      }
+
+      void connectToN(Graph &G, EdgeId ThisEdgeId, unsigned NIdx) {
+        assert(ThisEdgeAdjIdxs[NIdx] == NodeEntry::getInvalidAdjEdgeIdx() &&
+               "Edge already connected to NIds[NIdx].");
+        NodeEntry &N = G.getNode(NIds[NIdx]);
+        ThisEdgeAdjIdxs[NIdx] = N.addAdjEdgeId(ThisEdgeId);
+      }
+
+      void connectTo(Graph &G, EdgeId ThisEdgeId, NodeId NId) {
+        if (NId == NIds[0])
+          connectToN(G, ThisEdgeId, 0);
+        else {
+          assert(NId == NIds[1] && "Edge does not connect NId.");
+          connectToN(G, ThisEdgeId, 1);
+        }
+      }
+
+      void connect(Graph &G, EdgeId ThisEdgeId) {
+        connectToN(G, ThisEdgeId, 0);
+        connectToN(G, ThisEdgeId, 1);
+      }
+
+      void setAdjEdgeIdx(NodeId NId, typename NodeEntry::AdjEdgeIdx NewIdx) {
+        if (NId == NIds[0])
+          ThisEdgeAdjIdxs[0] = NewIdx;
+        else {
+          assert(NId == NIds[1] && "Edge not connected to NId");
+          ThisEdgeAdjIdxs[1] = NewIdx;
+        }
+      }
+
+      void disconnectFromN(Graph &G, unsigned NIdx) {
+        assert(ThisEdgeAdjIdxs[NIdx] != NodeEntry::getInvalidAdjEdgeIdx() &&
+               "Edge not connected to NIds[NIdx].");
+        NodeEntry &N = G.getNode(NIds[NIdx]);
+        N.removeAdjEdgeId(G, NIds[NIdx], ThisEdgeAdjIdxs[NIdx]);
+        ThisEdgeAdjIdxs[NIdx] = NodeEntry::getInvalidAdjEdgeIdx();
+      }
+
+      void disconnectFrom(Graph &G, NodeId NId) {
+        if (NId == NIds[0])
+          disconnectFromN(G, 0);
+        else {
+          assert(NId == NIds[1] && "Edge does not connect NId");
+          disconnectFromN(G, 1);
+        }
+      }
+
+      NodeId getN1Id() const { return NIds[0]; }
+      NodeId getN2Id() const { return NIds[1]; }
+      MatrixPtr Costs;
+      EdgeMetadata Metadata;
+    private:
+      NodeId NIds[2];
+      typename NodeEntry::AdjEdgeIdx ThisEdgeAdjIdxs[2];
     };
 
     // ----- MEMBERS -----
 
+    CostAllocator CostAlloc;
+    SolverT *Solver;
+
     typedef std::vector<NodeEntry> NodeVector;
     typedef std::vector<NodeId> FreeNodeVector;
-    NodeVector nodes;
-    FreeNodeVector freeNodes;
+    NodeVector Nodes;
+    FreeNodeVector FreeNodeIds;
 
     typedef std::vector<EdgeEntry> EdgeVector;
     typedef std::vector<EdgeId> FreeEdgeVector;
-    EdgeVector edges;
-    FreeEdgeVector freeEdges;
+    EdgeVector Edges;
+    FreeEdgeVector FreeEdgeIds;
 
     // ----- INTERNAL METHODS -----
 
-    NodeEntry& getNode(NodeId nId) { return nodes[nId]; }
-    const NodeEntry& getNode(NodeId nId) const { return nodes[nId]; }
+    NodeEntry& getNode(NodeId NId) { return Nodes[NId]; }
+    const NodeEntry& getNode(NodeId NId) const { return Nodes[NId]; }
 
-    EdgeEntry& getEdge(EdgeId eId) { return edges[eId]; }
-    const EdgeEntry& getEdge(EdgeId eId) const { return edges[eId]; }
+    EdgeEntry& getEdge(EdgeId EId) { return Edges[EId]; }
+    const EdgeEntry& getEdge(EdgeId EId) const { return Edges[EId]; }
 
-    NodeId addConstructedNode(const NodeEntry &n) {
-      NodeId nodeId = 0;
-      if (!freeNodes.empty()) {
-        nodeId = freeNodes.back();
-        freeNodes.pop_back();
-        nodes[nodeId] = n;
+    NodeId addConstructedNode(const NodeEntry &N) {
+      NodeId NId = 0;
+      if (!FreeNodeIds.empty()) {
+        NId = FreeNodeIds.back();
+        FreeNodeIds.pop_back();
+        Nodes[NId] = std::move(N);
       } else {
-        nodeId = nodes.size();
-        nodes.push_back(n);
+        NId = Nodes.size();
+        Nodes.push_back(std::move(N));
       }
-      return nodeId;
+      return NId;
     }
 
-    EdgeId addConstructedEdge(const EdgeEntry &e) {
-      assert(findEdge(e.getNode1(), e.getNode2()) == invalidEdgeId() &&
+    EdgeId addConstructedEdge(const EdgeEntry &E) {
+      assert(findEdge(E.getN1Id(), E.getN2Id()) == invalidEdgeId() &&
              "Attempt to add duplicate edge.");
-      EdgeId edgeId = 0;
-      if (!freeEdges.empty()) {
-        edgeId = freeEdges.back();
-        freeEdges.pop_back();
-        edges[edgeId] = e;
+      EdgeId EId = 0;
+      if (!FreeEdgeIds.empty()) {
+        EId = FreeEdgeIds.back();
+        FreeEdgeIds.pop_back();
+        Edges[EId] = std::move(E);
       } else {
-        edgeId = edges.size();
-        edges.push_back(e);
+        EId = Edges.size();
+        Edges.push_back(std::move(E));
       }
 
-      EdgeEntry &ne = getEdge(edgeId);
-      NodeEntry &n1 = getNode(ne.getNode1());
-      NodeEntry &n2 = getNode(ne.getNode2());
+      EdgeEntry &NE = getEdge(EId);
 
-      // Sanity check on matrix dimensions:
-      assert((n1.getCosts().getLength() == ne.getCosts().getRows()) &&
-             (n2.getCosts().getLength() == ne.getCosts().getCols()) &&
-             "Edge cost dimensions do not match node costs dimensions.");
-
-      ne.setNode1AEItr(n1.addEdge(edgeId));
-      ne.setNode2AEItr(n2.addEdge(edgeId));
-      return edgeId;
+      // Add the edge to the adjacency sets of its nodes.
+      NE.connect(*this, EId);
+      return EId;
     }
 
-    Graph(const Graph &other) {}
-    void operator=(const Graph &other) {}
+    Graph(const Graph &Other) {}
+    void operator=(const Graph &Other) {}
 
   public:
 
+    typedef typename NodeEntry::AdjEdgeItr AdjEdgeItr;
+
     class NodeItr {
     public:
-      NodeItr(NodeId nodeId, const Graph &g)
-        : nodeId(nodeId), endNodeId(g.nodes.size()), freeNodes(g.freeNodes) {
-        this->nodeId = findNextInUse(nodeId); // Move to the first in-use nodeId
+      NodeItr(NodeId CurNId, const Graph &G)
+        : CurNId(CurNId), EndNId(G.Nodes.size()), FreeNodeIds(G.FreeNodeIds) {
+        this->CurNId = findNextInUse(CurNId); // Move to first in-use node id
       }
 
-      bool operator==(const NodeItr& n) const { return nodeId == n.nodeId; }
-      bool operator!=(const NodeItr& n) const { return !(*this == n); }
-      NodeItr& operator++() { nodeId = findNextInUse(++nodeId); return *this; }
-      NodeId operator*() const { return nodeId; }
+      bool operator==(const NodeItr &O) const { return CurNId == O.CurNId; }
+      bool operator!=(const NodeItr &O) const { return !(*this == O); }
+      NodeItr& operator++() { CurNId = findNextInUse(++CurNId); return *this; }
+      NodeId operator*() const { return CurNId; }
 
     private:
-      NodeId findNextInUse(NodeId n) const {
-        while (n < endNodeId &&
-               std::find(freeNodes.begin(), freeNodes.end(), n) !=
-                 freeNodes.end()) {
-          ++n;
+      NodeId findNextInUse(NodeId NId) const {
+        while (NId < EndNId &&
+               std::find(FreeNodeIds.begin(), FreeNodeIds.end(), NId) !=
+                 FreeNodeIds.end()) {
+          ++NId;
         }
-        return n;
+        return NId;
       }
 
-      NodeId nodeId, endNodeId;
-      const FreeNodeVector& freeNodes;
+      NodeId CurNId, EndNId;
+      const FreeNodeVector &FreeNodeIds;
     };
 
     class EdgeItr {
     public:
-      EdgeItr(EdgeId edgeId, const Graph &g)
-        : edgeId(edgeId), endEdgeId(g.edges.size()), freeEdges(g.freeEdges) {
-        this->edgeId = findNextInUse(edgeId); // Move to the first in-use edgeId
+      EdgeItr(EdgeId CurEId, const Graph &G)
+        : CurEId(CurEId), EndEId(G.Edges.size()), FreeEdgeIds(G.FreeEdgeIds) {
+        this->CurEId = findNextInUse(CurEId); // Move to first in-use edge id
       }
 
-      bool operator==(const EdgeItr& n) const { return edgeId == n.edgeId; }
-      bool operator!=(const EdgeItr& n) const { return !(*this == n); }
-      EdgeItr& operator++() { edgeId = findNextInUse(++edgeId); return *this; }
-      EdgeId operator*() const { return edgeId; }
+      bool operator==(const EdgeItr &O) const { return CurEId == O.CurEId; }
+      bool operator!=(const EdgeItr &O) const { return !(*this == O); }
+      EdgeItr& operator++() { CurEId = findNextInUse(++CurEId); return *this; }
+      EdgeId operator*() const { return CurEId; }
 
     private:
-      EdgeId findNextInUse(EdgeId n) const {
-        while (n < endEdgeId &&
-               std::find(freeEdges.begin(), freeEdges.end(), n) !=
-                 freeEdges.end()) {
-          ++n;
+      EdgeId findNextInUse(EdgeId EId) const {
+        while (EId < EndEId &&
+               std::find(FreeEdgeIds.begin(), FreeEdgeIds.end(), EId) !=
+               FreeEdgeIds.end()) {
+          ++EId;
         }
-        return n;
+        return EId;
+      }
+
+      EdgeId CurEId, EndEId;
+      const FreeEdgeVector &FreeEdgeIds;
+    };
+
+    class NodeIdSet {
+    public:
+      NodeIdSet(const Graph &G) : G(G) { }
+      NodeItr begin() const { return NodeItr(0, G); }
+      NodeItr end() const { return NodeItr(G.Nodes.size(), G); }
+      bool empty() const { return G.Nodes.empty(); }
+      typename NodeVector::size_type size() const {
+        return G.Nodes.size() - G.FreeNodeIds.size();
       }
+    private:
+      const Graph& G;
+    };
+
+    class EdgeIdSet {
+    public:
+      EdgeIdSet(const Graph &G) : G(G) { }
+      EdgeItr begin() const { return EdgeItr(0, G); }
+      EdgeItr end() const { return EdgeItr(G.Edges.size(), G); }
+      bool empty() const { return G.Edges.empty(); }
+      typename NodeVector::size_type size() const {
+        return G.Edges.size() - G.FreeEdgeIds.size();
+      }
+    private:
+      const Graph& G;
+    };
 
-      EdgeId edgeId, endEdgeId;
-      const FreeEdgeVector& freeEdges;
+    class AdjEdgeIdSet {
+    public:
+      AdjEdgeIdSet(const NodeEntry &NE) : NE(NE) { }
+      typename NodeEntry::AdjEdgeItr begin() const {
+        return NE.getAdjEdgeIds().begin();
+      }
+      typename NodeEntry::AdjEdgeItr end() const {
+        return NE.getAdjEdgeIds().end();
+      }
+      bool empty() const { return NE.getAdjEdgeIds().empty(); }
+      typename NodeEntry::AdjEdgeList::size_type size() const {
+        return NE.getAdjEdgeIds().size();
+      }
+    private:
+      const NodeEntry &NE;
     };
 
     /// \brief Construct an empty PBQP graph.
-    Graph() {}
+    Graph() : Solver(nullptr) { }
+
+    /// \brief Lock this graph to the given solver instance in preparation
+    /// for running the solver. This method will call solver.handleAddNode for
+    /// each node in the graph, and handleAddEdge for each edge, to give the
+    /// solver an opportunity to set up any requried metadata.
+    void setSolver(SolverT &S) {
+      assert(!Solver && "Solver already set. Call unsetSolver().");
+      Solver = &S;
+      for (auto NId : nodeIds())
+        Solver->handleAddNode(NId);
+      for (auto EId : edgeIds())
+        Solver->handleAddEdge(EId);
+    }
+
+    /// \brief Release from solver instance.
+    void unsetSolver() {
+      assert(Solver && "Solver not set.");
+      Solver = nullptr;
+    }
 
     /// \brief Add a node with the given costs.
-    /// @param costs Cost vector for the new node.
+    /// @param Costs Cost vector for the new node.
     /// @return Node iterator for the added node.
-    NodeId addNode(const Vector &costs) {
-      return addConstructedNode(NodeEntry(costs));
+    template <typename OtherVectorT>
+    NodeId addNode(OtherVectorT Costs) {
+      // Get cost vector from the problem domain
+      VectorPtr AllocatedCosts = CostAlloc.getVector(std::move(Costs));
+      NodeId NId = addConstructedNode(NodeEntry(AllocatedCosts));
+      if (Solver)
+        Solver->handleAddNode(NId);
+      return NId;
     }
 
     /// \brief Add an edge between the given nodes with the given costs.
-    /// @param n1Id First node.
-    /// @param n2Id Second node.
+    /// @param N1Id First node.
+    /// @param N2Id Second node.
     /// @return Edge iterator for the added edge.
-    EdgeId addEdge(NodeId n1Id, NodeId n2Id, const Matrix &costs) {
-      assert(getNodeCosts(n1Id).getLength() == costs.getRows() &&
-             getNodeCosts(n2Id).getLength() == costs.getCols() &&
+    template <typename OtherVectorT>
+    EdgeId addEdge(NodeId N1Id, NodeId N2Id, OtherVectorT Costs) {
+      assert(getNodeCosts(N1Id).getLength() == Costs.getRows() &&
+             getNodeCosts(N2Id).getLength() == Costs.getCols() &&
              "Matrix dimensions mismatch.");
-      return addConstructedEdge(EdgeEntry(n1Id, n2Id, costs));
+      // Get cost matrix from the problem domain.
+      MatrixPtr AllocatedCosts = CostAlloc.getMatrix(std::move(Costs));
+      EdgeId EId = addConstructedEdge(EdgeEntry(N1Id, N2Id, AllocatedCosts));
+      if (Solver)
+        Solver->handleAddEdge(EId);
+      return EId;
     }
 
+    /// \brief Returns true if the graph is empty.
+    bool empty() const { return NodeIdSet(*this).empty(); }
+
+    NodeIdSet nodeIds() const { return NodeIdSet(*this); }
+    EdgeIdSet edgeIds() const { return EdgeIdSet(*this); }
+
+    AdjEdgeIdSet adjEdgeIds(NodeId NId) { return AdjEdgeIdSet(getNode(NId)); }
+
     /// \brief Get the number of nodes in the graph.
     /// @return Number of nodes in the graph.
-    unsigned getNumNodes() const { return nodes.size() - freeNodes.size(); }
+    unsigned getNumNodes() const { return NodeIdSet(*this).size(); }
 
     /// \brief Get the number of edges in the graph.
     /// @return Number of edges in the graph.
-    unsigned getNumEdges() const { return edges.size() - freeEdges.size(); }
+    unsigned getNumEdges() const { return EdgeIdSet(*this).size(); }
 
-    /// \brief Get a node's cost vector.
-    /// @param nId Node id.
-    /// @return Node cost vector.
-    Vector& getNodeCosts(NodeId nId) { return getNode(nId).getCosts(); }
+    /// \brief Set a node's cost vector.
+    /// @param NId Node to update.
+    /// @param Costs New costs to set.
+    template <typename OtherVectorT>
+    void setNodeCosts(NodeId NId, OtherVectorT Costs) {
+      VectorPtr AllocatedCosts = CostAlloc.getVector(std::move(Costs));
+      if (Solver)
+        Solver->handleSetNodeCosts(NId, *AllocatedCosts);
+      getNode(NId).Costs = AllocatedCosts;
+    }
 
     /// \brief Get a node's cost vector (const version).
-    /// @param nId Node id.
+    /// @param NId Node id.
     /// @return Node cost vector.
-    const Vector& getNodeCosts(NodeId nId) const {
-      return getNode(nId).getCosts();
+    const Vector& getNodeCosts(NodeId NId) const {
+      return *getNode(NId).Costs;
     }
 
-    /// \brief Set a node's data pointer.
-    /// @param nId Node id.
-    /// @param data Pointer to node data.
-    ///
-    /// Typically used by a PBQP solver to attach data to aid in solution.
-    void setNodeData(NodeId nId, void *data) { getNode(nId).setData(data); }
-
-    /// \brief Get the node's data pointer.
-    /// @param nId Node id.
-    /// @return Pointer to node data.
-    void* getNodeData(NodeId nId) { return getNode(nId).getData(); }
-
-    /// \brief Get an edge's cost matrix.
-    /// @param eId Edge id.
-    /// @return Edge cost matrix.
-    Matrix& getEdgeCosts(EdgeId eId) { return getEdge(eId).getCosts(); }
-
-    /// \brief Get an edge's cost matrix (const version).
-    /// @param eId Edge id.
-    /// @return Edge cost matrix.
-    const Matrix& getEdgeCosts(EdgeId eId) const {
-      return getEdge(eId).getCosts();
+    NodeMetadata& getNodeMetadata(NodeId NId) {
+      return getNode(NId).Metadata;
     }
 
-    /// \brief Set an edge's data pointer.
-    /// @param eId Edge id.
-    /// @param data Pointer to edge data.
-    ///
-    /// Typically used by a PBQP solver to attach data to aid in solution.
-    void setEdgeData(EdgeId eId, void *data) { getEdge(eId).setData(data); }
-
-    /// \brief Get an edge's data pointer.
-    /// @param eId Edge id.
-    /// @return Pointer to edge data.
-    void* getEdgeData(EdgeId eId) { return getEdge(eId).getData(); }
-
-    /// \brief Get a node's degree.
-    /// @param nId Node id.
-    /// @return The degree of the node.
-    unsigned getNodeDegree(NodeId nId) const {
-      return getNode(nId).getDegree();
+    const NodeMetadata& getNodeMetadata(NodeId NId) const {
+      return getNode(NId).Metadata;
     }
 
-    /// \brief Begin iterator for node set.
-    NodeItr nodesBegin() const { return NodeItr(0, *this);  }
-
-    /// \brief End iterator for node set.
-    NodeItr nodesEnd() const { return NodeItr(nodes.size(), *this); }
+    typename NodeEntry::AdjEdgeList::size_type getNodeDegree(NodeId NId) const {
+      return getNode(NId).getAdjEdgeIds().size();
+    }
 
-    /// \brief Begin iterator for edge set.
-    EdgeItr edgesBegin() const { return EdgeItr(0, *this); }
+    /// \brief Set an edge's cost matrix.
+    /// @param EId Edge id.
+    /// @param Costs New cost matrix.
+    template <typename OtherMatrixT>
+    void setEdgeCosts(EdgeId EId, OtherMatrixT Costs) {
+      MatrixPtr AllocatedCosts = CostAlloc.getMatrix(std::move(Costs));
+      if (Solver)
+        Solver->handleSetEdgeCosts(EId, *AllocatedCosts);
+      getEdge(EId).Costs = AllocatedCosts;
+    }
 
-    /// \brief End iterator for edge set.
-    EdgeItr edgesEnd() const { return EdgeItr(edges.size(), *this); }
+    /// \brief Get an edge's cost matrix (const version).
+    /// @param EId Edge id.
+    /// @return Edge cost matrix.
+    const Matrix& getEdgeCosts(EdgeId EId) const { return *getEdge(EId).Costs; }
 
-    /// \brief Get begin iterator for adjacent edge set.
-    /// @param nId Node id.
-    /// @return Begin iterator for the set of edges connected to the given node.
-    AdjEdgeItr adjEdgesBegin(NodeId nId) {
-      return getNode(nId).edgesBegin();
+    EdgeMetadata& getEdgeMetadata(EdgeId NId) {
+      return getEdge(NId).Metadata;
     }
 
-    /// \brief Get end iterator for adjacent edge set.
-    /// @param nId Node id.
-    /// @return End iterator for the set of edges connected to the given node.
-    AdjEdgeItr adjEdgesEnd(NodeId nId) {
-      return getNode(nId).edgesEnd();
+    const EdgeMetadata& getEdgeMetadata(EdgeId NId) const {
+      return getEdge(NId).Metadata;
     }
 
     /// \brief Get the first node connected to this edge.
-    /// @param eId Edge id.
+    /// @param EId Edge id.
     /// @return The first node connected to the given edge.
-    NodeId getEdgeNode1(EdgeId eId) {
-      return getEdge(eId).getNode1();
+    NodeId getEdgeNode1Id(EdgeId EId) {
+      return getEdge(EId).getN1Id();
     }
 
     /// \brief Get the second node connected to this edge.
-    /// @param eId Edge id.
+    /// @param EId Edge id.
     /// @return The second node connected to the given edge.
-    NodeId getEdgeNode2(EdgeId eId) {
-      return getEdge(eId).getNode2();
+    NodeId getEdgeNode2Id(EdgeId EId) {
+      return getEdge(EId).getN2Id();
     }
 
     /// \brief Get the "other" node connected to this edge.
-    /// @param eId Edge id.
-    /// @param nId Node id for the "given" node.
+    /// @param EId Edge id.
+    /// @param NId Node id for the "given" node.
     /// @return The iterator for the "other" node connected to this edge.
-    NodeId getEdgeOtherNode(EdgeId eId, NodeId nId) {
-      EdgeEntry &e = getEdge(eId);
-      if (e.getNode1() == nId) {
-        return e.getNode2();
+    NodeId getEdgeOtherNodeId(EdgeId EId, NodeId NId) {
+      EdgeEntry &E = getEdge(EId);
+      if (E.getN1Id() == NId) {
+        return E.getN2Id();
       } // else
-      return e.getNode1();
-    }
-
-    EdgeId invalidEdgeId() const {
-      return std::numeric_limits<EdgeId>::max();
+      return E.getN1Id();
     }
 
     /// \brief Get the edge connecting two nodes.
-    /// @param n1Id First node id.
-    /// @param n2Id Second node id.
-    /// @return An id for edge (n1Id, n2Id) if such an edge exists,
+    /// @param N1Id First node id.
+    /// @param N2Id Second node id.
+    /// @return An id for edge (N1Id, N2Id) if such an edge exists,
     ///         otherwise returns an invalid edge id.
-    EdgeId findEdge(NodeId n1Id, NodeId n2Id) {
-      for (AdjEdgeItr aeItr = adjEdgesBegin(n1Id), aeEnd = adjEdgesEnd(n1Id);
-         aeItr != aeEnd; ++aeItr) {
-        if ((getEdgeNode1(*aeItr) == n2Id) ||
-            (getEdgeNode2(*aeItr) == n2Id)) {
-          return *aeItr;
+    EdgeId findEdge(NodeId N1Id, NodeId N2Id) {
+      for (auto AEId : adjEdgeIds(N1Id)) {
+        if ((getEdgeNode1Id(AEId) == N2Id) ||
+            (getEdgeNode2Id(AEId) == N2Id)) {
+          return AEId;
         }
       }
       return invalidEdgeId();
     }
 
     /// \brief Remove a node from the graph.
-    /// @param nId Node id.
-    void removeNode(NodeId nId) {
-      NodeEntry &n = getNode(nId);
-      for (AdjEdgeItr itr = n.edgesBegin(), end = n.edgesEnd(); itr != end; ++itr) {
-        EdgeId eId = *itr;
-        removeEdge(eId);
+    /// @param NId Node id.
+    void removeNode(NodeId NId) {
+      if (Solver)
+        Solver->handleRemoveNode(NId);
+      NodeEntry &N = getNode(NId);
+      // TODO: Can this be for-each'd?
+      for (AdjEdgeItr AEItr = N.adjEdgesBegin(),
+                      AEEnd = N.adjEdgesEnd();
+           AEItr != AEEnd;) {
+        EdgeId EId = *AEItr;
+        ++AEItr;
+        removeEdge(EId);
       }
-      freeNodes.push_back(nId);
+      FreeNodeIds.push_back(NId);
+    }
+
+    /// \brief Disconnect an edge from the given node.
+    ///
+    /// Removes the given edge from the adjacency list of the given node.
+    /// This operation leaves the edge in an 'asymmetric' state: It will no
+    /// longer appear in an iteration over the given node's (NId's) edges, but
+    /// will appear in an iteration over the 'other', unnamed node's edges.
+    ///
+    /// This does not correspond to any normal graph operation, but exists to
+    /// support efficient PBQP graph-reduction based solvers. It is used to
+    /// 'effectively' remove the unnamed node from the graph while the solver
+    /// is performing the reduction. The solver will later call reconnectNode
+    /// to restore the edge in the named node's adjacency list.
+    ///
+    /// Since the degree of a node is the number of connected edges,
+    /// disconnecting an edge from a node 'u' will cause the degree of 'u' to
+    /// drop by 1.
+    ///
+    /// A disconnected edge WILL still appear in an iteration over the graph
+    /// edges.
+    ///
+    /// A disconnected edge should not be removed from the graph, it should be
+    /// reconnected first.
+    ///
+    /// A disconnected edge can be reconnected by calling the reconnectEdge
+    /// method.
+    void disconnectEdge(EdgeId EId, NodeId NId) {
+      if (Solver)
+        Solver->handleDisconnectEdge(EId, NId);
+
+      EdgeEntry &E = getEdge(EId);
+      E.disconnectFrom(*this, NId);
+    }
+
+    /// \brief Convenience method to disconnect all neighbours from the given
+    ///        node.
+    void disconnectAllNeighborsFromNode(NodeId NId) {
+      for (auto AEId : adjEdgeIds(NId))
+        disconnectEdge(AEId, getEdgeOtherNodeId(AEId, NId));
+    }
+
+    /// \brief Re-attach an edge to its nodes.
+    ///
+    /// Adds an edge that had been previously disconnected back into the
+    /// adjacency set of the nodes that the edge connects.
+    void reconnectEdge(EdgeId EId, NodeId NId) {
+      EdgeEntry &E = getEdge(EId);
+      E.connectTo(*this, EId, NId);
+      if (Solver)
+        Solver->handleReconnectEdge(EId, NId);
     }
 
     /// \brief Remove an edge from the graph.
-    /// @param eId Edge id.
-    void removeEdge(EdgeId eId) {
-      EdgeEntry &e = getEdge(eId);
-      NodeEntry &n1 = getNode(e.getNode1());
-      NodeEntry &n2 = getNode(e.getNode2());
-      n1.removeEdge(e.getNode1AEItr());
-      n2.removeEdge(e.getNode2AEItr());
-      freeEdges.push_back(eId);
+    /// @param EId Edge id.
+    void removeEdge(EdgeId EId) {
+      if (Solver)
+        Solver->handleRemoveEdge(EId);
+      EdgeEntry &E = getEdge(EId);
+      E.disconnect();
+      FreeEdgeIds.push_back(EId);
+      Edges[EId].invalidate();
     }
 
     /// \brief Remove all nodes and edges from the graph.
     void clear() {
-      nodes.clear();
-      freeNodes.clear();
-      edges.clear();
-      freeEdges.clear();
+      Nodes.clear();
+      FreeNodeIds.clear();
+      Edges.clear();
+      FreeEdgeIds.clear();
     }
 
     /// \brief Dump a graph to an output stream.
     template <typename OStream>
-    void dump(OStream &os) {
-      os << getNumNodes() << " " << getNumEdges() << "\n";
+    void dump(OStream &OS) {
+      OS << nodeIds().size() << " " << edgeIds().size() << "\n";
 
-      for (NodeItr nodeItr = nodesBegin(), nodeEnd = nodesEnd();
-           nodeItr != nodeEnd; ++nodeItr) {
-        const Vector& v = getNodeCosts(*nodeItr);
-        os << "\n" << v.getLength() << "\n";
-        assert(v.getLength() != 0 && "Empty vector in graph.");
-        os << v[0];
-        for (unsigned i = 1; i < v.getLength(); ++i) {
-          os << " " << v[i];
+      for (auto NId : nodeIds()) {
+        const Vector& V = getNodeCosts(NId);
+        OS << "\n" << V.getLength() << "\n";
+        assert(V.getLength() != 0 && "Empty vector in graph.");
+        OS << V[0];
+        for (unsigned i = 1; i < V.getLength(); ++i) {
+          OS << " " << V[i];
         }
-        os << "\n";
+        OS << "\n";
       }
 
-      for (EdgeItr edgeItr = edgesBegin(), edgeEnd = edgesEnd();
-           edgeItr != edgeEnd; ++edgeItr) {
-        NodeId n1 = getEdgeNode1(*edgeItr);
-        NodeId n2 = getEdgeNode2(*edgeItr);
-        assert(n1 != n2 && "PBQP graphs shound not have self-edges.");
-        const Matrix& m = getEdgeCosts(*edgeItr);
-        os << "\n" << n1 << " " << n2 << "\n"
-           << m.getRows() << " " << m.getCols() << "\n";
-        assert(m.getRows() != 0 && "No rows in matrix.");
-        assert(m.getCols() != 0 && "No cols in matrix.");
-        for (unsigned i = 0; i < m.getRows(); ++i) {
-          os << m[i][0];
-          for (unsigned j = 1; j < m.getCols(); ++j) {
-            os << " " << m[i][j];
+      for (auto EId : edgeIds()) {
+        NodeId N1Id = getEdgeNode1Id(EId);
+        NodeId N2Id = getEdgeNode2Id(EId);
+        assert(N1Id != N2Id && "PBQP graphs shound not have self-edges.");
+        const Matrix& M = getEdgeCosts(EId);
+        OS << "\n" << N1Id << " " << N2Id << "\n"
+           << M.getRows() << " " << M.getCols() << "\n";
+        assert(M.getRows() != 0 && "No rows in matrix.");
+        assert(M.getCols() != 0 && "No cols in matrix.");
+        for (unsigned i = 0; i < M.getRows(); ++i) {
+          OS << M[i][0];
+          for (unsigned j = 1; j < M.getCols(); ++j) {
+            OS << " " << M[i][j];
           }
-          os << "\n";
+          OS << "\n";
         }
       }
     }
 
     /// \brief Print a representation of this graph in DOT format.
-    /// @param os Output stream to print on.
+    /// @param OS Output stream to print on.
     template <typename OStream>
-    void printDot(OStream &os) {
-
-      os << "graph {\n";
-
-      for (NodeItr nodeItr = nodesBegin(), nodeEnd = nodesEnd();
-           nodeItr != nodeEnd; ++nodeItr) {
-
-        os << "  node" << nodeItr << " [ label=\""
-           << nodeItr << ": " << getNodeCosts(*nodeItr) << "\" ]\n";
+    void printDot(OStream &OS) {
+      OS << "graph {\n";
+      for (auto NId : nodeIds()) {
+        OS << "  node" << NId << " [ label=\""
+           << NId << ": " << getNodeCosts(NId) << "\" ]\n";
       }
-
-      os << "  edge [ len=" << getNumNodes() << " ]\n";
-
-      for (EdgeItr edgeItr = edgesBegin(), edgeEnd = edgesEnd();
-           edgeItr != edgeEnd; ++edgeItr) {
-
-        os << "  node" << getEdgeNode1(*edgeItr)
-           << " -- node" << getEdgeNode2(*edgeItr)
+      OS << "  edge [ len=" << nodeIds().size() << " ]\n";
+      for (auto EId : edgeIds()) {
+        OS << "  node" << getEdgeNode1Id(EId)
+           << " -- node" << getEdgeNode2Id(EId)
            << " [ label=\"";
-
-        const Matrix &edgeCosts = getEdgeCosts(*edgeItr);
-
-        for (unsigned i = 0; i < edgeCosts.getRows(); ++i) {
-          os << edgeCosts.getRowAsVector(i) << "\\n";
+        const Matrix &EdgeCosts = getEdgeCosts(EId);
+        for (unsigned i = 0; i < EdgeCosts.getRows(); ++i) {
+          OS << EdgeCosts.getRowAsVector(i) << "\\n";
         }
-        os << "\" ]\n";
+        OS << "\" ]\n";
       }
-      os << "}\n";
+      OS << "}\n";
     }
-
   };
 
-//  void Graph::copyFrom(const Graph &other) {
-//     std::map<Graph::ConstNodeItr, Graph::NodeItr,
-//              NodeItrComparator> nodeMap;
-
-//      for (Graph::ConstNodeItr nItr = other.nodesBegin(),
-//                              nEnd = other.nodesEnd();
-//          nItr != nEnd; ++nItr) {
-//       nodeMap[nItr] = addNode(other.getNodeCosts(nItr));
-//     }
-//  }
-
 }
 
 #endif // LLVM_CODEGEN_PBQP_GRAPH_HPP
diff --git a/include/llvm/CodeGen/PBQP/HeuristicBase.h b/include/llvm/CodeGen/PBQP/HeuristicBase.h
deleted file mode 100644
index 8bcbb9ed1d6b..000000000000
--- a/include/llvm/CodeGen/PBQP/HeuristicBase.h
+++ /dev/null
@@ -1,247 +0,0 @@
-//===-- HeuristcBase.h --- Heuristic base class for PBQP --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_PBQP_HEURISTICBASE_H
-#define LLVM_CODEGEN_PBQP_HEURISTICBASE_H
-
-#include "HeuristicSolver.h"
-
-namespace PBQP {
-
-  /// \brief Abstract base class for heuristic implementations.
-  ///
-  /// This class provides a handy base for heuristic implementations with common
-  /// solver behaviour implemented for a number of methods.
-  ///
-  /// To implement your own heuristic using this class as a base you'll have to
-  /// implement, as a minimum, the following methods:
-  /// <ul>
-  ///   <li> void addToHeuristicList(Graph::NodeItr) : Add a node to the
-  ///        heuristic reduction list.
-  ///   <li> void heuristicReduce() : Perform a single heuristic reduction.
-  ///   <li> void preUpdateEdgeCosts(Graph::EdgeItr) : Handle the (imminent)
-  ///        change to the cost matrix on the given edge (by R2).
-  ///   <li> void postUpdateEdgeCostts(Graph::EdgeItr) : Handle the new
-  ///        costs on the given edge.
-  ///   <li> void handleAddEdge(Graph::EdgeItr) : Handle the addition of a new
-  ///        edge into the PBQP graph (by R2).
-  ///   <li> void handleRemoveEdge(Graph::EdgeItr, Graph::NodeItr) : Handle the
-  ///        disconnection of the given edge from the given node.
-  ///   <li> A constructor for your derived class : to pass back a reference to
-  ///        the solver which is using this heuristic.
-  /// </ul>
-  ///
-  /// These methods are implemented in this class for documentation purposes,
-  /// but will assert if called.
-  ///
-  /// Note that this class uses the curiously recursive template idiom to
-  /// forward calls to the derived class. These methods need not be made
-  /// virtual, and indeed probably shouldn't for performance reasons.
-  ///
-  /// You'll also need to provide NodeData and EdgeData structs in your class.
-  /// These can be used to attach data relevant to your heuristic to each
-  /// node/edge in the PBQP graph.
-
-  template <typename HImpl>
-  class HeuristicBase {
-  private:
-
-    typedef std::list<Graph::NodeId> OptimalList;
-
-    HeuristicSolverImpl<HImpl> &s;
-    Graph &g;
-    OptimalList optimalList;
-
-    // Return a reference to the derived heuristic.
-    HImpl& impl() { return static_cast<HImpl&>(*this); }
-
-    // Add the given node to the optimal reductions list. Keep an iterator to
-    // its location for fast removal.
-    void addToOptimalReductionList(Graph::NodeId nId) {
-      optimalList.insert(optimalList.end(), nId);
-    }
-
-  public:
-
-    /// \brief Construct an instance with a reference to the given solver.
-    /// @param solver The solver which is using this heuristic instance.
-    HeuristicBase(HeuristicSolverImpl<HImpl> &solver)
-      : s(solver), g(s.getGraph()) { }
-
-    /// \brief Get the solver which is using this heuristic instance.
-    /// @return The solver which is using this heuristic instance.
-    ///
-    /// You can use this method to get access to the solver in your derived
-    /// heuristic implementation.
-    HeuristicSolverImpl<HImpl>& getSolver() { return s; }
-
-    /// \brief Get the graph representing the problem to be solved.
-    /// @return The graph representing the problem to be solved.
-    Graph& getGraph() { return g; }
-
-    /// \brief Tell the solver to simplify the graph before the reduction phase.
-    /// @return Whether or not the solver should run a simplification phase
-    ///         prior to the main setup and reduction.
-    ///
-    /// HeuristicBase returns true from this method as it's a sensible default,
-    /// however you can over-ride it in your derived class if you want different
-    /// behaviour.
-    bool solverRunSimplify() const { return true; }
-
-    /// \brief Decide whether a node should be optimally or heuristically
-    ///        reduced.
-    /// @return Whether or not the given node should be listed for optimal
-    ///         reduction (via R0, R1 or R2).
-    ///
-    /// HeuristicBase returns true for any node with degree less than 3. This is
-    /// sane and sensible for many situations, but not all. You can over-ride
-    /// this method in your derived class if you want a different selection
-    /// criteria. Note however that your criteria for selecting optimal nodes
-    /// should be <i>at least</i> as strong as this. I.e. Nodes of degree 3 or
-    /// higher should not be selected under any circumstances.
-    bool shouldOptimallyReduce(Graph::NodeId nId) {
-      if (g.getNodeDegree(nId) < 3)
-        return true;
-      // else
-      return false;
-    }
-
-    /// \brief Add the given node to the list of nodes to be optimally reduced.
-    /// @param nId Node id to be added.
-    ///
-    /// You probably don't want to over-ride this, except perhaps to record
-    /// statistics before calling this implementation. HeuristicBase relies on
-    /// its behaviour.
-    void addToOptimalReduceList(Graph::NodeId nId) {
-      optimalList.push_back(nId);
-    }
-
-    /// \brief Initialise the heuristic.
-    ///
-    /// HeuristicBase iterates over all nodes in the problem and adds them to
-    /// the appropriate list using addToOptimalReduceList or
-    /// addToHeuristicReduceList based on the result of shouldOptimallyReduce.
-    ///
-    /// This behaviour should be fine for most situations.
-    void setup() {
-      for (Graph::NodeItr nItr = g.nodesBegin(), nEnd = g.nodesEnd();
-           nItr != nEnd; ++nItr) {
-        if (impl().shouldOptimallyReduce(*nItr)) {
-          addToOptimalReduceList(*nItr);
-        } else {
-          impl().addToHeuristicReduceList(*nItr);
-        }
-      }
-    }
-
-    /// \brief Optimally reduce one of the nodes in the optimal reduce list.
-    /// @return True if a reduction takes place, false if the optimal reduce
-    ///         list is empty.
-    ///
-    /// Selects a node from the optimal reduce list and removes it, applying
-    /// R0, R1 or R2 as appropriate based on the selected node's degree.
-    bool optimalReduce() {
-      if (optimalList.empty())
-        return false;
-
-      Graph::NodeId nId = optimalList.front();
-      optimalList.pop_front();
-
-      switch (s.getSolverDegree(nId)) {
-        case 0: s.applyR0(nId); break;
-        case 1: s.applyR1(nId); break;
-        case 2: s.applyR2(nId); break;
-        default: llvm_unreachable(
-                        "Optimal reductions of degree > 2 nodes is invalid.");
-      }
-
-      return true;
-    }
-
-    /// \brief Perform the PBQP reduction process.
-    ///
-    /// Reduces the problem to the empty graph by repeated application of the
-    /// reduction rules R0, R1, R2 and RN.
-    /// R0, R1 or R2 are always applied if possible before RN is used.
-    void reduce() {
-      bool finished = false;
-
-      while (!finished) {
-        if (!optimalReduce()) {
-          if (impl().heuristicReduce()) {
-            getSolver().recordRN();
-          } else {
-            finished = true;
-          }
-        }
-      }
-    }
-
-    /// \brief Add a node to the heuristic reduce list.
-    /// @param nId Node id to add to the heuristic reduce list.
-    void addToHeuristicList(Graph::NodeId nId) {
-      llvm_unreachable("Must be implemented in derived class.");
-    }
-
-    /// \brief Heuristically reduce one of the nodes in the heuristic
-    ///        reduce list.
-    /// @return True if a reduction takes place, false if the heuristic reduce
-    ///         list is empty.
-    bool heuristicReduce() {
-      llvm_unreachable("Must be implemented in derived class.");
-      return false;
-    }
-
-    /// \brief Prepare a change in the costs on the given edge.
-    /// @param eId Edge id.
-    void preUpdateEdgeCosts(Graph::EdgeId eId) {
-      llvm_unreachable("Must be implemented in derived class.");
-    }
-
-    /// \brief Handle the change in the costs on the given edge.
-    /// @param eId Edge id.
-    void postUpdateEdgeCostts(Graph::EdgeId eId) {
-      llvm_unreachable("Must be implemented in derived class.");
-    }
-
-    /// \brief Handle the addition of a new edge into the PBQP graph.
-    /// @param eId Edge id for the added edge.
-    void handleAddEdge(Graph::EdgeId eId) {
-      llvm_unreachable("Must be implemented in derived class.");
-    }
-
-    /// \brief Handle disconnection of an edge from a node.
-    /// @param eId Edge id for edge being disconnected.
-    /// @param nId Node id for the node being disconnected from.
-    ///
-    /// Edges are frequently removed due to the removal of a node. This
-    /// method allows for the effect to be computed only for the remaining
-    /// node in the graph.
-    void handleRemoveEdge(Graph::EdgeId eId, Graph::NodeId nId) {
-      llvm_unreachable("Must be implemented in derived class.");
-    }
-
-    /// \brief Clean up any structures used by HeuristicBase.
-    ///
-    /// At present this just performs a sanity check: that the optimal reduce
-    /// list is empty now that reduction has completed.
-    ///
-    /// If your derived class has more complex structures which need tearing
-    /// down you should over-ride this method but include a call back to this
-    /// implementation.
-    void cleanup() {
-      assert(optimalList.empty() && "Nodes left over in optimal reduce list?");
-    }
-
-  };
-
-}
-
-
-#endif // LLVM_CODEGEN_PBQP_HEURISTICBASE_H
diff --git a/include/llvm/CodeGen/PBQP/HeuristicSolver.h b/include/llvm/CodeGen/PBQP/HeuristicSolver.h
deleted file mode 100644
index e26ca02fff7e..000000000000
--- a/include/llvm/CodeGen/PBQP/HeuristicSolver.h
+++ /dev/null
@@ -1,618 +0,0 @@
-//===-- HeuristicSolver.h - Heuristic PBQP Solver --------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Heuristic PBQP solver. This solver is able to perform optimal reductions for
-// nodes of degree 0, 1 or 2. For nodes of degree >2 a plugable heuristic is
-// used to select a node for reduction.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_PBQP_HEURISTICSOLVER_H
-#define LLVM_CODEGEN_PBQP_HEURISTICSOLVER_H
-
-#include "Graph.h"
-#include "Solution.h"
-#include <limits>
-#include <vector>
-
-namespace PBQP {
-
-  /// \brief Heuristic PBQP solver implementation.
-  ///
-  /// This class should usually be created (and destroyed) indirectly via a call
-  /// to HeuristicSolver<HImpl>::solve(Graph&).
-  /// See the comments for HeuristicSolver.
-  ///
-  /// HeuristicSolverImpl provides the R0, R1 and R2 reduction rules,
-  /// backpropagation phase, and maintains the internal copy of the graph on
-  /// which the reduction is carried out (the original being kept to facilitate
-  /// backpropagation).
-  template <typename HImpl>
-  class HeuristicSolverImpl {
-  private:
-
-    typedef typename HImpl::NodeData HeuristicNodeData;
-    typedef typename HImpl::EdgeData HeuristicEdgeData;
-
-    typedef std::list<Graph::EdgeId> SolverEdges;
-
-  public:
-
-    /// \brief Iterator type for edges in the solver graph.
-    typedef SolverEdges::iterator SolverEdgeItr;
-
-  private:
-
-    class NodeData {
-    public:
-      NodeData() : solverDegree(0) {}
-
-      HeuristicNodeData& getHeuristicData() { return hData; }
-
-      SolverEdgeItr addSolverEdge(Graph::EdgeId eId) {
-        ++solverDegree;
-        return solverEdges.insert(solverEdges.end(), eId);
-      }
-
-      void removeSolverEdge(SolverEdgeItr seItr) {
-        --solverDegree;
-        solverEdges.erase(seItr);
-      }
-
-      SolverEdgeItr solverEdgesBegin() { return solverEdges.begin(); }
-      SolverEdgeItr solverEdgesEnd() { return solverEdges.end(); }
-      unsigned getSolverDegree() const { return solverDegree; }
-      void clearSolverEdges() {
-        solverDegree = 0;
-        solverEdges.clear();
-      }
-
-    private:
-      HeuristicNodeData hData;
-      unsigned solverDegree;
-      SolverEdges solverEdges;
-    };
-
-    class EdgeData {
-    public:
-      HeuristicEdgeData& getHeuristicData() { return hData; }
-
-      void setN1SolverEdgeItr(SolverEdgeItr n1SolverEdgeItr) {
-        this->n1SolverEdgeItr = n1SolverEdgeItr;
-      }
-
-      SolverEdgeItr getN1SolverEdgeItr() { return n1SolverEdgeItr; }
-
-      void setN2SolverEdgeItr(SolverEdgeItr n2SolverEdgeItr){
-        this->n2SolverEdgeItr = n2SolverEdgeItr;
-      }
-
-      SolverEdgeItr getN2SolverEdgeItr() { return n2SolverEdgeItr; }
-
-    private:
-
-      HeuristicEdgeData hData;
-      SolverEdgeItr n1SolverEdgeItr, n2SolverEdgeItr;
-    };
-
-    Graph &g;
-    HImpl h;
-    Solution s;
-    std::vector<Graph::NodeId> stack;
-
-    typedef std::list<NodeData> NodeDataList;
-    NodeDataList nodeDataList;
-
-    typedef std::list<EdgeData> EdgeDataList;
-    EdgeDataList edgeDataList;
-
-  public:
-
-    /// \brief Construct a heuristic solver implementation to solve the given
-    ///        graph.
-    /// @param g The graph representing the problem instance to be solved.
-    HeuristicSolverImpl(Graph &g) : g(g), h(*this) {}
-
-    /// \brief Get the graph being solved by this solver.
-    /// @return The graph representing the problem instance being solved by this
-    ///         solver.
-    Graph& getGraph() { return g; }
-
-    /// \brief Get the heuristic data attached to the given node.
-    /// @param nId Node id.
-    /// @return The heuristic data attached to the given node.
-    HeuristicNodeData& getHeuristicNodeData(Graph::NodeId nId) {
-      return getSolverNodeData(nId).getHeuristicData();
-    }
-
-    /// \brief Get the heuristic data attached to the given edge.
-    /// @param eId Edge id.
-    /// @return The heuristic data attached to the given node.
-    HeuristicEdgeData& getHeuristicEdgeData(Graph::EdgeId eId) {
-      return getSolverEdgeData(eId).getHeuristicData();
-    }
-
-    /// \brief Begin iterator for the set of edges adjacent to the given node in
-    ///        the solver graph.
-    /// @param nId Node id.
-    /// @return Begin iterator for the set of edges adjacent to the given node
-    ///         in the solver graph.
-    SolverEdgeItr solverEdgesBegin(Graph::NodeId nId) {
-      return getSolverNodeData(nId).solverEdgesBegin();
-    }
-
-    /// \brief End iterator for the set of edges adjacent to the given node in
-    ///        the solver graph.
-    /// @param nId Node id.
-    /// @return End iterator for the set of edges adjacent to the given node in
-    ///         the solver graph.
-    SolverEdgeItr solverEdgesEnd(Graph::NodeId nId) {
-      return getSolverNodeData(nId).solverEdgesEnd();
-    }
-
-    /// \brief Remove a node from the solver graph.
-    /// @param eId Edge id for edge to be removed.
-    ///
-    /// Does <i>not</i> notify the heuristic of the removal. That should be
-    /// done manually if necessary.
-    void removeSolverEdge(Graph::EdgeId eId) {
-      EdgeData &eData = getSolverEdgeData(eId);
-      NodeData &n1Data = getSolverNodeData(g.getEdgeNode1(eId)),
-               &n2Data = getSolverNodeData(g.getEdgeNode2(eId));
-
-      n1Data.removeSolverEdge(eData.getN1SolverEdgeItr());
-      n2Data.removeSolverEdge(eData.getN2SolverEdgeItr());
-    }
-
-    /// \brief Compute a solution to the PBQP problem instance with which this
-    ///        heuristic solver was constructed.
-    /// @return A solution to the PBQP problem.
-    ///
-    /// Performs the full PBQP heuristic solver algorithm, including setup,
-    /// calls to the heuristic (which will call back to the reduction rules in
-    /// this class), and cleanup.
-    Solution computeSolution() {
-      setup();
-      h.setup();
-      h.reduce();
-      backpropagate();
-      h.cleanup();
-      cleanup();
-      return s;
-    }
-
-    /// \brief Add to the end of the stack.
-    /// @param nId Node id to add to the reduction stack.
-    void pushToStack(Graph::NodeId nId) {
-      getSolverNodeData(nId).clearSolverEdges();
-      stack.push_back(nId);
-    }
-
-    /// \brief Returns the solver degree of the given node.
-    /// @param nId Node id for which degree is requested.
-    /// @return Node degree in the <i>solver</i> graph (not the original graph).
-    unsigned getSolverDegree(Graph::NodeId nId) {
-      return  getSolverNodeData(nId).getSolverDegree();
-    }
-
-    /// \brief Set the solution of the given node.
-    /// @param nId Node id to set solution for.
-    /// @param selection Selection for node.
-    void setSolution(const Graph::NodeId &nId, unsigned selection) {
-      s.setSelection(nId, selection);
-
-      for (Graph::AdjEdgeItr aeItr = g.adjEdgesBegin(nId),
-                             aeEnd = g.adjEdgesEnd(nId);
-           aeItr != aeEnd; ++aeItr) {
-        Graph::EdgeId eId(*aeItr);
-        Graph::NodeId anId(g.getEdgeOtherNode(eId, nId));
-        getSolverNodeData(anId).addSolverEdge(eId);
-      }
-    }
-
-    /// \brief Apply rule R0.
-    /// @param nId Node id for node to apply R0 to.
-    ///
-    /// Node will be automatically pushed to the solver stack.
-    void applyR0(Graph::NodeId nId) {
-      assert(getSolverNodeData(nId).getSolverDegree() == 0 &&
-             "R0 applied to node with degree != 0.");
-
-      // Nothing to do. Just push the node onto the reduction stack.
-      pushToStack(nId);
-
-      s.recordR0();
-    }
-
-    /// \brief Apply rule R1.
-    /// @param xnId Node id for node to apply R1 to.
-    ///
-    /// Node will be automatically pushed to the solver stack.
-    void applyR1(Graph::NodeId xnId) {
-      NodeData &nd = getSolverNodeData(xnId);
-      assert(nd.getSolverDegree() == 1 &&
-             "R1 applied to node with degree != 1.");
-
-      Graph::EdgeId eId = *nd.solverEdgesBegin();
-
-      const Matrix &eCosts = g.getEdgeCosts(eId);
-      const Vector &xCosts = g.getNodeCosts(xnId);
-
-      // Duplicate a little to avoid transposing matrices.
-      if (xnId == g.getEdgeNode1(eId)) {
-        Graph::NodeId ynId = g.getEdgeNode2(eId);
-        Vector &yCosts = g.getNodeCosts(ynId);
-        for (unsigned j = 0; j < yCosts.getLength(); ++j) {
-          PBQPNum min = eCosts[0][j] + xCosts[0];
-          for (unsigned i = 1; i < xCosts.getLength(); ++i) {
-            PBQPNum c = eCosts[i][j] + xCosts[i];
-            if (c < min)
-              min = c;
-          }
-          yCosts[j] += min;
-        }
-        h.handleRemoveEdge(eId, ynId);
-     } else {
-        Graph::NodeId ynId = g.getEdgeNode1(eId);
-        Vector &yCosts = g.getNodeCosts(ynId);
-        for (unsigned i = 0; i < yCosts.getLength(); ++i) {
-          PBQPNum min = eCosts[i][0] + xCosts[0];
-          for (unsigned j = 1; j < xCosts.getLength(); ++j) {
-            PBQPNum c = eCosts[i][j] + xCosts[j];
-            if (c < min)
-              min = c;
-          }
-          yCosts[i] += min;
-        }
-        h.handleRemoveEdge(eId, ynId);
-      }
-      removeSolverEdge(eId);
-      assert(nd.getSolverDegree() == 0 &&
-             "Degree 1 with edge removed should be 0.");
-      pushToStack(xnId);
-      s.recordR1();
-    }
-
-    /// \brief Apply rule R2.
-    /// @param xnId Node id for node to apply R2 to.
-    ///
-    /// Node will be automatically pushed to the solver stack.
-    void applyR2(Graph::NodeId xnId) {
-      assert(getSolverNodeData(xnId).getSolverDegree() == 2 &&
-             "R2 applied to node with degree != 2.");
-
-      NodeData &nd = getSolverNodeData(xnId);
-      const Vector &xCosts = g.getNodeCosts(xnId);
-
-      SolverEdgeItr aeItr = nd.solverEdgesBegin();
-      Graph::EdgeId yxeId = *aeItr,
-                    zxeId = *(++aeItr);
-
-      Graph::NodeId ynId = g.getEdgeOtherNode(yxeId, xnId),
-                    znId = g.getEdgeOtherNode(zxeId, xnId);
-
-      bool flipEdge1 = (g.getEdgeNode1(yxeId) == xnId),
-           flipEdge2 = (g.getEdgeNode1(zxeId) == xnId);
-
-      const Matrix *yxeCosts = flipEdge1 ?
-        new Matrix(g.getEdgeCosts(yxeId).transpose()) :
-        &g.getEdgeCosts(yxeId);
-
-      const Matrix *zxeCosts = flipEdge2 ?
-        new Matrix(g.getEdgeCosts(zxeId).transpose()) :
-        &g.getEdgeCosts(zxeId);
-
-      unsigned xLen = xCosts.getLength(),
-               yLen = yxeCosts->getRows(),
-               zLen = zxeCosts->getRows();
-
-      Matrix delta(yLen, zLen);
-
-      for (unsigned i = 0; i < yLen; ++i) {
-        for (unsigned j = 0; j < zLen; ++j) {
-          PBQPNum min = (*yxeCosts)[i][0] + (*zxeCosts)[j][0] + xCosts[0];
-          for (unsigned k = 1; k < xLen; ++k) {
-            PBQPNum c = (*yxeCosts)[i][k] + (*zxeCosts)[j][k] + xCosts[k];
-            if (c < min) {
-              min = c;
-            }
-          }
-          delta[i][j] = min;
-        }
-      }
-
-      if (flipEdge1)
-        delete yxeCosts;
-
-      if (flipEdge2)
-        delete zxeCosts;
-
-      Graph::EdgeId yzeId = g.findEdge(ynId, znId);
-      bool addedEdge = false;
-
-      if (yzeId == g.invalidEdgeId()) {
-        yzeId = g.addEdge(ynId, znId, delta);
-        addedEdge = true;
-      } else {
-        Matrix &yzeCosts = g.getEdgeCosts(yzeId);
-        h.preUpdateEdgeCosts(yzeId);
-        if (ynId == g.getEdgeNode1(yzeId)) {
-          yzeCosts += delta;
-        } else {
-          yzeCosts += delta.transpose();
-        }
-      }
-
-      bool nullCostEdge = tryNormaliseEdgeMatrix(yzeId);
-
-      if (!addedEdge) {
-        // If we modified the edge costs let the heuristic know.
-        h.postUpdateEdgeCosts(yzeId);
-      }
-
-      if (nullCostEdge) {
-        // If this edge ended up null remove it.
-        if (!addedEdge) {
-          // We didn't just add it, so we need to notify the heuristic
-          // and remove it from the solver.
-          h.handleRemoveEdge(yzeId, ynId);
-          h.handleRemoveEdge(yzeId, znId);
-          removeSolverEdge(yzeId);
-        }
-        g.removeEdge(yzeId);
-      } else if (addedEdge) {
-        // If the edge was added, and non-null, finish setting it up, add it to
-        // the solver & notify heuristic.
-        edgeDataList.push_back(EdgeData());
-        g.setEdgeData(yzeId, &edgeDataList.back());
-        addSolverEdge(yzeId);
-        h.handleAddEdge(yzeId);
-      }
-
-      h.handleRemoveEdge(yxeId, ynId);
-      removeSolverEdge(yxeId);
-      h.handleRemoveEdge(zxeId, znId);
-      removeSolverEdge(zxeId);
-
-      pushToStack(xnId);
-      s.recordR2();
-    }
-
-    /// \brief Record an application of the RN rule.
-    ///
-    /// For use by the HeuristicBase.
-    void recordRN() { s.recordRN(); }
-
-  private:
-
-    NodeData& getSolverNodeData(Graph::NodeId nId) {
-      return *static_cast<NodeData*>(g.getNodeData(nId));
-    }
-
-    EdgeData& getSolverEdgeData(Graph::EdgeId eId) {
-      return *static_cast<EdgeData*>(g.getEdgeData(eId));
-    }
-
-    void addSolverEdge(Graph::EdgeId eId) {
-      EdgeData &eData = getSolverEdgeData(eId);
-      NodeData &n1Data = getSolverNodeData(g.getEdgeNode1(eId)),
-               &n2Data = getSolverNodeData(g.getEdgeNode2(eId));
-
-      eData.setN1SolverEdgeItr(n1Data.addSolverEdge(eId));
-      eData.setN2SolverEdgeItr(n2Data.addSolverEdge(eId));
-    }
-
-    void setup() {
-      if (h.solverRunSimplify()) {
-        simplify();
-      }
-
-      // Create node data objects.
-      for (Graph::NodeItr nItr = g.nodesBegin(), nEnd = g.nodesEnd();
-           nItr != nEnd; ++nItr) {
-        nodeDataList.push_back(NodeData());
-        g.setNodeData(*nItr, &nodeDataList.back());
-      }
-
-      // Create edge data objects.
-      for (Graph::EdgeItr eItr = g.edgesBegin(), eEnd = g.edgesEnd();
-           eItr != eEnd; ++eItr) {
-        edgeDataList.push_back(EdgeData());
-        g.setEdgeData(*eItr, &edgeDataList.back());
-        addSolverEdge(*eItr);
-      }
-    }
-
-    void simplify() {
-      disconnectTrivialNodes();
-      eliminateIndependentEdges();
-    }
-
-    // Eliminate trivial nodes.
-    void disconnectTrivialNodes() {
-      unsigned numDisconnected = 0;
-
-      for (Graph::NodeItr nItr = g.nodesBegin(), nEnd = g.nodesEnd();
-           nItr != nEnd; ++nItr) {
-
-        Graph::NodeId nId = *nItr;
-
-        if (g.getNodeCosts(nId).getLength() == 1) {
-
-          std::vector<Graph::EdgeId> edgesToRemove;
-
-          for (Graph::AdjEdgeItr aeItr = g.adjEdgesBegin(nId),
-                                 aeEnd = g.adjEdgesEnd(nId);
-               aeItr != aeEnd; ++aeItr) {
-
-            Graph::EdgeId eId = *aeItr;
-
-            if (g.getEdgeNode1(eId) == nId) {
-              Graph::NodeId otherNodeId = g.getEdgeNode2(eId);
-              g.getNodeCosts(otherNodeId) +=
-                g.getEdgeCosts(eId).getRowAsVector(0);
-            }
-            else {
-              Graph::NodeId otherNodeId = g.getEdgeNode1(eId);
-              g.getNodeCosts(otherNodeId) +=
-                g.getEdgeCosts(eId).getColAsVector(0);
-            }
-
-            edgesToRemove.push_back(eId);
-          }
-
-          if (!edgesToRemove.empty())
-            ++numDisconnected;
-
-          while (!edgesToRemove.empty()) {
-            g.removeEdge(edgesToRemove.back());
-            edgesToRemove.pop_back();
-          }
-        }
-      }
-    }
-
-    void eliminateIndependentEdges() {
-      std::vector<Graph::EdgeId> edgesToProcess;
-      unsigned numEliminated = 0;
-
-      for (Graph::EdgeItr eItr = g.edgesBegin(), eEnd = g.edgesEnd();
-           eItr != eEnd; ++eItr) {
-        edgesToProcess.push_back(*eItr);
-      }
-
-      while (!edgesToProcess.empty()) {
-        if (tryToEliminateEdge(edgesToProcess.back()))
-          ++numEliminated;
-        edgesToProcess.pop_back();
-      }
-    }
-
-    bool tryToEliminateEdge(Graph::EdgeId eId) {
-      if (tryNormaliseEdgeMatrix(eId)) {
-        g.removeEdge(eId);
-        return true;
-      }
-      return false;
-    }
-
-    bool tryNormaliseEdgeMatrix(Graph::EdgeId &eId) {
-
-      const PBQPNum infinity = std::numeric_limits<PBQPNum>::infinity();
-
-      Matrix &edgeCosts = g.getEdgeCosts(eId);
-      Vector &uCosts = g.getNodeCosts(g.getEdgeNode1(eId)),
-             &vCosts = g.getNodeCosts(g.getEdgeNode2(eId));
-
-      for (unsigned r = 0; r < edgeCosts.getRows(); ++r) {
-        PBQPNum rowMin = infinity;
-
-        for (unsigned c = 0; c < edgeCosts.getCols(); ++c) {
-          if (vCosts[c] != infinity && edgeCosts[r][c] < rowMin)
-            rowMin = edgeCosts[r][c];
-        }
-
-        uCosts[r] += rowMin;
-
-        if (rowMin != infinity) {
-          edgeCosts.subFromRow(r, rowMin);
-        }
-        else {
-          edgeCosts.setRow(r, 0);
-        }
-      }
-
-      for (unsigned c = 0; c < edgeCosts.getCols(); ++c) {
-        PBQPNum colMin = infinity;
-
-        for (unsigned r = 0; r < edgeCosts.getRows(); ++r) {
-          if (uCosts[r] != infinity && edgeCosts[r][c] < colMin)
-            colMin = edgeCosts[r][c];
-        }
-
-        vCosts[c] += colMin;
-
-        if (colMin != infinity) {
-          edgeCosts.subFromCol(c, colMin);
-        }
-        else {
-          edgeCosts.setCol(c, 0);
-        }
-      }
-
-      return edgeCosts.isZero();
-    }
-
-    void backpropagate() {
-      while (!stack.empty()) {
-        computeSolution(stack.back());
-        stack.pop_back();
-      }
-    }
-
-    void computeSolution(Graph::NodeId nId) {
-
-      NodeData &nodeData = getSolverNodeData(nId);
-
-      Vector v(g.getNodeCosts(nId));
-
-      // Solve based on existing solved edges.
-      for (SolverEdgeItr solvedEdgeItr = nodeData.solverEdgesBegin(),
-                         solvedEdgeEnd = nodeData.solverEdgesEnd();
-           solvedEdgeItr != solvedEdgeEnd; ++solvedEdgeItr) {
-
-        Graph::EdgeId eId(*solvedEdgeItr);
-        Matrix &edgeCosts = g.getEdgeCosts(eId);
-
-        if (nId == g.getEdgeNode1(eId)) {
-          Graph::NodeId adjNode(g.getEdgeNode2(eId));
-          unsigned adjSolution = s.getSelection(adjNode);
-          v += edgeCosts.getColAsVector(adjSolution);
-        }
-        else {
-          Graph::NodeId adjNode(g.getEdgeNode1(eId));
-          unsigned adjSolution = s.getSelection(adjNode);
-          v += edgeCosts.getRowAsVector(adjSolution);
-        }
-
-      }
-
-      setSolution(nId, v.minIndex());
-    }
-
-    void cleanup() {
-      h.cleanup();
-      nodeDataList.clear();
-      edgeDataList.clear();
-    }
-  };
-
-  /// \brief PBQP heuristic solver class.
-  ///
-  /// Given a PBQP Graph g representing a PBQP problem, you can find a solution
-  /// by calling
-  /// <tt>Solution s = HeuristicSolver<H>::solve(g);</tt>
-  ///
-  /// The choice of heuristic for the H parameter will affect both the solver
-  /// speed and solution quality. The heuristic should be chosen based on the
-  /// nature of the problem being solved.
-  /// Currently the only solver included with LLVM is the Briggs heuristic for
-  /// register allocation.
-  template <typename HImpl>
-  class HeuristicSolver {
-  public:
-    static Solution solve(Graph &g) {
-      HeuristicSolverImpl<HImpl> hs(g);
-      return hs.computeSolution();
-    }
-  };
-
-}
-
-#endif // LLVM_CODEGEN_PBQP_HEURISTICSOLVER_H
diff --git a/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h b/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h
deleted file mode 100644
index c355c2c2f81a..000000000000
--- a/include/llvm/CodeGen/PBQP/Heuristics/Briggs.h
+++ /dev/null
@@ -1,468 +0,0 @@
-//===-- Briggs.h --- Briggs Heuristic for PBQP ------------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This class implements the Briggs test for "allocability" of nodes in a
-// PBQP graph representing a register allocation problem. Nodes which can be
-// proven allocable (by a safe and relatively accurate test) are removed from
-// the PBQP graph first. If no provably allocable node is present in the graph
-// then the node with the minimal spill-cost to degree ratio is removed.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CODEGEN_PBQP_HEURISTICS_BRIGGS_H
-#define LLVM_CODEGEN_PBQP_HEURISTICS_BRIGGS_H
-
-#include "../HeuristicBase.h"
-#include "../HeuristicSolver.h"
-#include <limits>
-
-namespace PBQP {
-  namespace Heuristics {
-
-    /// \brief PBQP Heuristic which applies an allocability test based on
-    ///        Briggs.
-    ///
-    /// This heuristic assumes that the elements of cost vectors in the PBQP
-    /// problem represent storage options, with the first being the spill
-    /// option and subsequent elements representing legal registers for the
-    /// corresponding node. Edge cost matrices are likewise assumed to represent
-    /// register constraints.
-    /// If one or more nodes can be proven allocable by this heuristic (by
-    /// inspection of their constraint matrices) then the allocable node of
-    /// highest degree is selected for the next reduction and pushed to the
-    /// solver stack. If no nodes can be proven allocable then the node with
-    /// the lowest estimated spill cost is selected and push to the solver stack
-    /// instead.
-    ///
-    /// This implementation is built on top of HeuristicBase.
-    class Briggs : public HeuristicBase<Briggs> {
-    private:
-
-      class LinkDegreeComparator {
-      public:
-        LinkDegreeComparator(HeuristicSolverImpl<Briggs> &s) : s(&s) {}
-        bool operator()(Graph::NodeId n1Id, Graph::NodeId n2Id) const {
-          if (s->getSolverDegree(n1Id) > s->getSolverDegree(n2Id))
-            return true;
-          return false;
-        }
-      private:
-        HeuristicSolverImpl<Briggs> *s;
-      };
-
-      class SpillCostComparator {
-      public:
-        SpillCostComparator(HeuristicSolverImpl<Briggs> &s)
-          : s(&s), g(&s.getGraph()) {}
-        bool operator()(Graph::NodeId n1Id, Graph::NodeId n2Id) const {
-          const PBQP::Vector &cv1 = g->getNodeCosts(n1Id);
-          const PBQP::Vector &cv2 = g->getNodeCosts(n2Id);
-
-          PBQPNum cost1 = cv1[0] / s->getSolverDegree(n1Id);
-          PBQPNum cost2 = cv2[0] / s->getSolverDegree(n2Id);
-
-          if (cost1 < cost2)
-            return true;
-          return false;
-        }
-
-      private:
-        HeuristicSolverImpl<Briggs> *s;
-        Graph *g;
-      };
-
-      typedef std::list<Graph::NodeId> RNAllocableList;
-      typedef RNAllocableList::iterator RNAllocableListItr;
-
-      typedef std::list<Graph::NodeId> RNUnallocableList;
-      typedef RNUnallocableList::iterator RNUnallocableListItr;
-
-    public:
-
-      struct NodeData {
-        typedef std::vector<unsigned> UnsafeDegreesArray;
-        bool isHeuristic, isAllocable, isInitialized;
-        unsigned numDenied, numSafe;
-        UnsafeDegreesArray unsafeDegrees;
-        RNAllocableListItr rnaItr;
-        RNUnallocableListItr rnuItr;
-
-        NodeData()
-          : isHeuristic(false), isAllocable(false), isInitialized(false),
-            numDenied(0), numSafe(0) { }
-      };
-
-      struct EdgeData {
-        typedef std::vector<unsigned> UnsafeArray;
-        unsigned worst, reverseWorst;
-        UnsafeArray unsafe, reverseUnsafe;
-        bool isUpToDate;
-
-        EdgeData() : worst(0), reverseWorst(0), isUpToDate(false) {}
-      };
-
-      /// \brief Construct an instance of the Briggs heuristic.
-      /// @param solver A reference to the solver which is using this heuristic.
-      Briggs(HeuristicSolverImpl<Briggs> &solver) :
-        HeuristicBase<Briggs>(solver) {}
-
-      /// \brief Determine whether a node should be reduced using optimal
-      ///        reduction.
-      /// @param nId Node id to be considered.
-      /// @return True if the given node should be optimally reduced, false
-      ///         otherwise.
-      ///
-      /// Selects nodes of degree 0, 1 or 2 for optimal reduction, with one
-      /// exception. Nodes whose spill cost (element 0 of their cost vector) is
-      /// infinite are checked for allocability first. Allocable nodes may be
-      /// optimally reduced, but nodes whose allocability cannot be proven are
-      /// selected for heuristic reduction instead.
-      bool shouldOptimallyReduce(Graph::NodeId nId) {
-        if (getSolver().getSolverDegree(nId) < 3) {
-          return true;
-        }
-        // else
-        return false;
-      }
-
-      /// \brief Add a node to the heuristic reduce list.
-      /// @param nId Node id to add to the heuristic reduce list.
-      void addToHeuristicReduceList(Graph::NodeId nId) {
-        NodeData &nd = getHeuristicNodeData(nId);
-        initializeNode(nId);
-        nd.isHeuristic = true;
-        if (nd.isAllocable) {
-          nd.rnaItr = rnAllocableList.insert(rnAllocableList.end(), nId);
-        } else {
-          nd.rnuItr = rnUnallocableList.insert(rnUnallocableList.end(), nId);
-        }
-      }
-
-      /// \brief Heuristically reduce one of the nodes in the heuristic
-      ///        reduce list.
-      /// @return True if a reduction takes place, false if the heuristic reduce
-      ///         list is empty.
-      ///
-      /// If the list of allocable nodes is non-empty a node is selected
-      /// from it and pushed to the stack. Otherwise if the non-allocable list
-      /// is non-empty a node is selected from it and pushed to the stack.
-      /// If both lists are empty the method simply returns false with no action
-      /// taken.
-      bool heuristicReduce() {
-        if (!rnAllocableList.empty()) {
-          RNAllocableListItr rnaItr =
-            min_element(rnAllocableList.begin(), rnAllocableList.end(),
-                        LinkDegreeComparator(getSolver()));
-          Graph::NodeId nId = *rnaItr;
-          rnAllocableList.erase(rnaItr);
-          handleRemoveNode(nId);
-          getSolver().pushToStack(nId);
-          return true;
-        } else if (!rnUnallocableList.empty()) {
-          RNUnallocableListItr rnuItr =
-            min_element(rnUnallocableList.begin(), rnUnallocableList.end(),
-                        SpillCostComparator(getSolver()));
-          Graph::NodeId nId = *rnuItr;
-          rnUnallocableList.erase(rnuItr);
-          handleRemoveNode(nId);
-          getSolver().pushToStack(nId);
-          return true;
-        }
-        // else
-        return false;
-      }
-
-      /// \brief Prepare a change in the costs on the given edge.
-      /// @param eId Edge id.
-      void preUpdateEdgeCosts(Graph::EdgeId eId) {
-        Graph &g = getGraph();
-        Graph::NodeId n1Id = g.getEdgeNode1(eId),
-                      n2Id = g.getEdgeNode2(eId);
-        NodeData &n1 = getHeuristicNodeData(n1Id),
-                 &n2 = getHeuristicNodeData(n2Id);
-
-        if (n1.isHeuristic)
-          subtractEdgeContributions(eId, getGraph().getEdgeNode1(eId));
-        if (n2.isHeuristic)
-          subtractEdgeContributions(eId, getGraph().getEdgeNode2(eId));
-
-        EdgeData &ed = getHeuristicEdgeData(eId);
-        ed.isUpToDate = false;
-      }
-
-      /// \brief Handle the change in the costs on the given edge.
-      /// @param eId Edge id.
-      void postUpdateEdgeCosts(Graph::EdgeId eId) {
-        // This is effectively the same as adding a new edge now, since
-        // we've factored out the costs of the old one.
-        handleAddEdge(eId);
-      }
-
-      /// \brief Handle the addition of a new edge into the PBQP graph.
-      /// @param eId Edge id for the added edge.
-      ///
-      /// Updates allocability of any nodes connected by this edge which are
-      /// being managed by the heuristic. If allocability changes they are
-      /// moved to the appropriate list.
-      void handleAddEdge(Graph::EdgeId eId) {
-        Graph &g = getGraph();
-        Graph::NodeId n1Id = g.getEdgeNode1(eId),
-                      n2Id = g.getEdgeNode2(eId);
-        NodeData &n1 = getHeuristicNodeData(n1Id),
-                 &n2 = getHeuristicNodeData(n2Id);
-
-        // If neither node is managed by the heuristic there's nothing to be
-        // done.
-        if (!n1.isHeuristic && !n2.isHeuristic)
-          return;
-
-        // Ok - we need to update at least one node.
-        computeEdgeContributions(eId);
-
-        // Update node 1 if it's managed by the heuristic.
-        if (n1.isHeuristic) {
-          bool n1WasAllocable = n1.isAllocable;
-          addEdgeContributions(eId, n1Id);
-          updateAllocability(n1Id);
-          if (n1WasAllocable && !n1.isAllocable) {
-            rnAllocableList.erase(n1.rnaItr);
-            n1.rnuItr =
-              rnUnallocableList.insert(rnUnallocableList.end(), n1Id);
-          }
-        }
-
-        // Likewise for node 2.
-        if (n2.isHeuristic) {
-          bool n2WasAllocable = n2.isAllocable;
-          addEdgeContributions(eId, n2Id);
-          updateAllocability(n2Id);
-          if (n2WasAllocable && !n2.isAllocable) {
-            rnAllocableList.erase(n2.rnaItr);
-            n2.rnuItr =
-              rnUnallocableList.insert(rnUnallocableList.end(), n2Id);
-          }
-        }
-      }
-
-      /// \brief Handle disconnection of an edge from a node.
-      /// @param eId Edge id for edge being disconnected.
-      /// @param nId Node id for the node being disconnected from.
-      ///
-      /// Updates allocability of the given node and, if appropriate, moves the
-      /// node to a new list.
-      void handleRemoveEdge(Graph::EdgeId eId, Graph::NodeId nId) {
-        NodeData &nd =getHeuristicNodeData(nId);
-
-        // If the node is not managed by the heuristic there's nothing to be
-        // done.
-        if (!nd.isHeuristic)
-          return;
-
-        EdgeData &ed = getHeuristicEdgeData(eId);
-        (void)ed;
-        assert(ed.isUpToDate && "Edge data is not up to date.");
-
-        // Update node.
-        bool ndWasAllocable = nd.isAllocable;
-        subtractEdgeContributions(eId, nId);
-        updateAllocability(nId);
-
-        // If the node has gone optimal...
-        if (shouldOptimallyReduce(nId)) {
-          nd.isHeuristic = false;
-          addToOptimalReduceList(nId);
-          if (ndWasAllocable) {
-            rnAllocableList.erase(nd.rnaItr);
-          } else {
-            rnUnallocableList.erase(nd.rnuItr);
-          }
-        } else {
-          // Node didn't go optimal, but we might have to move it
-          // from "unallocable" to "allocable".
-          if (!ndWasAllocable && nd.isAllocable) {
-            rnUnallocableList.erase(nd.rnuItr);
-            nd.rnaItr = rnAllocableList.insert(rnAllocableList.end(), nId);
-          }
-        }
-      }
-
-    private:
-
-      NodeData& getHeuristicNodeData(Graph::NodeId nId) {
-        return getSolver().getHeuristicNodeData(nId);
-      }
-
-      EdgeData& getHeuristicEdgeData(Graph::EdgeId eId) {
-        return getSolver().getHeuristicEdgeData(eId);
-      }
-
-      // Work out what this edge will contribute to the allocability of the
-      // nodes connected to it.
-      void computeEdgeContributions(Graph::EdgeId eId) {
-        EdgeData &ed = getHeuristicEdgeData(eId);
-
-        if (ed.isUpToDate)
-          return; // Edge data is already up to date.
-
-        Matrix &eCosts = getGraph().getEdgeCosts(eId);
-
-        unsigned numRegs = eCosts.getRows() - 1,
-                 numReverseRegs = eCosts.getCols() - 1;
-
-        std::vector<unsigned> rowInfCounts(numRegs, 0),
-                              colInfCounts(numReverseRegs, 0);
-
-        ed.worst = 0;
-        ed.reverseWorst = 0;
-        ed.unsafe.clear();
-        ed.unsafe.resize(numRegs, 0);
-        ed.reverseUnsafe.clear();
-        ed.reverseUnsafe.resize(numReverseRegs, 0);
-
-        for (unsigned i = 0; i < numRegs; ++i) {
-          for (unsigned j = 0; j < numReverseRegs; ++j) {
-            if (eCosts[i + 1][j + 1] ==
-                  std::numeric_limits<PBQPNum>::infinity()) {
-              ed.unsafe[i] = 1;
-              ed.reverseUnsafe[j] = 1;
-              ++rowInfCounts[i];
-              ++colInfCounts[j];
-
-              if (colInfCounts[j] > ed.worst) {
-                ed.worst = colInfCounts[j];
-              }
-
-              if (rowInfCounts[i] > ed.reverseWorst) {
-                ed.reverseWorst = rowInfCounts[i];
-              }
-            }
-          }
-        }
-
-        ed.isUpToDate = true;
-      }
-
-      // Add the contributions of the given edge to the given node's
-      // numDenied and safe members. No action is taken other than to update
-      // these member values. Once updated these numbers can be used by clients
-      // to update the node's allocability.
-      void addEdgeContributions(Graph::EdgeId eId, Graph::NodeId nId) {
-        EdgeData &ed = getHeuristicEdgeData(eId);
-
-        assert(ed.isUpToDate && "Using out-of-date edge numbers.");
-
-        NodeData &nd = getHeuristicNodeData(nId);
-        unsigned numRegs = getGraph().getNodeCosts(nId).getLength() - 1;
-
-        bool nIsNode1 = nId == getGraph().getEdgeNode1(eId);
-        EdgeData::UnsafeArray &unsafe =
-          nIsNode1 ? ed.unsafe : ed.reverseUnsafe;
-        nd.numDenied += nIsNode1 ? ed.worst : ed.reverseWorst;
-
-        for (unsigned r = 0; r < numRegs; ++r) {
-          if (unsafe[r]) {
-            if (nd.unsafeDegrees[r]==0) {
-              --nd.numSafe;
-            }
-            ++nd.unsafeDegrees[r];
-          }
-        }
-      }
-
-      // Subtract the contributions of the given edge to the given node's
-      // numDenied and safe members. No action is taken other than to update
-      // these member values. Once updated these numbers can be used by clients
-      // to update the node's allocability.
-      void subtractEdgeContributions(Graph::EdgeId eId, Graph::NodeId nId) {
-        EdgeData &ed = getHeuristicEdgeData(eId);
-
-        assert(ed.isUpToDate && "Using out-of-date edge numbers.");
-
-        NodeData &nd = getHeuristicNodeData(nId);
-        unsigned numRegs = getGraph().getNodeCosts(nId).getLength() - 1;
-
-        bool nIsNode1 = nId == getGraph().getEdgeNode1(eId);
-        EdgeData::UnsafeArray &unsafe =
-          nIsNode1 ? ed.unsafe : ed.reverseUnsafe;
-        nd.numDenied -= nIsNode1 ? ed.worst : ed.reverseWorst;
-
-        for (unsigned r = 0; r < numRegs; ++r) {
-          if (unsafe[r]) {
-            if (nd.unsafeDegrees[r] == 1) {
-              ++nd.numSafe;
-            }
-            --nd.unsafeDegrees[r];
-          }
-        }
-      }
-
-      void updateAllocability(Graph::NodeId nId) {
-        NodeData &nd = getHeuristicNodeData(nId);
-        unsigned numRegs = getGraph().getNodeCosts(nId).getLength() - 1;
-        nd.isAllocable = nd.numDenied < numRegs || nd.numSafe > 0;
-      }
-
-      void initializeNode(Graph::NodeId nId) {
-        NodeData &nd = getHeuristicNodeData(nId);
-
-        if (nd.isInitialized)
-          return; // Node data is already up to date.
-
-        unsigned numRegs = getGraph().getNodeCosts(nId).getLength() - 1;
-
-        nd.numDenied = 0;
-        const Vector& nCosts = getGraph().getNodeCosts(nId);
-        for (unsigned i = 1; i < nCosts.getLength(); ++i) {
-          if (nCosts[i] == std::numeric_limits<PBQPNum>::infinity())
-            ++nd.numDenied;
-        }
-
-        nd.numSafe = numRegs;
-        nd.unsafeDegrees.resize(numRegs, 0);
-
-        typedef HeuristicSolverImpl<Briggs>::SolverEdgeItr SolverEdgeItr;
-
-        for (SolverEdgeItr aeItr = getSolver().solverEdgesBegin(nId),
-                           aeEnd = getSolver().solverEdgesEnd(nId);
-             aeItr != aeEnd; ++aeItr) {
-
-          Graph::EdgeId eId = *aeItr;
-          computeEdgeContributions(eId);
-          addEdgeContributions(eId, nId);
-        }
-
-        updateAllocability(nId);
-        nd.isInitialized = true;
-      }
-
-      void handleRemoveNode(Graph::NodeId xnId) {
-        typedef HeuristicSolverImpl<Briggs>::SolverEdgeItr SolverEdgeItr;
-        std::vector<Graph::EdgeId> edgesToRemove;
-        for (SolverEdgeItr aeItr = getSolver().solverEdgesBegin(xnId),
-                           aeEnd = getSolver().solverEdgesEnd(xnId);
-             aeItr != aeEnd; ++aeItr) {
-          Graph::NodeId ynId = getGraph().getEdgeOtherNode(*aeItr, xnId);
-          handleRemoveEdge(*aeItr, ynId);
-          edgesToRemove.push_back(*aeItr);
-        }
-        while (!edgesToRemove.empty()) {
-          getSolver().removeSolverEdge(edgesToRemove.back());
-          edgesToRemove.pop_back();
-        }
-      }
-
-      RNAllocableList rnAllocableList;
-      RNUnallocableList rnUnallocableList;
-    };
-
-  }
-}
-
-
-#endif // LLVM_CODEGEN_PBQP_HEURISTICS_BRIGGS_H
diff --git a/include/llvm/CodeGen/PBQP/Math.h b/include/llvm/CodeGen/PBQP/Math.h
index 08f8b981ae27..69a9d83cc092 100644
--- a/include/llvm/CodeGen/PBQP/Math.h
+++ b/include/llvm/CodeGen/PBQP/Math.h
@@ -20,268 +20,418 @@ typedef float PBQPNum;
 
 /// \brief PBQP Vector class.
 class Vector {
-  public:
+  friend class VectorComparator;
+public:
 
-    /// \brief Construct a PBQP vector of the given size.
-    explicit Vector(unsigned length) :
-      length(length), data(new PBQPNum[length]) {
-      }
+  /// \brief Construct a PBQP vector of the given size.
+  explicit Vector(unsigned Length)
+    : Length(Length), Data(new PBQPNum[Length]) {
+    // llvm::dbgs() << "Constructing PBQP::Vector "
+    //              << this << " (length " << Length << ")\n";
+  }
+
+  /// \brief Construct a PBQP vector with initializer.
+  Vector(unsigned Length, PBQPNum InitVal)
+    : Length(Length), Data(new PBQPNum[Length]) {
+    // llvm::dbgs() << "Constructing PBQP::Vector "
+    //              << this << " (length " << Length << ", fill "
+    //              << InitVal << ")\n";
+    std::fill(Data, Data + Length, InitVal);
+  }
 
-    /// \brief Construct a PBQP vector with initializer.
-    Vector(unsigned length, PBQPNum initVal) :
-      length(length), data(new PBQPNum[length]) {
-        std::fill(data, data + length, initVal);
-      }
+  /// \brief Copy construct a PBQP vector.
+  Vector(const Vector &V)
+    : Length(V.Length), Data(new PBQPNum[Length]) {
+    // llvm::dbgs() << "Copy-constructing PBQP::Vector " << this
+    //              << " from PBQP::Vector " << &V << "\n";
+    std::copy(V.Data, V.Data + Length, Data);
+  }
 
-    /// \brief Copy construct a PBQP vector.
-    Vector(const Vector &v) :
-      length(v.length), data(new PBQPNum[length]) {
-        std::copy(v.data, v.data + length, data);
-      }
+  /// \brief Move construct a PBQP vector.
+  Vector(Vector &&V)
+    : Length(V.Length), Data(V.Data) {
+    V.Length = 0;
+    V.Data = nullptr;
+  }
 
-    /// \brief Destroy this vector, return its memory.
-    ~Vector() { delete[] data; }
+  /// \brief Destroy this vector, return its memory.
+  ~Vector() {
+    // llvm::dbgs() << "Deleting PBQP::Vector " << this << "\n";
+    delete[] Data;
+  }
 
-    /// \brief Assignment operator.
-    Vector& operator=(const Vector &v) {
-      delete[] data;
-      length = v.length;
-      data = new PBQPNum[length];
-      std::copy(v.data, v.data + length, data);
-      return *this;
-    }
+  /// \brief Copy-assignment operator.
+  Vector& operator=(const Vector &V) {
+    // llvm::dbgs() << "Assigning to PBQP::Vector " << this
+    //              << " from PBQP::Vector " << &V << "\n";
+    delete[] Data;
+    Length = V.Length;
+    Data = new PBQPNum[Length];
+    std::copy(V.Data, V.Data + Length, Data);
+    return *this;
+  }
 
-    /// \brief Return the length of the vector
-    unsigned getLength() const {
-      return length;
-    }
+  /// \brief Move-assignment operator.
+  Vector& operator=(Vector &&V) {
+    delete[] Data;
+    Length = V.Length;
+    Data = V.Data;
+    V.Length = 0;
+    V.Data = nullptr;
+    return *this;
+  }
 
-    /// \brief Element access.
-    PBQPNum& operator[](unsigned index) {
-      assert(index < length && "Vector element access out of bounds.");
-      return data[index];
-    }
+  /// \brief Comparison operator.
+  bool operator==(const Vector &V) const {
+    assert(Length != 0 && Data != nullptr && "Invalid vector");
+    if (Length != V.Length)
+      return false;
+    return std::equal(Data, Data + Length, V.Data);
+  }
+
+  /// \brief Return the length of the vector
+  unsigned getLength() const {
+    assert(Length != 0 && Data != nullptr && "Invalid vector");
+    return Length;
+  }
+
+  /// \brief Element access.
+  PBQPNum& operator[](unsigned Index) {
+    assert(Length != 0 && Data != nullptr && "Invalid vector");
+    assert(Index < Length && "Vector element access out of bounds.");
+    return Data[Index];
+  }
+
+  /// \brief Const element access.
+  const PBQPNum& operator[](unsigned Index) const {
+    assert(Length != 0 && Data != nullptr && "Invalid vector");
+    assert(Index < Length && "Vector element access out of bounds.");
+    return Data[Index];
+  }
 
-    /// \brief Const element access.
-    const PBQPNum& operator[](unsigned index) const {
-      assert(index < length && "Vector element access out of bounds.");
-      return data[index];
-    }
+  /// \brief Add another vector to this one.
+  Vector& operator+=(const Vector &V) {
+    assert(Length != 0 && Data != nullptr && "Invalid vector");
+    assert(Length == V.Length && "Vector length mismatch.");
+    std::transform(Data, Data + Length, V.Data, Data, std::plus<PBQPNum>());
+    return *this;
+  }
 
-    /// \brief Add another vector to this one.
-    Vector& operator+=(const Vector &v) {
-      assert(length == v.length && "Vector length mismatch.");
-      std::transform(data, data + length, v.data, data, std::plus<PBQPNum>()); 
-      return *this;
-    }
+  /// \brief Subtract another vector from this one.
+  Vector& operator-=(const Vector &V) {
+    assert(Length != 0 && Data != nullptr && "Invalid vector");
+    assert(Length == V.Length && "Vector length mismatch.");
+    std::transform(Data, Data + Length, V.Data, Data, std::minus<PBQPNum>());
+    return *this;
+  }
 
-    /// \brief Subtract another vector from this one.
-    Vector& operator-=(const Vector &v) {
-      assert(length == v.length && "Vector length mismatch.");
-      std::transform(data, data + length, v.data, data, std::minus<PBQPNum>()); 
-      return *this;
-    }
+  /// \brief Returns the index of the minimum value in this vector
+  unsigned minIndex() const {
+    assert(Length != 0 && Data != nullptr && "Invalid vector");
+    return std::min_element(Data, Data + Length) - Data;
+  }
 
-    /// \brief Returns the index of the minimum value in this vector
-    unsigned minIndex() const {
-      return std::min_element(data, data + length) - data;
-    }
+private:
+  unsigned Length;
+  PBQPNum *Data;
+};
 
-  private:
-    unsigned length;
-    PBQPNum *data;
+class VectorComparator {
+public:
+  bool operator()(const Vector &A, const Vector &B) {
+    if (A.Length < B.Length)
+      return true;
+    if (B.Length < A.Length)
+      return false;
+    char *AData = reinterpret_cast<char*>(A.Data);
+    char *BData = reinterpret_cast<char*>(B.Data);
+    return std::lexicographical_compare(AData,
+                                        AData + A.Length * sizeof(PBQPNum),
+                                        BData,
+                                        BData + A.Length * sizeof(PBQPNum));
+  }
 };
 
 /// \brief Output a textual representation of the given vector on the given
 ///        output stream.
 template <typename OStream>
-OStream& operator<<(OStream &os, const Vector &v) {
-  assert((v.getLength() != 0) && "Zero-length vector badness.");
+OStream& operator<<(OStream &OS, const Vector &V) {
+  assert((V.getLength() != 0) && "Zero-length vector badness.");
 
-  os << "[ " << v[0];
-  for (unsigned i = 1; i < v.getLength(); ++i) {
-    os << ", " << v[i];
-  }
-  os << " ]";
+  OS << "[ " << V[0];
+  for (unsigned i = 1; i < V.getLength(); ++i)
+    OS << ", " << V[i];
+  OS << " ]";
 
-  return os;
-} 
+  return OS;
+}
 
 
 /// \brief PBQP Matrix class
 class Matrix {
-  public:
+private:
+  friend class MatrixComparator;
+public:
 
-    /// \brief Construct a PBQP Matrix with the given dimensions.
-    Matrix(unsigned rows, unsigned cols) :
-      rows(rows), cols(cols), data(new PBQPNum[rows * cols]) {
-    }
+  /// \brief Construct a PBQP Matrix with the given dimensions.
+  Matrix(unsigned Rows, unsigned Cols) :
+    Rows(Rows), Cols(Cols), Data(new PBQPNum[Rows * Cols]) {
+  }
 
-    /// \brief Construct a PBQP Matrix with the given dimensions and initial
-    /// value.
-    Matrix(unsigned rows, unsigned cols, PBQPNum initVal) :
-      rows(rows), cols(cols), data(new PBQPNum[rows * cols]) {
-        std::fill(data, data + (rows * cols), initVal);
-    }
+  /// \brief Construct a PBQP Matrix with the given dimensions and initial
+  /// value.
+  Matrix(unsigned Rows, unsigned Cols, PBQPNum InitVal)
+    : Rows(Rows), Cols(Cols), Data(new PBQPNum[Rows * Cols]) {
+    std::fill(Data, Data + (Rows * Cols), InitVal);
+  }
 
-    /// \brief Copy construct a PBQP matrix.
-    Matrix(const Matrix &m) :
-      rows(m.rows), cols(m.cols), data(new PBQPNum[rows * cols]) {
-        std::copy(m.data, m.data + (rows * cols), data);  
-    }
+  /// \brief Copy construct a PBQP matrix.
+  Matrix(const Matrix &M)
+    : Rows(M.Rows), Cols(M.Cols), Data(new PBQPNum[Rows * Cols]) {
+    std::copy(M.Data, M.Data + (Rows * Cols), Data);
+  }
 
-    /// \brief Destroy this matrix, return its memory.
-    ~Matrix() { delete[] data; }
+  /// \brief Move construct a PBQP matrix.
+  Matrix(Matrix &&M)
+    : Rows(M.Rows), Cols(M.Cols), Data(M.Data) {
+    M.Rows = M.Cols = 0;
+    M.Data = nullptr;
+  }
 
-    /// \brief Assignment operator.
-    Matrix& operator=(const Matrix &m) {
-      delete[] data;
-      rows = m.rows; cols = m.cols;
-      data = new PBQPNum[rows * cols];
-      std::copy(m.data, m.data + (rows * cols), data);
-      return *this;
-    }
+  /// \brief Destroy this matrix, return its memory.
+  ~Matrix() { delete[] Data; }
 
-    /// \brief Return the number of rows in this matrix.
-    unsigned getRows() const { return rows; }
+  /// \brief Copy-assignment operator.
+  Matrix& operator=(const Matrix &M) {
+    delete[] Data;
+    Rows = M.Rows; Cols = M.Cols;
+    Data = new PBQPNum[Rows * Cols];
+    std::copy(M.Data, M.Data + (Rows * Cols), Data);
+    return *this;
+  }
 
-    /// \brief Return the number of cols in this matrix.
-    unsigned getCols() const { return cols; }
+  /// \brief Move-assignment operator.
+  Matrix& operator=(Matrix &&M) {
+    delete[] Data;
+    Rows = M.Rows;
+    Cols = M.Cols;
+    Data = M.Data;
+    M.Rows = M.Cols = 0;
+    M.Data = nullptr;
+    return *this;
+  }
 
-    /// \brief Matrix element access.
-    PBQPNum* operator[](unsigned r) {
-      assert(r < rows && "Row out of bounds.");
-      return data + (r * cols);
-    }
+  /// \brief Comparison operator.
+  bool operator==(const Matrix &M) const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    if (Rows != M.Rows || Cols != M.Cols)
+      return false;
+    return std::equal(Data, Data + (Rows * Cols), M.Data);
+  }
 
-    /// \brief Matrix element access.
-    const PBQPNum* operator[](unsigned r) const {
-      assert(r < rows && "Row out of bounds.");
-      return data + (r * cols);
-    }
+  /// \brief Return the number of rows in this matrix.
+  unsigned getRows() const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    return Rows;
+  }
 
-    /// \brief Returns the given row as a vector.
-    Vector getRowAsVector(unsigned r) const {
-      Vector v(cols);
-      for (unsigned c = 0; c < cols; ++c)
-        v[c] = (*this)[r][c];
-      return v; 
-    }
+  /// \brief Return the number of cols in this matrix.
+  unsigned getCols() const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    return Cols;
+  }
 
-    /// \brief Returns the given column as a vector.
-    Vector getColAsVector(unsigned c) const {
-      Vector v(rows);
-      for (unsigned r = 0; r < rows; ++r)
-        v[r] = (*this)[r][c];
-      return v;
-    }
+  /// \brief Matrix element access.
+  PBQPNum* operator[](unsigned R) {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    assert(R < Rows && "Row out of bounds.");
+    return Data + (R * Cols);
+  }
 
-    /// \brief Reset the matrix to the given value.
-    Matrix& reset(PBQPNum val = 0) {
-      std::fill(data, data + (rows * cols), val);
-      return *this;
-    }
+  /// \brief Matrix element access.
+  const PBQPNum* operator[](unsigned R) const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    assert(R < Rows && "Row out of bounds.");
+    return Data + (R * Cols);
+  }
 
-    /// \brief Set a single row of this matrix to the given value.
-    Matrix& setRow(unsigned r, PBQPNum val) {
-      assert(r < rows && "Row out of bounds.");
-      std::fill(data + (r * cols), data + ((r + 1) * cols), val);
-      return *this;
-    }
+  /// \brief Returns the given row as a vector.
+  Vector getRowAsVector(unsigned R) const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    Vector V(Cols);
+    for (unsigned C = 0; C < Cols; ++C)
+      V[C] = (*this)[R][C];
+    return V;
+  }
 
-    /// \brief Set a single column of this matrix to the given value.
-    Matrix& setCol(unsigned c, PBQPNum val) {
-      assert(c < cols && "Column out of bounds.");
-      for (unsigned r = 0; r < rows; ++r)
-        (*this)[r][c] = val;
-      return *this;
-    }
+  /// \brief Returns the given column as a vector.
+  Vector getColAsVector(unsigned C) const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    Vector V(Rows);
+    for (unsigned R = 0; R < Rows; ++R)
+      V[R] = (*this)[R][C];
+    return V;
+  }
 
-    /// \brief Matrix transpose.
-    Matrix transpose() const {
-      Matrix m(cols, rows);
-      for (unsigned r = 0; r < rows; ++r)
-        for (unsigned c = 0; c < cols; ++c)
-          m[c][r] = (*this)[r][c];
-      return m;
-    }
+  /// \brief Reset the matrix to the given value.
+  Matrix& reset(PBQPNum Val = 0) {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    std::fill(Data, Data + (Rows * Cols), Val);
+    return *this;
+  }
 
-    /// \brief Returns the diagonal of the matrix as a vector.
-    ///
-    /// Matrix must be square.
-    Vector diagonalize() const {
-      assert(rows == cols && "Attempt to diagonalize non-square matrix.");
+  /// \brief Set a single row of this matrix to the given value.
+  Matrix& setRow(unsigned R, PBQPNum Val) {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    assert(R < Rows && "Row out of bounds.");
+    std::fill(Data + (R * Cols), Data + ((R + 1) * Cols), Val);
+    return *this;
+  }
 
-      Vector v(rows);
-      for (unsigned r = 0; r < rows; ++r)
-        v[r] = (*this)[r][r];
-      return v;
-    } 
+  /// \brief Set a single column of this matrix to the given value.
+  Matrix& setCol(unsigned C, PBQPNum Val) {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    assert(C < Cols && "Column out of bounds.");
+    for (unsigned R = 0; R < Rows; ++R)
+      (*this)[R][C] = Val;
+    return *this;
+  }
 
-    /// \brief Add the given matrix to this one.
-    Matrix& operator+=(const Matrix &m) {
-      assert(rows == m.rows && cols == m.cols &&
-          "Matrix dimensions mismatch.");
-      std::transform(data, data + (rows * cols), m.data, data,
-          std::plus<PBQPNum>());
-      return *this;
-    }
+  /// \brief Matrix transpose.
+  Matrix transpose() const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    Matrix M(Cols, Rows);
+    for (unsigned r = 0; r < Rows; ++r)
+      for (unsigned c = 0; c < Cols; ++c)
+        M[c][r] = (*this)[r][c];
+    return M;
+  }
 
-    /// \brief Returns the minimum of the given row
-    PBQPNum getRowMin(unsigned r) const {
-      assert(r < rows && "Row out of bounds");
-      return *std::min_element(data + (r * cols), data + ((r + 1) * cols));
-    }
+  /// \brief Returns the diagonal of the matrix as a vector.
+  ///
+  /// Matrix must be square.
+  Vector diagonalize() const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    assert(Rows == Cols && "Attempt to diagonalize non-square matrix.");
+    Vector V(Rows);
+    for (unsigned r = 0; r < Rows; ++r)
+      V[r] = (*this)[r][r];
+    return V;
+  }
 
-    /// \brief Returns the minimum of the given column
-    PBQPNum getColMin(unsigned c) const {
-      PBQPNum minElem = (*this)[0][c];
-      for (unsigned r = 1; r < rows; ++r)
-        if ((*this)[r][c] < minElem) minElem = (*this)[r][c];
-      return minElem;
-    }
+  /// \brief Add the given matrix to this one.
+  Matrix& operator+=(const Matrix &M) {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    assert(Rows == M.Rows && Cols == M.Cols &&
+           "Matrix dimensions mismatch.");
+    std::transform(Data, Data + (Rows * Cols), M.Data, Data,
+                   std::plus<PBQPNum>());
+    return *this;
+  }
 
-    /// \brief Subtracts the given scalar from the elements of the given row.
-    Matrix& subFromRow(unsigned r, PBQPNum val) {
-      assert(r < rows && "Row out of bounds");
-      std::transform(data + (r * cols), data + ((r + 1) * cols),
-          data + (r * cols),
-          std::bind2nd(std::minus<PBQPNum>(), val));
-      return *this;
-    }
+  Matrix operator+(const Matrix &M) {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    Matrix Tmp(*this);
+    Tmp += M;
+    return Tmp;
+  }
 
-    /// \brief Subtracts the given scalar from the elements of the given column.
-    Matrix& subFromCol(unsigned c, PBQPNum val) {
-      for (unsigned r = 0; r < rows; ++r)
-        (*this)[r][c] -= val;
-      return *this;
-    }
+  /// \brief Returns the minimum of the given row
+  PBQPNum getRowMin(unsigned R) const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    assert(R < Rows && "Row out of bounds");
+    return *std::min_element(Data + (R * Cols), Data + ((R + 1) * Cols));
+  }
 
-    /// \brief Returns true if this is a zero matrix.
-    bool isZero() const {
-      return find_if(data, data + (rows * cols),
-          std::bind2nd(std::not_equal_to<PBQPNum>(), 0)) ==
-        data + (rows * cols);
-    }
+  /// \brief Returns the minimum of the given column
+  PBQPNum getColMin(unsigned C) const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    PBQPNum MinElem = (*this)[0][C];
+    for (unsigned R = 1; R < Rows; ++R)
+      if ((*this)[R][C] < MinElem)
+        MinElem = (*this)[R][C];
+    return MinElem;
+  }
 
-  private:
-    unsigned rows, cols;
-    PBQPNum *data;
+  /// \brief Subtracts the given scalar from the elements of the given row.
+  Matrix& subFromRow(unsigned R, PBQPNum Val) {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    assert(R < Rows && "Row out of bounds");
+    std::transform(Data + (R * Cols), Data + ((R + 1) * Cols),
+                   Data + (R * Cols),
+                   std::bind2nd(std::minus<PBQPNum>(), Val));
+    return *this;
+  }
+
+  /// \brief Subtracts the given scalar from the elements of the given column.
+  Matrix& subFromCol(unsigned C, PBQPNum Val) {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    for (unsigned R = 0; R < Rows; ++R)
+      (*this)[R][C] -= Val;
+    return *this;
+  }
+
+  /// \brief Returns true if this is a zero matrix.
+  bool isZero() const {
+    assert(Rows != 0 && Cols != 0 && Data != nullptr && "Invalid matrix");
+    return find_if(Data, Data + (Rows * Cols),
+                   std::bind2nd(std::not_equal_to<PBQPNum>(), 0)) ==
+      Data + (Rows * Cols);
+  }
+
+private:
+  unsigned Rows, Cols;
+  PBQPNum *Data;
+};
+
+class MatrixComparator {
+public:
+  bool operator()(const Matrix &A, const Matrix &B) {
+    if (A.Rows < B.Rows)
+      return true;
+    if (B.Rows < A.Rows)
+      return false;
+    if (A.Cols < B.Cols)
+      return true;
+    if (B.Cols < A.Cols)
+      return false;
+    char *AData = reinterpret_cast<char*>(A.Data);
+    char *BData = reinterpret_cast<char*>(B.Data);
+    return std::lexicographical_compare(
+             AData, AData + (A.Rows * A.Cols * sizeof(PBQPNum)),
+             BData, BData + (A.Rows * A.Cols * sizeof(PBQPNum)));
+  }
 };
 
 /// \brief Output a textual representation of the given matrix on the given
 ///        output stream.
 template <typename OStream>
-OStream& operator<<(OStream &os, const Matrix &m) {
-
-  assert((m.getRows() != 0) && "Zero-row matrix badness.");
+OStream& operator<<(OStream &OS, const Matrix &M) {
+  assert((M.getRows() != 0) && "Zero-row matrix badness.");
+  for (unsigned i = 0; i < M.getRows(); ++i)
+    OS << M.getRowAsVector(i);
+  return OS;
+}
 
-  for (unsigned i = 0; i < m.getRows(); ++i) {
-    os << m.getRowAsVector(i);
-  }
+template <typename Metadata>
+class MDVector : public Vector {
+public:
+  MDVector(const Vector &v) : Vector(v), md(*this) { }
+  MDVector(Vector &&v) : Vector(std::move(v)), md(*this) { }
+  const Metadata& getMetadata() const { return md; }
+private:
+  Metadata md;
+};
 
-  return os;
-}
+template <typename Metadata>
+class MDMatrix : public Matrix {
+public:
+  MDMatrix(const Matrix &m) : Matrix(m), md(*this) { }
+  MDMatrix(Matrix &&m) : Matrix(std::move(m)), md(*this) { }
+  const Metadata& getMetadata() const { return md; }
+private:
+  Metadata md;
+};
 
 }
 
diff --git a/include/llvm/CodeGen/PBQP/ReductionRules.h b/include/llvm/CodeGen/PBQP/ReductionRules.h
new file mode 100644
index 000000000000..a55a06033c4e
--- /dev/null
+++ b/include/llvm/CodeGen/PBQP/ReductionRules.h
@@ -0,0 +1,191 @@
+//===----------- ReductionRules.h - Reduction Rules -------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Reduction Rules.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_REDUCTIONRULES_H
+#define LLVM_REDUCTIONRULES_H
+
+#include "Graph.h"
+#include "Math.h"
+#include "Solution.h"
+
+namespace PBQP {
+
+  /// \brief Reduce a node of degree one.
+  ///
+  /// Propagate costs from the given node, which must be of degree one, to its
+  /// neighbor. Notify the problem domain.
+  template <typename GraphT>
+  void applyR1(GraphT &G, typename GraphT::NodeId NId) {
+    typedef typename GraphT::NodeId NodeId;
+    typedef typename GraphT::EdgeId EdgeId;
+    typedef typename GraphT::Vector Vector;
+    typedef typename GraphT::Matrix Matrix;
+    typedef typename GraphT::RawVector RawVector;
+
+    assert(G.getNodeDegree(NId) == 1 &&
+           "R1 applied to node with degree != 1.");
+
+    EdgeId EId = *G.adjEdgeIds(NId).begin();
+    NodeId MId = G.getEdgeOtherNodeId(EId, NId);
+
+    const Matrix &ECosts = G.getEdgeCosts(EId);
+    const Vector &XCosts = G.getNodeCosts(NId);
+    RawVector YCosts = G.getNodeCosts(MId);
+
+    // Duplicate a little to avoid transposing matrices.
+    if (NId == G.getEdgeNode1Id(EId)) {
+      for (unsigned j = 0; j < YCosts.getLength(); ++j) {
+        PBQPNum Min = ECosts[0][j] + XCosts[0];
+        for (unsigned i = 1; i < XCosts.getLength(); ++i) {
+          PBQPNum C = ECosts[i][j] + XCosts[i];
+          if (C < Min)
+            Min = C;
+        }
+        YCosts[j] += Min;
+      }
+    } else {
+      for (unsigned i = 0; i < YCosts.getLength(); ++i) {
+        PBQPNum Min = ECosts[i][0] + XCosts[0];
+        for (unsigned j = 1; j < XCosts.getLength(); ++j) {
+          PBQPNum C = ECosts[i][j] + XCosts[j];
+          if (C < Min)
+            Min = C;
+        }
+        YCosts[i] += Min;
+      }
+    }
+    G.setNodeCosts(MId, YCosts);
+    G.disconnectEdge(EId, MId);
+  }
+
+  template <typename GraphT>
+  void applyR2(GraphT &G, typename GraphT::NodeId NId) {
+    typedef typename GraphT::NodeId NodeId;
+    typedef typename GraphT::EdgeId EdgeId;
+    typedef typename GraphT::Vector Vector;
+    typedef typename GraphT::Matrix Matrix;
+    typedef typename GraphT::RawMatrix RawMatrix;
+
+    assert(G.getNodeDegree(NId) == 2 &&
+           "R2 applied to node with degree != 2.");
+
+    const Vector &XCosts = G.getNodeCosts(NId);
+
+    typename GraphT::AdjEdgeItr AEItr = G.adjEdgeIds(NId).begin();
+    EdgeId YXEId = *AEItr,
+           ZXEId = *(++AEItr);
+
+    NodeId YNId = G.getEdgeOtherNodeId(YXEId, NId),
+           ZNId = G.getEdgeOtherNodeId(ZXEId, NId);
+
+    bool FlipEdge1 = (G.getEdgeNode1Id(YXEId) == NId),
+         FlipEdge2 = (G.getEdgeNode1Id(ZXEId) == NId);
+
+    const Matrix *YXECosts = FlipEdge1 ?
+      new Matrix(G.getEdgeCosts(YXEId).transpose()) :
+      &G.getEdgeCosts(YXEId);
+
+    const Matrix *ZXECosts = FlipEdge2 ?
+      new Matrix(G.getEdgeCosts(ZXEId).transpose()) :
+      &G.getEdgeCosts(ZXEId);
+
+    unsigned XLen = XCosts.getLength(),
+      YLen = YXECosts->getRows(),
+      ZLen = ZXECosts->getRows();
+
+    RawMatrix Delta(YLen, ZLen);
+
+    for (unsigned i = 0; i < YLen; ++i) {
+      for (unsigned j = 0; j < ZLen; ++j) {
+        PBQPNum Min = (*YXECosts)[i][0] + (*ZXECosts)[j][0] + XCosts[0];
+        for (unsigned k = 1; k < XLen; ++k) {
+          PBQPNum C = (*YXECosts)[i][k] + (*ZXECosts)[j][k] + XCosts[k];
+          if (C < Min) {
+            Min = C;
+          }
+        }
+        Delta[i][j] = Min;
+      }
+    }
+
+    if (FlipEdge1)
+      delete YXECosts;
+
+    if (FlipEdge2)
+      delete ZXECosts;
+
+    EdgeId YZEId = G.findEdge(YNId, ZNId);
+
+    if (YZEId == G.invalidEdgeId()) {
+      YZEId = G.addEdge(YNId, ZNId, Delta);
+    } else {
+      const Matrix &YZECosts = G.getEdgeCosts(YZEId);
+      if (YNId == G.getEdgeNode1Id(YZEId)) {
+        G.setEdgeCosts(YZEId, Delta + YZECosts);
+      } else {
+        G.setEdgeCosts(YZEId, Delta.transpose() + YZECosts);
+      }
+    }
+
+    G.disconnectEdge(YXEId, YNId);
+    G.disconnectEdge(ZXEId, ZNId);
+
+    // TODO: Try to normalize newly added/modified edge.
+  }
+
+
+  // \brief Find a solution to a fully reduced graph by backpropagation.
+  //
+  // Given a graph and a reduction order, pop each node from the reduction
+  // order and greedily compute a minimum solution based on the node costs, and
+  // the dependent costs due to previously solved nodes.
+  //
+  // Note - This does not return the graph to its original (pre-reduction)
+  //        state: the existing solvers destructively alter the node and edge
+  //        costs. Given that, the backpropagate function doesn't attempt to
+  //        replace the edges either, but leaves the graph in its reduced
+  //        state.
+  template <typename GraphT, typename StackT>
+  Solution backpropagate(GraphT& G, StackT stack) {
+    typedef GraphBase::NodeId NodeId;
+    typedef typename GraphT::Matrix Matrix;
+    typedef typename GraphT::RawVector RawVector;
+
+    Solution s;
+
+    while (!stack.empty()) {
+      NodeId NId = stack.back();
+      stack.pop_back();
+
+      RawVector v = G.getNodeCosts(NId);
+
+      for (auto EId : G.adjEdgeIds(NId)) {
+        const Matrix& edgeCosts = G.getEdgeCosts(EId);
+        if (NId == G.getEdgeNode1Id(EId)) {
+          NodeId mId = G.getEdgeNode2Id(EId);
+          v += edgeCosts.getColAsVector(s.getSelection(mId));
+        } else {
+          NodeId mId = G.getEdgeNode1Id(EId);
+          v += edgeCosts.getRowAsVector(s.getSelection(mId));
+        }
+      }
+
+      s.setSelection(NId, v.minIndex());
+    }
+
+    return s;
+  }
+
+}
+
+#endif // LLVM_REDUCTIONRULES_H
diff --git a/include/llvm/CodeGen/PBQP/RegAllocSolver.h b/include/llvm/CodeGen/PBQP/RegAllocSolver.h
new file mode 100644
index 000000000000..977c34843bbd
--- /dev/null
+++ b/include/llvm/CodeGen/PBQP/RegAllocSolver.h
@@ -0,0 +1,359 @@
+//===-- RegAllocSolver.h - Heuristic PBQP Solver for reg alloc --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Heuristic PBQP solver for register allocation problems. This solver uses a
+// graph reduction approach. Nodes of degree 0, 1 and 2 are eliminated with
+// optimality-preserving rules (see ReductionRules.h). When no low-degree (<3)
+// nodes are present, a heuristic derived from Brigg's graph coloring approach
+// is used.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H
+#define LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H
+
+#include "CostAllocator.h"
+#include "Graph.h"
+#include "ReductionRules.h"
+#include "Solution.h"
+#include "llvm/Support/ErrorHandling.h"
+#include <limits>
+#include <vector>
+
+namespace PBQP {
+
+  namespace RegAlloc {
+
+    /// \brief Metadata to speed allocatability test.
+    ///
+    /// Keeps track of the number of infinities in each row and column.
+    class MatrixMetadata {
+    private:
+      MatrixMetadata(const MatrixMetadata&);
+      void operator=(const MatrixMetadata&);
+    public:
+      MatrixMetadata(const PBQP::Matrix& M)
+        : WorstRow(0), WorstCol(0),
+          UnsafeRows(new bool[M.getRows() - 1]()),
+          UnsafeCols(new bool[M.getCols() - 1]()) {
+
+        unsigned* ColCounts = new unsigned[M.getCols() - 1]();
+
+        for (unsigned i = 1; i < M.getRows(); ++i) {
+          unsigned RowCount = 0;
+          for (unsigned j = 1; j < M.getCols(); ++j) {
+            if (M[i][j] == std::numeric_limits<PBQP::PBQPNum>::infinity()) {
+              ++RowCount;
+              ++ColCounts[j - 1];
+              UnsafeRows[i - 1] = true;
+              UnsafeCols[j - 1] = true;
+            }
+          }
+          WorstRow = std::max(WorstRow, RowCount);
+        }
+        unsigned WorstColCountForCurRow =
+          *std::max_element(ColCounts, ColCounts + M.getCols() - 1);
+        WorstCol = std::max(WorstCol, WorstColCountForCurRow);
+        delete[] ColCounts;
+      }
+
+      ~MatrixMetadata() {
+        delete[] UnsafeRows;
+        delete[] UnsafeCols;
+      }
+
+      unsigned getWorstRow() const { return WorstRow; }
+      unsigned getWorstCol() const { return WorstCol; }
+      const bool* getUnsafeRows() const { return UnsafeRows; }
+      const bool* getUnsafeCols() const { return UnsafeCols; }
+
+    private:
+      unsigned WorstRow, WorstCol;
+      bool* UnsafeRows;
+      bool* UnsafeCols;
+    };
+
+    class NodeMetadata {
+    public:
+      typedef enum { Unprocessed,
+                     OptimallyReducible,
+                     ConservativelyAllocatable,
+                     NotProvablyAllocatable } ReductionState;
+
+      NodeMetadata() : RS(Unprocessed), DeniedOpts(0), OptUnsafeEdges(nullptr){}
+      ~NodeMetadata() { delete[] OptUnsafeEdges; }
+
+      void setup(const Vector& Costs) {
+        NumOpts = Costs.getLength() - 1;
+        OptUnsafeEdges = new unsigned[NumOpts]();
+      }
+
+      ReductionState getReductionState() const { return RS; }
+      void setReductionState(ReductionState RS) { this->RS = RS; }
+
+      void handleAddEdge(const MatrixMetadata& MD, bool Transpose) {
+        DeniedOpts += Transpose ? MD.getWorstCol() : MD.getWorstRow();
+        const bool* UnsafeOpts =
+          Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows();
+        for (unsigned i = 0; i < NumOpts; ++i)
+          OptUnsafeEdges[i] += UnsafeOpts[i];
+      }
+
+      void handleRemoveEdge(const MatrixMetadata& MD, bool Transpose) {
+        DeniedOpts -= Transpose ? MD.getWorstCol() : MD.getWorstRow();
+        const bool* UnsafeOpts =
+          Transpose ? MD.getUnsafeCols() : MD.getUnsafeRows();
+        for (unsigned i = 0; i < NumOpts; ++i)
+          OptUnsafeEdges[i] -= UnsafeOpts[i];
+      }
+
+      bool isConservativelyAllocatable() const {
+        return (DeniedOpts < NumOpts) ||
+               (std::find(OptUnsafeEdges, OptUnsafeEdges + NumOpts, 0) !=
+                  OptUnsafeEdges + NumOpts);
+      }
+
+    private:
+      ReductionState RS;
+      unsigned NumOpts;
+      unsigned DeniedOpts;
+      unsigned* OptUnsafeEdges;
+    };
+
+    class RegAllocSolverImpl {
+    private:
+      typedef PBQP::MDMatrix<MatrixMetadata> RAMatrix;
+    public:
+      typedef PBQP::Vector RawVector;
+      typedef PBQP::Matrix RawMatrix;
+      typedef PBQP::Vector Vector;
+      typedef RAMatrix     Matrix;
+      typedef PBQP::PoolCostAllocator<
+                Vector, PBQP::VectorComparator,
+                Matrix, PBQP::MatrixComparator> CostAllocator;
+
+      typedef PBQP::GraphBase::NodeId NodeId;
+      typedef PBQP::GraphBase::EdgeId EdgeId;
+
+      typedef RegAlloc::NodeMetadata NodeMetadata;
+
+      struct EdgeMetadata { };
+
+      typedef PBQP::Graph<RegAllocSolverImpl> Graph;
+
+      RegAllocSolverImpl(Graph &G) : G(G) {}
+
+      Solution solve() {
+        G.setSolver(*this);
+        Solution S;
+        setup();
+        S = backpropagate(G, reduce());
+        G.unsetSolver();
+        return S;
+      }
+
+      void handleAddNode(NodeId NId) {
+        G.getNodeMetadata(NId).setup(G.getNodeCosts(NId));
+      }
+      void handleRemoveNode(NodeId NId) {}
+      void handleSetNodeCosts(NodeId NId, const Vector& newCosts) {}
+
+      void handleAddEdge(EdgeId EId) {
+        handleReconnectEdge(EId, G.getEdgeNode1Id(EId));
+        handleReconnectEdge(EId, G.getEdgeNode2Id(EId));
+      }
+
+      void handleRemoveEdge(EdgeId EId) {
+        handleDisconnectEdge(EId, G.getEdgeNode1Id(EId));
+        handleDisconnectEdge(EId, G.getEdgeNode2Id(EId));
+      }
+
+      void handleDisconnectEdge(EdgeId EId, NodeId NId) {
+        NodeMetadata& NMd = G.getNodeMetadata(NId);
+        const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata();
+        NMd.handleRemoveEdge(MMd, NId == G.getEdgeNode2Id(EId));
+        if (G.getNodeDegree(NId) == 3) {
+          // This node is becoming optimally reducible.
+          moveToOptimallyReducibleNodes(NId);
+        } else if (NMd.getReductionState() ==
+                     NodeMetadata::NotProvablyAllocatable &&
+                   NMd.isConservativelyAllocatable()) {
+          // This node just became conservatively allocatable.
+          moveToConservativelyAllocatableNodes(NId);
+        }
+      }
+
+      void handleReconnectEdge(EdgeId EId, NodeId NId) {
+        NodeMetadata& NMd = G.getNodeMetadata(NId);
+        const MatrixMetadata& MMd = G.getEdgeCosts(EId).getMetadata();
+        NMd.handleAddEdge(MMd, NId == G.getEdgeNode2Id(EId));
+      }
+
+      void handleSetEdgeCosts(EdgeId EId, const Matrix& NewCosts) {
+        handleRemoveEdge(EId);
+
+        NodeId N1Id = G.getEdgeNode1Id(EId);
+        NodeId N2Id = G.getEdgeNode2Id(EId);
+        NodeMetadata& N1Md = G.getNodeMetadata(N1Id);
+        NodeMetadata& N2Md = G.getNodeMetadata(N2Id);
+        const MatrixMetadata& MMd = NewCosts.getMetadata();
+        N1Md.handleAddEdge(MMd, N1Id != G.getEdgeNode1Id(EId));
+        N2Md.handleAddEdge(MMd, N2Id != G.getEdgeNode1Id(EId));
+      }
+
+    private:
+
+      void removeFromCurrentSet(NodeId NId) {
+        switch (G.getNodeMetadata(NId).getReductionState()) {
+          case NodeMetadata::Unprocessed: break;
+          case NodeMetadata::OptimallyReducible:
+            assert(OptimallyReducibleNodes.find(NId) !=
+                     OptimallyReducibleNodes.end() &&
+                   "Node not in optimally reducible set.");
+            OptimallyReducibleNodes.erase(NId);
+            break;
+          case NodeMetadata::ConservativelyAllocatable:
+            assert(ConservativelyAllocatableNodes.find(NId) !=
+                     ConservativelyAllocatableNodes.end() &&
+                   "Node not in conservatively allocatable set.");
+            ConservativelyAllocatableNodes.erase(NId);
+            break;
+          case NodeMetadata::NotProvablyAllocatable:
+            assert(NotProvablyAllocatableNodes.find(NId) !=
+                     NotProvablyAllocatableNodes.end() &&
+                   "Node not in not-provably-allocatable set.");
+            NotProvablyAllocatableNodes.erase(NId);
+            break;
+        }
+      }
+
+      void moveToOptimallyReducibleNodes(NodeId NId) {
+        removeFromCurrentSet(NId);
+        OptimallyReducibleNodes.insert(NId);
+        G.getNodeMetadata(NId).setReductionState(
+          NodeMetadata::OptimallyReducible);
+      }
+
+      void moveToConservativelyAllocatableNodes(NodeId NId) {
+        removeFromCurrentSet(NId);
+        ConservativelyAllocatableNodes.insert(NId);
+        G.getNodeMetadata(NId).setReductionState(
+          NodeMetadata::ConservativelyAllocatable);
+      }
+
+      void moveToNotProvablyAllocatableNodes(NodeId NId) {
+        removeFromCurrentSet(NId);
+        NotProvablyAllocatableNodes.insert(NId);
+        G.getNodeMetadata(NId).setReductionState(
+          NodeMetadata::NotProvablyAllocatable);
+      }
+
+      void setup() {
+        // Set up worklists.
+        for (auto NId : G.nodeIds()) {
+          if (G.getNodeDegree(NId) < 3)
+            moveToOptimallyReducibleNodes(NId);
+          else if (G.getNodeMetadata(NId).isConservativelyAllocatable())
+            moveToConservativelyAllocatableNodes(NId);
+          else
+            moveToNotProvablyAllocatableNodes(NId);
+        }
+      }
+
+      // Compute a reduction order for the graph by iteratively applying PBQP
+      // reduction rules. Locally optimal rules are applied whenever possible (R0,
+      // R1, R2). If no locally-optimal rules apply then any conservatively
+      // allocatable node is reduced. Finally, if no conservatively allocatable
+      // node exists then the node with the lowest spill-cost:degree ratio is
+      // selected.
+      std::vector<GraphBase::NodeId> reduce() {
+        assert(!G.empty() && "Cannot reduce empty graph.");
+
+        typedef GraphBase::NodeId NodeId;
+        std::vector<NodeId> NodeStack;
+
+        // Consume worklists.
+        while (true) {
+          if (!OptimallyReducibleNodes.empty()) {
+            NodeSet::iterator NItr = OptimallyReducibleNodes.begin();
+            NodeId NId = *NItr;
+            OptimallyReducibleNodes.erase(NItr);
+            NodeStack.push_back(NId);
+            switch (G.getNodeDegree(NId)) {
+              case 0:
+                break;
+              case 1:
+                applyR1(G, NId);
+                break;
+              case 2:
+                applyR2(G, NId);
+                break;
+              default: llvm_unreachable("Not an optimally reducible node.");
+            }
+          } else if (!ConservativelyAllocatableNodes.empty()) {
+            // Conservatively allocatable nodes will never spill. For now just
+            // take the first node in the set and push it on the stack. When we
+            // start optimizing more heavily for register preferencing, it may
+            // would be better to push nodes with lower 'expected' or worst-case
+            // register costs first (since early nodes are the most
+            // constrained).
+            NodeSet::iterator NItr = ConservativelyAllocatableNodes.begin();
+            NodeId NId = *NItr;
+            ConservativelyAllocatableNodes.erase(NItr);
+            NodeStack.push_back(NId);
+            G.disconnectAllNeighborsFromNode(NId);
+
+          } else if (!NotProvablyAllocatableNodes.empty()) {
+            NodeSet::iterator NItr =
+              std::min_element(NotProvablyAllocatableNodes.begin(),
+                               NotProvablyAllocatableNodes.end(),
+                               SpillCostComparator(G));
+            NodeId NId = *NItr;
+            NotProvablyAllocatableNodes.erase(NItr);
+            NodeStack.push_back(NId);
+            G.disconnectAllNeighborsFromNode(NId);
+          } else
+            break;
+        }
+
+        return NodeStack;
+      }
+
+      class SpillCostComparator {
+      public:
+        SpillCostComparator(const Graph& G) : G(G) {}
+        bool operator()(NodeId N1Id, NodeId N2Id) {
+          PBQPNum N1SC = G.getNodeCosts(N1Id)[0] / G.getNodeDegree(N1Id);
+          PBQPNum N2SC = G.getNodeCosts(N2Id)[0] / G.getNodeDegree(N2Id);
+          return N1SC < N2SC;
+        }
+      private:
+        const Graph& G;
+      };
+
+      Graph& G;
+      typedef std::set<NodeId> NodeSet;
+      NodeSet OptimallyReducibleNodes;
+      NodeSet ConservativelyAllocatableNodes;
+      NodeSet NotProvablyAllocatableNodes;
+    };
+
+    typedef Graph<RegAllocSolverImpl> Graph;
+
+    inline Solution solve(Graph& G) {
+      if (G.empty())
+        return Solution();
+      RegAllocSolverImpl RegAllocSolver(G);
+      return RegAllocSolver.solve();
+    }
+
+  }
+}
+
+#endif // LLVM_CODEGEN_PBQP_REGALLOCSOLVER_H
diff --git a/include/llvm/CodeGen/PBQP/Solution.h b/include/llvm/CodeGen/PBQP/Solution.h
index 091805d00f3e..3556e60f3967 100644
--- a/include/llvm/CodeGen/PBQP/Solution.h
+++ b/include/llvm/CodeGen/PBQP/Solution.h
@@ -26,7 +26,7 @@ namespace PBQP {
   class Solution {
   private:
 
-    typedef std::map<Graph::NodeId, unsigned> SelectionsMap;
+    typedef std::map<GraphBase::NodeId, unsigned> SelectionsMap;
     SelectionsMap selections;
 
     unsigned r0Reductions, r1Reductions, r2Reductions, rNReductions;
@@ -72,14 +72,14 @@ namespace PBQP {
     /// \brief Set the selection for a given node.
     /// @param nodeId Node id.
     /// @param selection Selection for nodeId.
-    void setSelection(Graph::NodeId nodeId, unsigned selection) {
+    void setSelection(GraphBase::NodeId nodeId, unsigned selection) {
       selections[nodeId] = selection;
     }
 
     /// \brief Get a node's selection.
     /// @param nodeId Node id.
     /// @return The selection for nodeId;
-    unsigned getSelection(Graph::NodeId nodeId) const {
+    unsigned getSelection(GraphBase::NodeId nodeId) const {
       SelectionsMap::const_iterator sItr = selections.find(nodeId);
       assert(sItr != selections.end() && "No selection for node.");
       return sItr->second;
diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h
index ae4a2fa0bf83..87f55e8572fe 100644
--- a/include/llvm/CodeGen/Passes.h
+++ b/include/llvm/CodeGen/Passes.h
@@ -59,7 +59,7 @@ class IdentifyingPassPtr {
   };
   bool IsInstance;
 public:
-  IdentifyingPassPtr() : P(0), IsInstance(false) {}
+  IdentifyingPassPtr() : P(nullptr), IsInstance(false) {}
   IdentifyingPassPtr(AnalysisID IDPtr) : ID(IDPtr), IsInstance(false) {}
   IdentifyingPassPtr(Pass *InstancePtr) : P(InstancePtr), IsInstance(true) {}
 
@@ -133,10 +133,6 @@ public:
     return *static_cast<TMC*>(TM);
   }
 
-  const TargetLowering *getTargetLowering() const {
-    return TM->getTargetLowering();
-  }
-
   //
   void setInitialized() { Initialized = true; }
 
@@ -151,7 +147,7 @@ public:
   void setStartStopPasses(AnalysisID Start, AnalysisID Stop) {
     StartAfter = Start;
     StopAfter = Stop;
-    Started = (StartAfter == 0);
+    Started = (StartAfter == nullptr);
   }
 
   void setDisableVerify(bool Disable) { setOpt(DisableVerify, Disable); }
@@ -207,9 +203,9 @@ public:
   /// Fully developed targets will not generally override this.
   virtual void addMachinePasses();
 
-  /// createTargetScheduler - Create an instance of ScheduleDAGInstrs to be run
-  /// within the standard MachineScheduler pass for this function and target at
-  /// the current optimization level.
+  /// Create an instance of ScheduleDAGInstrs to be run within the standard
+  /// MachineScheduler pass for this function and target at the current
+  /// optimization level.
   ///
   /// This can also be used to plug a new MachineSchedStrategy into an instance
   /// of the standard ScheduleDAGMI:
@@ -218,7 +214,14 @@ public:
   /// Return NULL to select the default (generic) machine scheduler.
   virtual ScheduleDAGInstrs *
   createMachineScheduler(MachineSchedContext *C) const {
-    return 0;
+    return nullptr;
+  }
+
+  /// Similar to createMachineScheduler but used when postRA machine scheduling
+  /// is enabled.
+  virtual ScheduleDAGInstrs *
+  createPostMachineScheduler(MachineSchedContext *C) const {
+    return nullptr;
   }
 
 protected:
@@ -342,6 +345,8 @@ protected:
 
 /// List of target independent CodeGen pass IDs.
 namespace llvm {
+  FunctionPass *createAtomicExpandLoadLinkedPass(const TargetMachine *TM);
+
   /// \brief Create a basic TargetTransformInfo analysis pass.
   ///
   /// This pass implements the target transform info analysis using the target
@@ -363,12 +368,22 @@ namespace llvm {
   createMachineFunctionPrinterPass(raw_ostream &OS,
                                    const std::string &Banner ="");
 
+  /// createCodeGenPreparePass - Transform the code to expose more pattern
+  /// matching during instruction selection.
+  FunctionPass *createCodeGenPreparePass(const TargetMachine *TM = nullptr);
+
+  /// AtomicExpandLoadLinkedID -- FIXME
+  extern char &AtomicExpandLoadLinkedID;
+
   /// MachineLoopInfo - This pass is a loop analysis pass.
   extern char &MachineLoopInfoID;
 
   /// MachineDominators - This pass is a machine dominators analysis pass.
   extern char &MachineDominatorsID;
 
+/// MachineDominanaceFrontier - This pass is a machine dominators analysis pass.
+  extern char &MachineDominanceFrontierID;
+
   /// EdgeBundles analysis - Bundle machine CFG edges.
   extern char &EdgeBundlesID;
 
@@ -403,6 +418,9 @@ namespace llvm {
   /// MachineScheduler - This pass schedules machine instructions.
   extern char &MachineSchedulerID;
 
+  /// PostMachineScheduler - This pass schedules machine instructions postRA.
+  extern char &PostMachineSchedulerID;
+
   /// SpillPlacement analysis. Suggest optimal placement of spill code between
   /// basic blocks.
   extern char &SpillPlacementID;
@@ -533,7 +551,7 @@ namespace llvm {
   /// createMachineVerifierPass - This pass verifies cenerated machine code
   /// instructions for correctness.
   ///
-  FunctionPass *createMachineVerifierPass(const char *Banner = 0);
+  FunctionPass *createMachineVerifierPass(const char *Banner = nullptr);
 
   /// createDwarfEHPass - This pass mulches exception handling code into a form
   /// adapted to code generation.  Required if using dwarf exception handling.
@@ -568,6 +586,30 @@ namespace llvm {
   /// bundles (created earlier, e.g. during pre-RA scheduling).
   extern char &FinalizeMachineBundlesID;
 
+  /// StackMapLiveness - This pass analyses the register live-out set of
+  /// stackmap/patchpoint intrinsics and attaches the calculated information to
+  /// the intrinsic for later emission to the StackMap.
+  extern char &StackMapLivenessID;
+
+  /// createJumpInstrTables - This pass creates jump-instruction tables.
+  ModulePass *createJumpInstrTablesPass();
 } // End llvm namespace
 
+/// This initializer registers TargetMachine constructor, so the pass being
+/// initialized can use target dependent interfaces. Please do not move this
+/// macro to be together with INITIALIZE_PASS, which is a complete target
+/// independent initializer, and we don't want to make libScalarOpts depend
+/// on libCodeGen.
+#define INITIALIZE_TM_PASS(passName, arg, name, cfg, analysis) \
+  static void* initialize##passName##PassOnce(PassRegistry &Registry) { \
+    PassInfo *PI = new PassInfo(name, arg, & passName ::ID, \
+      PassInfo::NormalCtor_t(callDefaultCtor< passName >), cfg, analysis, \
+      PassInfo::TargetMachineCtor_t(callTargetMachineCtor< passName >)); \
+    Registry.registerPass(*PI, true); \
+    return PI; \
+  } \
+  void llvm::initialize##passName##Pass(PassRegistry &Registry) { \
+    CALL_ONCE_INITIALIZATION(initialize##passName##PassOnce) \
+  }
+
 #endif
diff --git a/include/llvm/CodeGen/PseudoSourceValue.h b/include/llvm/CodeGen/PseudoSourceValue.h
index 705086c22b45..cc3e25aeb89f 100644
--- a/include/llvm/CodeGen/PseudoSourceValue.h
+++ b/include/llvm/CodeGen/PseudoSourceValue.h
@@ -18,21 +18,32 @@
 
 namespace llvm {
   class MachineFrameInfo;
+  class MachineMemOperand;
   class raw_ostream;
 
+  raw_ostream &operator<<(raw_ostream &OS, const MachineMemOperand &MMO);
+
   /// PseudoSourceValue - Special value supplied for machine level alias
   /// analysis. It indicates that a memory access references the functions
   /// stack frame (e.g., a spill slot), below the stack frame (e.g., argument
   /// space), or constant pool.
-  class PseudoSourceValue : public Value {
+  class PseudoSourceValue {
   private:
+    friend raw_ostream &llvm::operator<<(raw_ostream &OS,
+                                         const MachineMemOperand &MMO);
+
     /// printCustom - Implement printing for PseudoSourceValue. This is called
     /// from Value::print or Value's operator<<.
     ///
     virtual void printCustom(raw_ostream &O) const;
 
   public:
-    explicit PseudoSourceValue(enum ValueTy Subclass = PseudoSourceValueVal);
+    /// isFixed - Whether this is a FixedStackPseudoSourceValue.
+    bool isFixed;
+
+    explicit PseudoSourceValue(bool isFixed = false);
+
+    virtual ~PseudoSourceValue();
 
     /// isConstant - Test whether the memory pointed to by this
     /// PseudoSourceValue has a constant value.
@@ -47,14 +58,6 @@ namespace llvm {
     /// PseudoSourceValue can ever alias an LLVM IR Value.
     virtual bool mayAlias(const MachineFrameInfo *) const;
 
-    /// classof - Methods for support type inquiry through isa, cast, and
-    /// dyn_cast:
-    ///
-    static inline bool classof(const Value *V) {
-      return V->getValueID() == PseudoSourceValueVal ||
-             V->getValueID() == FixedStackPseudoSourceValueVal;
-    }
-
     /// A pseudo source value referencing a fixed stack frame entry,
     /// e.g., a spill slot.
     static const PseudoSourceValue *getFixedStack(int FI);
@@ -84,22 +87,22 @@ namespace llvm {
     const int FI;
   public:
     explicit FixedStackPseudoSourceValue(int fi) :
-        PseudoSourceValue(FixedStackPseudoSourceValueVal), FI(fi) {}
+        PseudoSourceValue(true), FI(fi) {}
 
     /// classof - Methods for support type inquiry through isa, cast, and
     /// dyn_cast:
     ///
-    static inline bool classof(const Value *V) {
-      return V->getValueID() == FixedStackPseudoSourceValueVal;
+    static inline bool classof(const PseudoSourceValue *V) {
+      return V->isFixed == true;
     }
 
-    virtual bool isConstant(const MachineFrameInfo *MFI) const;
+    bool isConstant(const MachineFrameInfo *MFI) const override;
 
-    virtual bool isAliased(const MachineFrameInfo *MFI) const;
+    bool isAliased(const MachineFrameInfo *MFI) const override;
 
-    virtual bool mayAlias(const MachineFrameInfo *) const;
+    bool mayAlias(const MachineFrameInfo *) const override;
 
-    virtual void printCustom(raw_ostream &OS) const;
+    void printCustom(raw_ostream &OS) const override;
 
     int getFrameIndex() const { return FI; }
   };
diff --git a/include/llvm/CodeGen/RegAllocPBQP.h b/include/llvm/CodeGen/RegAllocPBQP.h
index 7472e5a62d6b..441b0f084e69 100644
--- a/include/llvm/CodeGen/RegAllocPBQP.h
+++ b/include/llvm/CodeGen/RegAllocPBQP.h
@@ -17,9 +17,9 @@
 #define LLVM_CODEGEN_REGALLOCPBQP_H
 
 #include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
-#include "llvm/CodeGen/PBQP/Graph.h"
-#include "llvm/CodeGen/PBQP/Solution.h"
+#include "llvm/CodeGen/PBQP/RegAllocSolver.h"
 #include <map>
 #include <set>
 
@@ -29,30 +29,30 @@ namespace llvm {
   class MachineBlockFrequencyInfo;
   class MachineFunction;
   class TargetRegisterInfo;
-  template<class T> class OwningPtr;
+
+  typedef PBQP::RegAlloc::Graph PBQPRAGraph;
 
   /// This class wraps up a PBQP instance representing a register allocation
   /// problem, plus the structures necessary to map back from the PBQP solution
   /// to a register allocation solution. (i.e. The PBQP-node <--> vreg map,
   /// and the PBQP option <--> storage location map).
-
   class PBQPRAProblem {
   public:
 
     typedef SmallVector<unsigned, 16> AllowedSet;
 
-    PBQP::Graph& getGraph() { return graph; }
+    PBQPRAGraph& getGraph() { return graph; }
 
-    const PBQP::Graph& getGraph() const { return graph; }
+    const PBQPRAGraph& getGraph() const { return graph; }
 
     /// Record the mapping between the given virtual register and PBQP node,
     /// and the set of allowed pregs for the vreg.
     ///
     /// If you are extending
     /// PBQPBuilder you are unlikely to need this: Nodes and options for all
-    /// vregs will already have been set up for you by the base class. 
+    /// vregs will already have been set up for you by the base class.
     template <typename AllowedRegsItr>
-    void recordVReg(unsigned vreg, PBQP::Graph::NodeId nodeId,
+    void recordVReg(unsigned vreg, PBQPRAGraph::NodeId nodeId,
                     AllowedRegsItr arBegin, AllowedRegsItr arEnd) {
       assert(node2VReg.find(nodeId) == node2VReg.end() && "Re-mapping node.");
       assert(vreg2Node.find(vreg) == vreg2Node.end() && "Re-mapping vreg.");
@@ -64,10 +64,10 @@ namespace llvm {
     }
 
     /// Get the virtual register corresponding to the given PBQP node.
-    unsigned getVRegForNode(PBQP::Graph::NodeId nodeId) const;
+    unsigned getVRegForNode(PBQPRAGraph::NodeId nodeId) const;
 
     /// Get the PBQP node corresponding to the given virtual register.
-    PBQP::Graph::NodeId getNodeForVReg(unsigned vreg) const;
+    PBQPRAGraph::NodeId getNodeForVReg(unsigned vreg) const;
 
     /// Returns true if the given PBQP option represents a physical register,
     /// false otherwise.
@@ -92,16 +92,16 @@ namespace llvm {
 
   private:
 
-    typedef std::map<PBQP::Graph::NodeId, unsigned>  Node2VReg;
-    typedef DenseMap<unsigned, PBQP::Graph::NodeId> VReg2Node;
+    typedef std::map<PBQPRAGraph::NodeId, unsigned>  Node2VReg;
+    typedef DenseMap<unsigned, PBQPRAGraph::NodeId> VReg2Node;
     typedef DenseMap<unsigned, AllowedSet> AllowedSetMap;
 
-    PBQP::Graph graph;
+    PBQPRAGraph graph;
     Node2VReg node2VReg;
     VReg2Node vreg2Node;
 
     AllowedSetMap allowedSets;
-    
+
   };
 
   /// Builds PBQP instances to represent register allocation problems. Includes
@@ -114,7 +114,7 @@ namespace llvm {
   public:
 
     typedef std::set<unsigned> RegSet;
- 
+
     /// Default constructor.
     PBQPBuilder() {}
 
@@ -139,12 +139,12 @@ namespace llvm {
   /// Extended builder which adds coalescing constraints to a problem.
   class PBQPBuilderWithCoalescing : public PBQPBuilder {
   public:
- 
+
     /// Build a PBQP instance to represent the register allocation problem for
     /// the given MachineFunction.
-    virtual PBQPRAProblem *build(MachineFunction *mf, const LiveIntervals *lis,
-                                 const MachineBlockFrequencyInfo *mbfi,
-                                 const RegSet &vregs);   
+    PBQPRAProblem *build(MachineFunction *mf, const LiveIntervals *lis,
+                         const MachineBlockFrequencyInfo *mbfi,
+                         const RegSet &vregs) override;
 
   private:
 
@@ -157,8 +157,9 @@ namespace llvm {
                             PBQP::PBQPNum benefit);
   };
 
-  FunctionPass* createPBQPRegisterAllocator(OwningPtr<PBQPBuilder> &builder,
-                                            char *customPassID=0);
+  FunctionPass *
+  createPBQPRegisterAllocator(std::unique_ptr<PBQPBuilder> builder,
+                              char *customPassID = nullptr);
 }
 
 #endif /* LLVM_CODEGEN_REGALLOCPBQP_H */
diff --git a/include/llvm/CodeGen/RegisterClassInfo.h b/include/llvm/CodeGen/RegisterClassInfo.h
index 9ec12bdea0ad..d784dfbda7ec 100644
--- a/include/llvm/CodeGen/RegisterClassInfo.h
+++ b/include/llvm/CodeGen/RegisterClassInfo.h
@@ -19,7 +19,6 @@
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
-#include "llvm/ADT/OwningPtr.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 
 namespace llvm {
@@ -31,7 +30,7 @@ class RegisterClassInfo {
     bool ProperSubClass;
     uint8_t MinCost;
     uint16_t LastCostChange;
-    OwningArrayPtr<MCPhysReg> Order;
+    std::unique_ptr<MCPhysReg[]> Order;
 
     RCInfo()
       : Tag(0), NumRegs(0), ProperSubClass(false), MinCost(0),
@@ -43,7 +42,7 @@ class RegisterClassInfo {
   };
 
   // Brief cached information for each register class.
-  OwningArrayPtr<RCInfo> RegClass;
+  std::unique_ptr<RCInfo[]> RegClass;
 
   // Tag changes whenever cached information needs to be recomputed. An RCInfo
   // entry is valid when its tag matches.
@@ -54,7 +53,7 @@ class RegisterClassInfo {
 
   // Callee saved registers of last MF. Assumed to be valid until the next
   // runOnFunction() call.
-  const uint16_t *CalleeSaved;
+  const MCPhysReg *CalleeSaved;
 
   // Map register number to CalleeSaved index + 1;
   SmallVector<uint8_t, 4> CSRNum;
@@ -62,7 +61,7 @@ class RegisterClassInfo {
   // Reserved registers in the current MF.
   BitVector Reserved;
 
-  OwningArrayPtr<unsigned> PSetLimits;
+  std::unique_ptr<unsigned[]> PSetLimits;
 
   // Compute all information about RC.
   void compute(const TargetRegisterClass *RC) const;
diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h
index a801d1d1eee9..cc9e00055146 100644
--- a/include/llvm/CodeGen/RegisterPressure.h
+++ b/include/llvm/CodeGen/RegisterPressure.h
@@ -158,7 +158,7 @@ class PressureDiffs {
   unsigned Size;
   unsigned Max;
 public:
-  PressureDiffs(): PDiffArray(0), Size(0), Max(0) {}
+  PressureDiffs(): PDiffArray(nullptr), Size(0), Max(0) {}
   ~PressureDiffs() { free(PDiffArray); }
 
   void clear() { Size = 0; }
@@ -285,12 +285,12 @@ class RegPressureTracker {
 
 public:
   RegPressureTracker(IntervalPressure &rp) :
-    MF(0), TRI(0), RCI(0), LIS(0), MBB(0), P(rp), RequireIntervals(true),
-    TrackUntiedDefs(false) {}
+    MF(nullptr), TRI(nullptr), RCI(nullptr), LIS(nullptr), MBB(nullptr), P(rp),
+    RequireIntervals(true), TrackUntiedDefs(false) {}
 
   RegPressureTracker(RegionPressure &rp) :
-    MF(0), TRI(0), RCI(0), LIS(0), MBB(0), P(rp), RequireIntervals(false),
-    TrackUntiedDefs(false) {}
+    MF(nullptr), TRI(nullptr), RCI(nullptr), LIS(nullptr), MBB(nullptr), P(rp),
+    RequireIntervals(false), TrackUntiedDefs(false) {}
 
   void reset();
 
@@ -318,7 +318,8 @@ public:
   SlotIndex getCurrSlot() const;
 
   /// Recede across the previous instruction.
-  bool recede(SmallVectorImpl<unsigned> *LiveUses = 0, PressureDiff *PDiff = 0);
+  bool recede(SmallVectorImpl<unsigned> *LiveUses = nullptr,
+              PressureDiff *PDiff = nullptr);
 
   /// Advance across the current instruction.
   bool advance();
@@ -393,7 +394,7 @@ public:
                                          MaxPressureLimit);
 
     assert(isBottomClosed() && "Uninitialized pressure tracker");
-    return getMaxUpwardPressureDelta(MI, 0, Delta, CriticalPSets,
+    return getMaxUpwardPressureDelta(MI, nullptr, Delta, CriticalPSets,
                                      MaxPressureLimit);
   }
 
@@ -433,10 +434,8 @@ protected:
   void bumpDownwardPressure(const MachineInstr *MI);
 };
 
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void dumpRegSetPressure(ArrayRef<unsigned> SetPressure,
                         const TargetRegisterInfo *TRI);
-#endif
 } // end namespace llvm
 
 #endif
diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h
index 28ebe5361b4a..335dd7f084c1 100644
--- a/include/llvm/CodeGen/RegisterScavenging.h
+++ b/include/llvm/CodeGen/RegisterScavenging.h
@@ -42,7 +42,7 @@ class RegScavenger {
 
   /// Information on scavenged registers (held in a spill slot).
   struct ScavengedInfo {
-    ScavengedInfo(int FI = -1) : FrameIndex(FI), Reg(0), Restore(NULL) {}
+    ScavengedInfo(int FI = -1) : FrameIndex(FI), Reg(0), Restore(nullptr) {}
 
     /// A spill slot used for scavenging a register post register allocation.
     int FrameIndex;
@@ -73,7 +73,7 @@ class RegScavenger {
 
 public:
   RegScavenger()
-    : MBB(NULL), NumPhysRegs(0), Tracking(false) {}
+    : MBB(nullptr), NumPhysRegs(0), Tracking(false) {}
 
   /// enterBasicBlock - Start tracking liveness from the begin of the specific
   /// basic block.
@@ -104,7 +104,7 @@ public:
 
   /// skipTo - Move the internal MBB iterator but do not update register states.
   void skipTo(MachineBasicBlock::iterator I) {
-    if (I == MachineBasicBlock::iterator(NULL))
+    if (I == MachineBasicBlock::iterator(nullptr))
       Tracking = false;
     MBBI = I;
   }
diff --git a/include/llvm/CodeGen/ResourcePriorityQueue.h b/include/llvm/CodeGen/ResourcePriorityQueue.h
index f20a9fce2ae8..114fe7c35b82 100644
--- a/include/llvm/CodeGen/ResourcePriorityQueue.h
+++ b/include/llvm/CodeGen/ResourcePriorityQueue.h
@@ -81,18 +81,18 @@ namespace llvm {
       delete ResourcesModel;
     }
 
-    bool isBottomUp() const { return false; }
+    bool isBottomUp() const override { return false; }
 
-    void initNodes(std::vector<SUnit> &sunits);
+    void initNodes(std::vector<SUnit> &sunits) override;
 
-    void addNode(const SUnit *SU) {
+    void addNode(const SUnit *SU) override {
       NumNodesSolelyBlocking.resize(SUnits->size(), 0);
     }
 
-    void updateNode(const SUnit *SU) {}
+    void updateNode(const SUnit *SU) override {}
 
-    void releaseState() {
-      SUnits = 0;
+    void releaseState() override {
+      SUnits = nullptr;
     }
 
     unsigned getLatency(unsigned NodeNum) const {
@@ -116,18 +116,18 @@ namespace llvm {
     signed regPressureDelta(SUnit *SU, bool RawPressure = false);
     signed rawRegPressureDelta (SUnit *SU, unsigned RCId);
 
-    bool empty() const { return Queue.empty(); }
+    bool empty() const override { return Queue.empty(); }
 
-    virtual void push(SUnit *U);
+    void push(SUnit *U) override;
 
-    virtual SUnit *pop();
+    SUnit *pop() override;
 
-    virtual void remove(SUnit *SU);
+    void remove(SUnit *SU) override;
 
-    virtual void dump(ScheduleDAG* DAG) const;
+    void dump(ScheduleDAG* DAG) const override;
 
     /// scheduledNode - Main resource tracking point.
-    void scheduledNode(SUnit *Node);
+    void scheduledNode(SUnit *Node) override;
     bool isResourceAvailable(SUnit *SU);
     void reserveResources(SUnit *SU);
 
diff --git a/include/llvm/CodeGen/RuntimeLibcalls.h b/include/llvm/CodeGen/RuntimeLibcalls.h
index 009b8a0f697a..81db8a2f79b5 100644
--- a/include/llvm/CodeGen/RuntimeLibcalls.h
+++ b/include/llvm/CodeGen/RuntimeLibcalls.h
@@ -210,6 +210,10 @@ namespace RTLIB {
     FPEXT_F32_F64,
     FPEXT_F16_F32,
     FPROUND_F32_F16,
+    FPROUND_F64_F16,
+    FPROUND_F80_F16,
+    FPROUND_F128_F16,
+    FPROUND_PPCF128_F16,
     FPROUND_F64_F32,
     FPROUND_F80_F32,
     FPROUND_F128_F32,
diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h
index ccba1b0364e4..5a65d590802a 100644
--- a/include/llvm/CodeGen/ScheduleDAG.h
+++ b/include/llvm/CodeGen/ScheduleDAG.h
@@ -95,7 +95,7 @@ namespace llvm {
     /// SDep - Construct a null SDep. This is only for use by container
     /// classes which require default constructors. SUnits may not
     /// have null SDep edges.
-    SDep() : Dep(0, Data) {}
+    SDep() : Dep(nullptr, Data) {}
 
     /// SDep - Construct an SDep with the specified values.
     SDep(SUnit *S, Kind kind, unsigned Reg)
@@ -184,6 +184,12 @@ namespace llvm {
                                     || Contents.OrdKind == MustAliasMem);
     }
 
+    /// isBarrier - Test if this is an Order dependence that is marked
+    /// as a barrier.
+    bool isBarrier() const {
+      return getKind() == Order && Contents.OrdKind == Barrier;
+    }
+
     /// isMustAlias - Test if this is an Order dependence that is marked
     /// as "must alias", meaning that the SUnits at either end of the edge
     /// have a memory dependence on a known memory location.
@@ -248,7 +254,7 @@ namespace llvm {
   /// SUnit - Scheduling unit. This is a node in the scheduling DAG.
   class SUnit {
   private:
-    enum LLVM_ENUM_INT_TYPE(unsigned) { BoundaryID = ~0u };
+    enum : unsigned { BoundaryID = ~0u };
 
     SDNode *Node;                       // Representative node.
     MachineInstr *Instr;                // Alternatively, a MachineInstr.
@@ -292,6 +298,8 @@ namespace llvm {
     bool isScheduleHigh   : 1;          // True if preferable to schedule high.
     bool isScheduleLow    : 1;          // True if preferable to schedule low.
     bool isCloned         : 1;          // True if this node has been cloned.
+    bool isUnbuffered     : 1;          // Uses an unbuffered resource.
+    bool hasReservedResource : 1;       // Uses a reserved resource.
     Sched::Preference SchedulingPref;   // Scheduling preference.
 
   private:
@@ -309,43 +317,49 @@ namespace llvm {
     /// SUnit - Construct an SUnit for pre-regalloc scheduling to represent
     /// an SDNode and any nodes flagged to it.
     SUnit(SDNode *node, unsigned nodenum)
-      : Node(node), Instr(0), OrigNode(0), SchedClass(0), NodeNum(nodenum),
-        NodeQueueId(0), NumPreds(0), NumSuccs(0), NumPredsLeft(0),
-        NumSuccsLeft(0), WeakPredsLeft(0), WeakSuccsLeft(0), NumRegDefsLeft(0),
-        Latency(0), isVRegCycle(false), isCall(false), isCallOp(false),
-        isTwoAddress(false), isCommutable(false), hasPhysRegUses(false),
-        hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false),
-        isAvailable(false), isScheduled(false), isScheduleHigh(false),
-        isScheduleLow(false), isCloned(false), SchedulingPref(Sched::None),
-        isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0),
-        TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}
+      : Node(node), Instr(nullptr), OrigNode(nullptr), SchedClass(nullptr),
+        NodeNum(nodenum), NodeQueueId(0), NumPreds(0), NumSuccs(0),
+        NumPredsLeft(0), NumSuccsLeft(0), WeakPredsLeft(0), WeakSuccsLeft(0),
+        NumRegDefsLeft(0), Latency(0), isVRegCycle(false), isCall(false),
+        isCallOp(false), isTwoAddress(false), isCommutable(false),
+        hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false),
+        isPending(false), isAvailable(false), isScheduled(false),
+        isScheduleHigh(false), isScheduleLow(false), isCloned(false),
+        isUnbuffered(false), hasReservedResource(false),
+        SchedulingPref(Sched::None), isDepthCurrent(false),
+        isHeightCurrent(false), Depth(0), Height(0), TopReadyCycle(0),
+        BotReadyCycle(0), CopyDstRC(nullptr), CopySrcRC(nullptr) {}
 
     /// SUnit - Construct an SUnit for post-regalloc scheduling to represent
     /// a MachineInstr.
     SUnit(MachineInstr *instr, unsigned nodenum)
-      : Node(0), Instr(instr), OrigNode(0), SchedClass(0), NodeNum(nodenum),
-        NodeQueueId(0), NumPreds(0), NumSuccs(0), NumPredsLeft(0),
-        NumSuccsLeft(0), WeakPredsLeft(0), WeakSuccsLeft(0), NumRegDefsLeft(0),
-        Latency(0), isVRegCycle(false), isCall(false), isCallOp(false),
-        isTwoAddress(false), isCommutable(false), hasPhysRegUses(false),
-        hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false),
-        isAvailable(false), isScheduled(false), isScheduleHigh(false),
-        isScheduleLow(false), isCloned(false), SchedulingPref(Sched::None),
-        isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0),
-        TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}
+      : Node(nullptr), Instr(instr), OrigNode(nullptr), SchedClass(nullptr),
+        NodeNum(nodenum), NodeQueueId(0), NumPreds(0), NumSuccs(0),
+        NumPredsLeft(0), NumSuccsLeft(0), WeakPredsLeft(0), WeakSuccsLeft(0),
+        NumRegDefsLeft(0), Latency(0), isVRegCycle(false), isCall(false),
+        isCallOp(false), isTwoAddress(false), isCommutable(false),
+        hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false),
+        isPending(false), isAvailable(false), isScheduled(false),
+        isScheduleHigh(false), isScheduleLow(false), isCloned(false),
+        isUnbuffered(false), hasReservedResource(false),
+        SchedulingPref(Sched::None), isDepthCurrent(false),
+        isHeightCurrent(false), Depth(0), Height(0), TopReadyCycle(0),
+        BotReadyCycle(0), CopyDstRC(nullptr), CopySrcRC(nullptr) {}
 
     /// SUnit - Construct a placeholder SUnit.
     SUnit()
-      : Node(0), Instr(0), OrigNode(0), SchedClass(0), NodeNum(BoundaryID),
-        NodeQueueId(0), NumPreds(0), NumSuccs(0), NumPredsLeft(0),
-        NumSuccsLeft(0), WeakPredsLeft(0), WeakSuccsLeft(0), NumRegDefsLeft(0),
-        Latency(0), isVRegCycle(false), isCall(false), isCallOp(false),
-        isTwoAddress(false), isCommutable(false), hasPhysRegUses(false),
-        hasPhysRegDefs(false), hasPhysRegClobbers(false), isPending(false),
-        isAvailable(false), isScheduled(false), isScheduleHigh(false),
-        isScheduleLow(false), isCloned(false), SchedulingPref(Sched::None),
-        isDepthCurrent(false), isHeightCurrent(false), Depth(0), Height(0),
-        TopReadyCycle(0), BotReadyCycle(0), CopyDstRC(NULL), CopySrcRC(NULL) {}
+      : Node(nullptr), Instr(nullptr), OrigNode(nullptr), SchedClass(nullptr),
+        NodeNum(BoundaryID), NodeQueueId(0), NumPreds(0), NumSuccs(0),
+        NumPredsLeft(0), NumSuccsLeft(0), WeakPredsLeft(0), WeakSuccsLeft(0),
+        NumRegDefsLeft(0), Latency(0), isVRegCycle(false), isCall(false),
+        isCallOp(false), isTwoAddress(false), isCommutable(false),
+        hasPhysRegUses(false), hasPhysRegDefs(false), hasPhysRegClobbers(false),
+        isPending(false), isAvailable(false), isScheduled(false),
+        isScheduleHigh(false), isScheduleLow(false), isCloned(false),
+        isUnbuffered(false), hasReservedResource(false),
+        SchedulingPref(Sched::None), isDepthCurrent(false),
+        isHeightCurrent(false), Depth(0), Height(0), TopReadyCycle(0),
+        BotReadyCycle(0), CopyDstRC(nullptr), CopySrcRC(nullptr) {}
 
     /// \brief Boundary nodes are placeholders for the boundary of the
     /// scheduling region.
diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h
index fe4f3c2de3b6..e6754a2c0342 100644
--- a/include/llvm/CodeGen/ScheduleDAGInstrs.h
+++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h
@@ -15,8 +15,8 @@
 #ifndef LLVM_CODEGEN_SCHEDULEDAGINSTRS_H
 #define LLVM_CODEGEN_SCHEDULEDAGINSTRS_H
 
-#include "llvm/ADT/SparseSet.h"
 #include "llvm/ADT/SparseMultiSet.h"
+#include "llvm/ADT/SparseSet.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
 #include "llvm/CodeGen/TargetSchedule.h"
 #include "llvm/Support/Compiler.h"
@@ -43,7 +43,7 @@ namespace llvm {
   };
 
   /// Record a physical register access.
-  /// For non data-dependent uses, OpIdx == -1.
+  /// For non-data-dependent uses, OpIdx == -1.
   struct PhysRegSUOper {
     SUnit *SU;
     int OpIdx;
@@ -88,9 +88,13 @@ namespace llvm {
     /// isPostRA flag indicates vregs cannot be present.
     bool IsPostRA;
 
+    /// True if the DAG builder should remove kill flags (in preparation for
+    /// rescheduling).
+    bool RemoveKillFlags;
+
     /// The standard DAG builder does not normally include terminators as DAG
     /// nodes because it does not create the necessary dependencies to prevent
-    /// reordering. A specialized scheduler can overide
+    /// reordering. A specialized scheduler can override
     /// TargetInstrInfo::isSchedulingBoundary then enable this flag to indicate
     /// it has taken responsibility for scheduling the terminator correctly.
     bool CanHandleTerminators;
@@ -145,15 +149,21 @@ namespace llvm {
     DbgValueVector DbgValues;
     MachineInstr *FirstDbgValue;
 
+    /// Set of live physical registers for updating kill flags.
+    BitVector LiveRegs;
+
   public:
     explicit ScheduleDAGInstrs(MachineFunction &mf,
                                const MachineLoopInfo &mli,
                                const MachineDominatorTree &mdt,
                                bool IsPostRAFlag,
-                               LiveIntervals *LIS = 0);
+                               bool RemoveKillFlags = false,
+                               LiveIntervals *LIS = nullptr);
 
     virtual ~ScheduleDAGInstrs() {}
 
+    bool isPostRA() const { return IsPostRA; }
+
     /// \brief Expose LiveIntervals for use in DAG mutators and such.
     LiveIntervals *getLIS() const { return LIS; }
 
@@ -196,8 +206,9 @@ namespace llvm {
 
     /// buildSchedGraph - Build SUnits from the MachineBasicBlock that we are
     /// input.
-    void buildSchedGraph(AliasAnalysis *AA, RegPressureTracker *RPTracker = 0,
-                         PressureDiffs *PDiffs = 0);
+    void buildSchedGraph(AliasAnalysis *AA,
+                         RegPressureTracker *RPTracker = nullptr,
+                         PressureDiffs *PDiffs = nullptr);
 
     /// addSchedBarrierDeps - Add dependencies from instructions in the current
     /// list of instructions being scheduled to scheduling barrier. We want to
@@ -219,29 +230,40 @@ namespace llvm {
     /// the level of the whole MachineFunction. By default does nothing.
     virtual void finalizeSchedule() {}
 
-    virtual void dumpNode(const SUnit *SU) const;
+    void dumpNode(const SUnit *SU) const override;
 
     /// Return a label for a DAG node that points to an instruction.
-    virtual std::string getGraphNodeLabel(const SUnit *SU) const;
+    std::string getGraphNodeLabel(const SUnit *SU) const override;
 
     /// Return a label for the region of code covered by the DAG.
-    virtual std::string getDAGName() const;
+    std::string getDAGName() const override;
 
+    /// \brief Fix register kill flags that scheduling has made invalid.
+    void fixupKills(MachineBasicBlock *MBB);
   protected:
     void initSUnits();
     void addPhysRegDataDeps(SUnit *SU, unsigned OperIdx);
     void addPhysRegDeps(SUnit *SU, unsigned OperIdx);
     void addVRegDefDeps(SUnit *SU, unsigned OperIdx);
     void addVRegUseDeps(SUnit *SU, unsigned OperIdx);
+
+    /// \brief PostRA helper for rewriting kill flags.
+    void startBlockForKills(MachineBasicBlock *BB);
+
+    /// \brief Toggle a register operand kill flag.
+    ///
+    /// Other adjustments may be made to the instruction if necessary. Return
+    /// true if the operand has been deleted, false if not.
+    bool toggleKillFlag(MachineInstr *MI, MachineOperand &MO);
   };
 
   /// newSUnit - Creates a new SUnit and return a ptr to it.
   inline SUnit *ScheduleDAGInstrs::newSUnit(MachineInstr *MI) {
 #ifndef NDEBUG
-    const SUnit *Addr = SUnits.empty() ? 0 : &SUnits[0];
+    const SUnit *Addr = SUnits.empty() ? nullptr : &SUnits[0];
 #endif
     SUnits.push_back(SUnit(MI, (unsigned)SUnits.size()));
-    assert((Addr == 0 || Addr == &SUnits[0]) &&
+    assert((Addr == nullptr || Addr == &SUnits[0]) &&
            "SUnits std::vector reallocated on the fly!");
     SUnits.back().OrigNode = &SUnits.back();
     return &SUnits.back();
@@ -251,7 +273,7 @@ namespace llvm {
   inline SUnit *ScheduleDAGInstrs::getSUnit(MachineInstr *MI) const {
     DenseMap<MachineInstr*, SUnit*>::const_iterator I = MISUnitMap.find(MI);
     if (I == MISUnitMap.end())
-      return 0;
+      return nullptr;
     return I->second;
   }
 } // namespace llvm
diff --git a/include/llvm/CodeGen/ScheduleDFS.h b/include/llvm/CodeGen/ScheduleDFS.h
index 73ce99f4713d..b2108ad3bedb 100644
--- a/include/llvm/CodeGen/ScheduleDFS.h
+++ b/include/llvm/CodeGen/ScheduleDFS.h
@@ -57,11 +57,9 @@ struct ILPValue {
     return RHS <= *this;
   }
 
-#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void print(raw_ostream &OS) const;
 
   void dump() const;
-#endif
 };
 
 /// \brief Compute the values of each DAG node for various metrics during DFS.
diff --git a/include/llvm/CodeGen/ScheduleHazardRecognizer.h b/include/llvm/CodeGen/ScheduleHazardRecognizer.h
index 9dfa3446ef50..8a40e7212ff6 100644
--- a/include/llvm/CodeGen/ScheduleHazardRecognizer.h
+++ b/include/llvm/CodeGen/ScheduleHazardRecognizer.h
@@ -70,6 +70,22 @@ public:
   /// emitted, to advance the hazard state.
   virtual void EmitInstruction(SUnit *) {}
 
+  /// PreEmitNoops - This callback is invoked prior to emitting an instruction.
+  /// It should return the number of noops to emit prior to the provided
+  /// instruction.
+  /// Note: This is only used during PostRA scheduling. EmitNoop is not called
+  /// for these noops.
+  virtual unsigned PreEmitNoops(SUnit *) {
+    return 0;
+  }
+
+  /// ShouldPreferAnother - This callback may be invoked if getHazardType
+  /// returns NoHazard. If, even though there is no hazard, it would be better to
+  /// schedule another available instruction, this callback should return true.
+  virtual bool ShouldPreferAnother(SUnit *) {
+    return false;
+  }
+
   /// AdvanceCycle - This callback is invoked whenever the next top-down
   /// instruction to be scheduled cannot issue in the current cycle, either
   /// because of latency or resource conflicts.  This should increment the
diff --git a/include/llvm/CodeGen/ScoreboardHazardRecognizer.h b/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
index c2103fb233f8..ab14c2de32b0 100644
--- a/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
+++ b/include/llvm/CodeGen/ScoreboardHazardRecognizer.h
@@ -47,7 +47,7 @@ class ScoreboardHazardRecognizer : public ScheduleHazardRecognizer {
     // Indices into the Scoreboard that represent the current cycle.
     size_t Head;
   public:
-    Scoreboard():Data(NULL), Depth(0), Head(0) { }
+    Scoreboard():Data(nullptr), Depth(0), Head(0) { }
     ~Scoreboard() {
       delete[] Data;
     }
@@ -62,7 +62,7 @@ class ScoreboardHazardRecognizer : public ScheduleHazardRecognizer {
     }
 
     void reset(size_t d = 1) {
-      if (Data == NULL) {
+      if (!Data) {
         Depth = d;
         Data = new unsigned[Depth];
       }
@@ -110,15 +110,15 @@ public:
 
   /// atIssueLimit - Return true if no more instructions may be issued in this
   /// cycle.
-  virtual bool atIssueLimit() const;
+  bool atIssueLimit() const override;
 
   // Stalls provides an cycle offset at which SU will be scheduled. It will be
   // negative for bottom-up scheduling.
-  virtual HazardType getHazardType(SUnit *SU, int Stalls);
-  virtual void Reset();
-  virtual void EmitInstruction(SUnit *SU);
-  virtual void AdvanceCycle();
-  virtual void RecedeCycle();
+  HazardType getHazardType(SUnit *SU, int Stalls) override;
+  void Reset() override;
+  void EmitInstruction(SUnit *SU) override;
+  void AdvanceCycle() override;
+  void RecedeCycle() override;
 };
 
 }
diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h
index 82becca315a1..bb87f82d2def 100644
--- a/include/llvm/CodeGen/SelectionDAG.h
+++ b/include/llvm/CodeGen/SelectionDAG.h
@@ -36,7 +36,6 @@ class MDNode;
 class SDDbgValue;
 class TargetLowering;
 class TargetSelectionDAGInfo;
-class TargetTransformInfo;
 
 class SDVTListNode : public FoldingSetNode {
   friend struct FoldingSetTrait<SDVTListNode>;
@@ -152,8 +151,7 @@ public:
 };
 
 class SelectionDAG;
-void checkForCycles(const SDNode *N);
-void checkForCycles(const SelectionDAG *DAG);
+void checkForCycles(const SelectionDAG *DAG, bool force = false);
 
 /// SelectionDAG class - This is used to represent a portion of an LLVM function
 /// in a low-level Data Dependence DAG representation suitable for instruction
@@ -169,7 +167,6 @@ void checkForCycles(const SelectionDAG *DAG);
 class SelectionDAG {
   const TargetMachine &TM;
   const TargetSelectionDAGInfo &TSI;
-  const TargetTransformInfo *TTI;
   const TargetLowering *TLI;
   MachineFunction *MF;
   LLVMContext *Context;
@@ -269,8 +266,7 @@ public:
   /// init - Prepare this SelectionDAG to process code in the given
   /// MachineFunction.
   ///
-  void init(MachineFunction &mf, const TargetTransformInfo *TTI,
-            const TargetLowering *TLI);
+  void init(MachineFunction &mf, const TargetLowering *TLI);
 
   /// clear - Clear state and free memory necessary to make this
   /// SelectionDAG ready to process a new block.
@@ -281,7 +277,6 @@ public:
   const TargetMachine &getTarget() const { return TM; }
   const TargetLowering &getTargetLoweringInfo() const { return *TLI; }
   const TargetSelectionDAGInfo &getSelectionDAGInfo() const { return TSI; }
-  const TargetTransformInfo *getTargetTransformInfo() const { return TTI; }
   LLVMContext *getContext() const {return Context; }
 
   /// viewGraph - Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
@@ -339,7 +334,7 @@ public:
     assert((!N.getNode() || N.getValueType() == MVT::Other) &&
            "DAG root value is not a chain!");
     if (N.getNode())
-      checkForCycles(N.getNode());
+      checkForCycles(N.getNode(), this);
     Root = N;
     if (N.getNode())
       checkForCycles(this);
@@ -396,23 +391,27 @@ public:
   SDVTList getVTList(EVT VT1, EVT VT2);
   SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3);
   SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4);
-  SDVTList getVTList(const EVT *VTs, unsigned NumVTs);
+  SDVTList getVTList(ArrayRef<EVT> VTs);
 
   //===--------------------------------------------------------------------===//
   // Node creation methods.
   //
-  SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false);
-  SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false);
-  SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false);
+  SDValue getConstant(uint64_t Val, EVT VT, bool isTarget = false,
+                      bool isOpaque = false);
+  SDValue getConstant(const APInt &Val, EVT VT, bool isTarget = false,
+                      bool isOpaque = false);
+  SDValue getConstant(const ConstantInt &Val, EVT VT, bool isTarget = false,
+                      bool isOpaque = false);
   SDValue getIntPtrConstant(uint64_t Val, bool isTarget = false);
-  SDValue getTargetConstant(uint64_t Val, EVT VT) {
-    return getConstant(Val, VT, true);
+  SDValue getTargetConstant(uint64_t Val, EVT VT, bool isOpaque = false) {
+    return getConstant(Val, VT, true, isOpaque);
   }
-  SDValue getTargetConstant(const APInt &Val, EVT VT) {
-    return getConstant(Val, VT, true);
+  SDValue getTargetConstant(const APInt &Val, EVT VT, bool isOpaque = false) {
+    return getConstant(Val, VT, true, isOpaque);
   }
-  SDValue getTargetConstant(const ConstantInt &Val, EVT VT) {
-    return getConstant(Val, VT, true);
+  SDValue getTargetConstant(const ConstantInt &Val, EVT VT,
+                            bool isOpaque = false) {
+    return getConstant(Val, VT, true, isOpaque);
   }
   // The forms below that take a double should only be used for simple
   // constants that can be exactly represented in VT.  No checks are made.
@@ -496,7 +495,8 @@ public:
                        SDValue Glue) {
     SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
     SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue };
-    return getNode(ISD::CopyToReg, dl, VTs, Ops, Glue.getNode() ? 4 : 3);
+    return getNode(ISD::CopyToReg, dl, VTs,
+                   ArrayRef<SDValue>(Ops, Glue.getNode() ? 4 : 3));
   }
 
   // Similar to last getCopyToReg() except parameter Reg is a SDValue
@@ -504,13 +504,14 @@ public:
                          SDValue Glue) {
     SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
     SDValue Ops[] = { Chain, Reg, N, Glue };
-    return getNode(ISD::CopyToReg, dl, VTs, Ops, Glue.getNode() ? 4 : 3);
+    return getNode(ISD::CopyToReg, dl, VTs,
+                   ArrayRef<SDValue>(Ops, Glue.getNode() ? 4 : 3));
   }
 
   SDValue getCopyFromReg(SDValue Chain, SDLoc dl, unsigned Reg, EVT VT) {
     SDVTList VTs = getVTList(VT, MVT::Other);
     SDValue Ops[] = { Chain, getRegister(Reg, VT) };
-    return getNode(ISD::CopyFromReg, dl, VTs, Ops, 2);
+    return getNode(ISD::CopyFromReg, dl, VTs, Ops);
   }
 
   // This version of the getCopyFromReg method takes an extra operand, which
@@ -520,7 +521,8 @@ public:
                            SDValue Glue) {
     SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue);
     SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue };
-    return getNode(ISD::CopyFromReg, dl, VTs, Ops, Glue.getNode() ? 3 : 2);
+    return getNode(ISD::CopyFromReg, dl, VTs,
+                   ArrayRef<SDValue>(Ops, Glue.getNode() ? 3 : 2));
   }
 
   SDValue getCondCode(ISD::CondCode Cond);
@@ -537,6 +539,18 @@ public:
   /// undefined.
   SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2,
                            const int *MaskElts);
+  SDValue getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, SDValue N2,
+                           ArrayRef<int> MaskElts) {
+    assert(VT.getVectorNumElements() == MaskElts.size() &&
+           "Must have the same number of vector elements as mask elements!");
+    return getVectorShuffle(VT, dl, N1, N2, MaskElts.data());
+  }
+
+  /// \brief Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to
+  /// the shuffle node in input but with swapped operands.
+  ///
+  /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3>
+  SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV);
 
   /// getAnyExtOrTrunc - Convert Op, which must be of integer type, to the
   /// integer type VT, by either any-extending or truncating it.
@@ -554,16 +568,42 @@ public:
   /// value assuming it was the smaller SrcTy value.
   SDValue getZeroExtendInReg(SDValue Op, SDLoc DL, EVT SrcTy);
 
+  /// getAnyExtendVectorInReg - Return an operation which will any-extend the
+  /// low lanes of the operand into the specified vector type. For example,
+  /// this can convert a v16i8 into a v4i32 by any-extending the low four
+  /// lanes of the operand from i8 to i32.
+  SDValue getAnyExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT);
+
+  /// getSignExtendVectorInReg - Return an operation which will sign extend the
+  /// low lanes of the operand into the specified vector type. For example,
+  /// this can convert a v16i8 into a v4i32 by sign extending the low four
+  /// lanes of the operand from i8 to i32.
+  SDValue getSignExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT);
+
+  /// getZeroExtendVectorInReg - Return an operation which will zero extend the
+  /// low lanes of the operand into the specified vector type. For example,
+  /// this can convert a v16i8 into a v4i32 by zero extending the low four
+  /// lanes of the operand from i8 to i32.
+  SDValue getZeroExtendVectorInReg(SDValue Op, SDLoc DL, EVT VT);
+
+  /// getBoolExtOrTrunc - Convert Op, which must be of integer type, to the
+  /// integer type VT, by using an extension appropriate for the target's
+  /// BooleanContent for type OpVT or truncating it.
+  SDValue getBoolExtOrTrunc(SDValue Op, SDLoc SL, EVT VT, EVT OpVT);
+
   /// getNOT - Create a bitwise NOT operation as (XOR Val, -1).
   SDValue getNOT(SDLoc DL, SDValue Val, EVT VT);
 
+  /// \brief Create a logical NOT operation as (XOR Val, BooleanOne).
+  SDValue getLogicalNOT(SDLoc DL, SDValue Val, EVT VT);
+
   /// getCALLSEQ_START - Return a new CALLSEQ_START node, which always must have
   /// a glue result (to ensure it's not CSE'd).  CALLSEQ_START does not have a
   /// useful SDLoc.
   SDValue getCALLSEQ_START(SDValue Chain, SDValue Op, SDLoc DL) {
     SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
     SDValue Ops[] = { Chain,  Op };
-    return getNode(ISD::CALLSEQ_START, DL, VTs, Ops, 2);
+    return getNode(ISD::CALLSEQ_START, DL, VTs, Ops);
   }
 
   /// getCALLSEQ_END - Return a new CALLSEQ_END node, which always must have a
@@ -576,9 +616,9 @@ public:
     Ops.push_back(Chain);
     Ops.push_back(Op1);
     Ops.push_back(Op2);
-    Ops.push_back(InGlue);
-    return getNode(ISD::CALLSEQ_END, DL, NodeTys, &Ops[0],
-                   (unsigned)Ops.size() - (InGlue.getNode() == 0 ? 1 : 0));
+    if (InGlue.getNode())
+      Ops.push_back(InGlue);
+    return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops);
   }
 
   /// getUNDEF - Return an UNDEF node.  UNDEF does not have a useful SDLoc.
@@ -596,25 +636,22 @@ public:
   ///
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N);
-  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2);
-  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
-                  SDValue N1, SDValue N2, SDValue N3);
+  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
+                  bool nuw = false, bool nsw = false, bool exact = false);
+  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
+                  SDValue N3);
+  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
+                  SDValue N3, SDValue N4);
+  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, SDValue N2,
+                  SDValue N3, SDValue N4, SDValue N5);
+  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT, ArrayRef<SDUse> Ops);
   SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
-                  SDValue N1, SDValue N2, SDValue N3, SDValue N4);
-  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
-                  SDValue N1, SDValue N2, SDValue N3, SDValue N4,
-                  SDValue N5);
-  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
-                  const SDUse *Ops, unsigned NumOps);
-  SDValue getNode(unsigned Opcode, SDLoc DL, EVT VT,
-                  const SDValue *Ops, unsigned NumOps);
+                  ArrayRef<SDValue> Ops);
   SDValue getNode(unsigned Opcode, SDLoc DL,
                   ArrayRef<EVT> ResultTys,
-                  const SDValue *Ops, unsigned NumOps);
-  SDValue getNode(unsigned Opcode, SDLoc DL, const EVT *VTs, unsigned NumVTs,
-                  const SDValue *Ops, unsigned NumOps);
+                  ArrayRef<SDValue> Ops);
   SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
-                  const SDValue *Ops, unsigned NumOps);
+                  ArrayRef<SDValue> Ops);
   SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs);
   SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N);
   SDValue getNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
@@ -687,23 +724,27 @@ public:
   SDValue getVAArg(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr,
                    SDValue SV, unsigned Align);
 
-  /// getAtomic - Gets a node for an atomic op, produces result and chain and
-  /// takes 3 operands
-  SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
-                    SDValue Ptr, SDValue Cmp, SDValue Swp,
-                    MachinePointerInfo PtrInfo, unsigned Alignment,
-                    AtomicOrdering Ordering,
-                    SynchronizationScope SynchScope);
-  SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
-                    SDValue Ptr, SDValue Cmp, SDValue Swp,
-                    MachineMemOperand *MMO,
-                    AtomicOrdering Ordering,
-                    SynchronizationScope SynchScope);
+  /// getAtomicCmpSwap - Gets a node for an atomic cmpxchg op. There are two
+  /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces a the value loaded and a
+  /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded,
+  /// a success flag (initially i1), and a chain.
+  SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs,
+                           SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp,
+                           MachinePointerInfo PtrInfo, unsigned Alignment,
+                           AtomicOrdering SuccessOrdering,
+                           AtomicOrdering FailureOrdering,
+                           SynchronizationScope SynchScope);
+  SDValue getAtomicCmpSwap(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTs,
+                           SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp,
+                           MachineMemOperand *MMO,
+                           AtomicOrdering SuccessOrdering,
+                           AtomicOrdering FailureOrdering,
+                           SynchronizationScope SynchScope);
 
   /// getAtomic - Gets a node for an atomic op, produces result (if relevant)
   /// and chain and takes 2 operands.
   SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
-                    SDValue Ptr, SDValue Val, const Value* PtrVal,
+                    SDValue Ptr, SDValue Val, const Value *PtrVal,
                     unsigned Alignment, AtomicOrdering Ordering,
                     SynchronizationScope SynchScope);
   SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDValue Chain,
@@ -714,11 +755,6 @@ public:
   /// getAtomic - Gets a node for an atomic op, produces result and chain and
   /// takes 1 operand.
   SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, EVT VT,
-                    SDValue Chain, SDValue Ptr, const Value* PtrVal,
-                    unsigned Alignment,
-                    AtomicOrdering Ordering,
-                    SynchronizationScope SynchScope);
-  SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, EVT VT,
                     SDValue Chain, SDValue Ptr, MachineMemOperand *MMO,
                     AtomicOrdering Ordering,
                     SynchronizationScope SynchScope);
@@ -726,33 +762,30 @@ public:
   /// getAtomic - Gets a node for an atomic op, produces result and chain and
   /// takes N operands.
   SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTList,
-                    SDValue* Ops, unsigned NumOps, MachineMemOperand *MMO,
-                    AtomicOrdering Ordering,
+                    ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
+                    AtomicOrdering SuccessOrdering,
+                    AtomicOrdering FailureOrdering,
                     SynchronizationScope SynchScope);
+  SDValue getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, SDVTList VTList,
+                    ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
+                    AtomicOrdering Ordering, SynchronizationScope SynchScope);
 
   /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a
   /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
   /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
   /// less than FIRST_TARGET_MEMORY_OPCODE.
-  SDValue getMemIntrinsicNode(unsigned Opcode, SDLoc dl,
-                              const EVT *VTs, unsigned NumVTs,
-                              const SDValue *Ops, unsigned NumOps,
-                              EVT MemVT, MachinePointerInfo PtrInfo,
-                              unsigned Align = 0, bool Vol = false,
-                              bool ReadMem = true, bool WriteMem = true);
-
   SDValue getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
-                              const SDValue *Ops, unsigned NumOps,
+                              ArrayRef<SDValue> Ops,
                               EVT MemVT, MachinePointerInfo PtrInfo,
                               unsigned Align = 0, bool Vol = false,
                               bool ReadMem = true, bool WriteMem = true);
 
   SDValue getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
-                              const SDValue *Ops, unsigned NumOps,
+                              ArrayRef<SDValue> Ops,
                               EVT MemVT, MachineMemOperand *MMO);
 
   /// getMergeValues - Create a MERGE_VALUES node from the given operands.
-  SDValue getMergeValues(const SDValue *Ops, unsigned NumOps, SDLoc dl);
+  SDValue getMergeValues(ArrayRef<SDValue> Ops, SDLoc dl);
 
   /// getLoad - Loads are not normal binary operators: their result type is not
   /// determined by their operands, and they produce a value AND a token chain.
@@ -760,14 +793,15 @@ public:
   SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr,
                   MachinePointerInfo PtrInfo, bool isVolatile,
                   bool isNonTemporal, bool isInvariant, unsigned Alignment,
-                  const MDNode *TBAAInfo = 0, const MDNode *Ranges = 0);
+                  const MDNode *TBAAInfo = nullptr,
+                  const MDNode *Ranges = nullptr);
   SDValue getLoad(EVT VT, SDLoc dl, SDValue Chain, SDValue Ptr,
                   MachineMemOperand *MMO);
   SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
                      SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo,
                      EVT MemVT, bool isVolatile,
                      bool isNonTemporal, unsigned Alignment,
-                     const MDNode *TBAAInfo = 0);
+                     const MDNode *TBAAInfo = nullptr);
   SDValue getExtLoad(ISD::LoadExtType ExtType, SDLoc dl, EVT VT,
                      SDValue Chain, SDValue Ptr, EVT MemVT,
                      MachineMemOperand *MMO);
@@ -778,8 +812,8 @@ public:
                   SDValue Chain, SDValue Ptr, SDValue Offset,
                   MachinePointerInfo PtrInfo, EVT MemVT,
                   bool isVolatile, bool isNonTemporal, bool isInvariant,
-                  unsigned Alignment, const MDNode *TBAAInfo = 0,
-                  const MDNode *Ranges = 0);
+                  unsigned Alignment, const MDNode *TBAAInfo = nullptr,
+                  const MDNode *Ranges = nullptr);
   SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                   EVT VT, SDLoc dl,
                   SDValue Chain, SDValue Ptr, SDValue Offset,
@@ -790,14 +824,14 @@ public:
   SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr,
                    MachinePointerInfo PtrInfo, bool isVolatile,
                    bool isNonTemporal, unsigned Alignment,
-                   const MDNode *TBAAInfo = 0);
+                   const MDNode *TBAAInfo = nullptr);
   SDValue getStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr,
                    MachineMemOperand *MMO);
   SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr,
                         MachinePointerInfo PtrInfo, EVT TVT,
                         bool isNonTemporal, bool isVolatile,
                         unsigned Alignment,
-                        const MDNode *TBAAInfo = 0);
+                        const MDNode *TBAAInfo = nullptr);
   SDValue getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, SDValue Ptr,
                         EVT TVT, MachineMemOperand *MMO);
   SDValue getIndexedStore(SDValue OrigStoe, SDLoc dl, SDValue Base,
@@ -831,8 +865,7 @@ public:
                                SDValue Op3, SDValue Op4);
   SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
                                SDValue Op3, SDValue Op4, SDValue Op5);
-  SDNode *UpdateNodeOperands(SDNode *N,
-                               const SDValue *Ops, unsigned NumOps);
+  SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops);
 
   /// SelectNodeTo - These are used for target selectors to *mutate* the
   /// specified node to have the specified return type, Target opcode, and
@@ -845,15 +878,14 @@ public:
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT,
                        SDValue Op1, SDValue Op2, SDValue Op3);
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT,
-                       const SDValue *Ops, unsigned NumOps);
+                       ArrayRef<SDValue> Ops);
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1, EVT VT2);
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1,
-                       EVT VT2, const SDValue *Ops, unsigned NumOps);
+                       EVT VT2, ArrayRef<SDValue> Ops);
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1,
-                       EVT VT2, EVT VT3, const SDValue *Ops, unsigned NumOps);
+                       EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
   SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
-                       EVT VT2, EVT VT3, EVT VT4, const SDValue *Ops,
-                       unsigned NumOps);
+                       EVT VT2, EVT VT3, EVT VT4, ArrayRef<SDValue> Ops);
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1,
                        EVT VT2, SDValue Op1);
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1,
@@ -863,12 +895,12 @@ public:
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1,
                        EVT VT2, EVT VT3, SDValue Op1, SDValue Op2, SDValue Op3);
   SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, SDVTList VTs,
-                       const SDValue *Ops, unsigned NumOps);
+                       ArrayRef<SDValue> Ops);
 
   /// MorphNodeTo - This *mutates* the specified node to have the specified
   /// return type, opcode, and operands.
   SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs,
-                      const SDValue *Ops, unsigned NumOps);
+                      ArrayRef<SDValue> Ops);
 
   /// getMachineNode - These are used for target selectors to create a new node
   /// with specified return type(s), MachineInstr opcode, and operands.
@@ -921,17 +953,21 @@ public:
 
   /// getNodeIfExists - Get the specified node if it's already available, or
   /// else return NULL.
-  SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTs,
-                          const SDValue *Ops, unsigned NumOps);
+  SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTs, ArrayRef<SDValue> Ops,
+                          bool nuw = false, bool nsw = false,
+                          bool exact = false);
 
   /// getDbgValue - Creates a SDDbgValue node.
   ///
-  SDDbgValue *getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R, uint64_t Off,
-                          DebugLoc DL, unsigned O);
-  SDDbgValue *getDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off,
-                          DebugLoc DL, unsigned O);
-  SDDbgValue *getDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off,
+  SDDbgValue *getDbgValue(MDNode *MDPtr, SDNode *N, unsigned R,
+			  bool IsIndirect, uint64_t Off,
                           DebugLoc DL, unsigned O);
+  /// Constant.
+  SDDbgValue *getConstantDbgValue(MDNode *MDPtr, const Value *C, uint64_t Off,
+				  DebugLoc DL, unsigned O);
+  /// Frame index.
+  SDDbgValue *getFrameIndexDbgValue(MDNode *MDPtr, unsigned FI, uint64_t Off,
+				    DebugLoc DL, unsigned O);
 
   /// RemoveDeadNode - Remove the specified node from the system. If any of its
   /// operands then becomes dead, remove them as well. Inform UpdateListener
@@ -1076,13 +1112,12 @@ public:
   bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth = 0)
     const;
 
-  /// ComputeMaskedBits - Determine which of the bits specified in Mask are
-  /// known to be either zero or one and return them in the KnownZero/KnownOne
-  /// bitsets.  This code only analyzes bits in Mask, in order to short-circuit
-  /// processing.  Targets can implement the computeMaskedBitsForTargetNode
-  /// method in the TargetLowering class to allow target nodes to be understood.
-  void ComputeMaskedBits(SDValue Op, APInt &KnownZero, APInt &KnownOne,
-                         unsigned Depth = 0) const;
+  /// Determine which bits of Op are known to be either zero or one and return
+  /// them in the KnownZero/KnownOne bitsets.  Targets can implement the
+  /// computeKnownBitsForTargetNode method in the TargetLowering class to allow
+  /// target nodes to be understood.
+  void computeKnownBits(SDValue Op, APInt &KnownZero, APInt &KnownOne,
+                        unsigned Depth = 0) const;
 
   /// ComputeNumSignBits - Return the number of times the sign bit of the
   /// register is replicated into the other bits.  We know that at least 1 bit
@@ -1143,7 +1178,7 @@ public:
   /// low/high part.
   std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
     EVT LoVT, HiVT;
-    llvm::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType());
+    std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType());
     return SplitVector(N, DL, LoVT, HiVT);
   }
 
@@ -1154,23 +1189,33 @@ public:
     return SplitVector(N->getOperand(OpNo), SDLoc(N));
   }
 
+  /// ExtractVectorElements - Append the extracted elements from Start to Count
+  /// out of the vector Op in Args. If Count is 0, all of the elements will be
+  /// extracted.
+  void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
+                             unsigned Start = 0, unsigned Count = 0);
+
+  unsigned getEVTAlignment(EVT MemoryVT) const;
+
 private:
   bool RemoveNodeFromCSEMaps(SDNode *N);
   void AddModifiedNodeToCSEMaps(SDNode *N);
   SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos);
   SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
                                void *&InsertPos);
-  SDNode *FindModifiedNodeSlot(SDNode *N, const SDValue *Ops, unsigned NumOps,
+  SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
                                void *&InsertPos);
   SDNode *UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc loc);
 
   void DeleteNodeNotInCSEMaps(SDNode *N);
   void DeallocateNode(SDNode *N);
 
-  unsigned getEVTAlignment(EVT MemoryVT) const;
-
   void allnodes_clear();
 
+  BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
+                                SDValue N1, SDValue N2, bool nuw, bool nsw,
+                                bool exact);
+
   /// VTList - List of non-single value types.
   FoldingSet<SDVTListNode> VTListMap;
 
diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h
index b5ec8cb075dc..520be402cfc5 100644
--- a/include/llvm/CodeGen/SelectionDAGISel.h
+++ b/include/llvm/CodeGen/SelectionDAGISel.h
@@ -30,7 +30,6 @@ namespace llvm {
   class MachineInstr;
   class TargetLowering;
   class TargetLibraryInfo;
-  class TargetTransformInfo;
   class FunctionLoweringInfo;
   class ScheduleHazardRecognizer;
   class GCFunctionInfo;
@@ -43,7 +42,6 @@ class SelectionDAGISel : public MachineFunctionPass {
 public:
   TargetMachine &TM;
   const TargetLibraryInfo *LibInfo;
-  const TargetTransformInfo *TTI;
   FunctionLoweringInfo *FuncInfo;
   MachineFunction *MF;
   MachineRegisterInfo *RegInfo;
@@ -62,9 +60,9 @@ public:
     return TM.getTargetLowering();
   }
 
-  virtual void getAnalysisUsage(AnalysisUsage &AU) const;
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
 
-  virtual bool runOnMachineFunction(MachineFunction &MF);
+  bool runOnMachineFunction(MachineFunction &MF) override;
 
   virtual void EmitFunctionEntryCode() {}
 
@@ -125,6 +123,8 @@ public:
     OPC_CheckChild3Type, OPC_CheckChild4Type, OPC_CheckChild5Type,
     OPC_CheckChild6Type, OPC_CheckChild7Type,
     OPC_CheckInteger,
+    OPC_CheckChild0Integer, OPC_CheckChild1Integer, OPC_CheckChild2Integer,
+    OPC_CheckChild3Integer, OPC_CheckChild4Integer,
     OPC_CheckCondCode,
     OPC_CheckValueType,
     OPC_CheckComplexPat,
@@ -242,13 +242,15 @@ private:
 
   // Calls to these functions are generated by tblgen.
   SDNode *Select_INLINEASM(SDNode *N);
+  SDNode *Select_READ_REGISTER(SDNode *N);
+  SDNode *Select_WRITE_REGISTER(SDNode *N);
   SDNode *Select_UNDEF(SDNode *N);
   void CannotYetSelect(SDNode *N);
 
 private:
   void DoInstructionSelection();
   SDNode *MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTs,
-                    const SDValue *Ops, unsigned NumOps, unsigned EmitNodeInfo);
+                    ArrayRef<SDValue> Ops, unsigned EmitNodeInfo);
 
   void PrepareEHLandingPad();
 
diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h
index 70c15e6c6e6f..223151105b0d 100644
--- a/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -19,6 +19,8 @@
 #ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
 #define LLVM_CODEGEN_SELECTIONDAGNODES_H
 
+#include "llvm/ADT/iterator_range.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/GraphTraits.h"
 #include "llvm/ADT/STLExtras.h"
@@ -29,9 +31,9 @@
 #include "llvm/CodeGen/MachineMemOperand.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/IR/Constants.h"
+#include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/Instructions.h"
 #include "llvm/Support/DataTypes.h"
-#include "llvm/Support/DebugLoc.h"
 #include "llvm/Support/MathExtras.h"
 #include <cassert>
 
@@ -48,7 +50,26 @@ template <typename T> struct DenseMapInfo;
 template <typename T> struct simplify_type;
 template <typename T> struct ilist_traits;
 
-void checkForCycles(const SDNode *N);
+/// isBinOpWithFlags - Returns true if the opcode is a binary operation
+/// with flags.
+static bool isBinOpWithFlags(unsigned Opcode) {
+  switch (Opcode) {
+  case ISD::SDIV:
+  case ISD::UDIV:
+  case ISD::SRA:
+  case ISD::SRL:
+  case ISD::MUL:
+  case ISD::ADD:
+  case ISD::SUB:
+  case ISD::SHL:
+    return true;
+  default:
+    return false;
+  }
+}
+
+void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
+                    bool force = false);
 
 /// SDVTList - This represents a list of ValueType's that has been intern'd by
 /// a SelectionDAG.  Instances of this simple value class are returned by
@@ -70,6 +91,10 @@ namespace ISD {
   /// BUILD_VECTOR where all of the elements are 0 or undef.
   bool isBuildVectorAllZeros(const SDNode *N);
 
+  /// \brief Return true if the specified node is a BUILD_VECTOR node of
+  /// all ConstantSDNode or undef.
+  bool isBuildVectorOfConstantSDNodes(const SDNode *N);
+
   /// isScalarToVector - Return true if the specified node is a
   /// ISD::SCALAR_TO_VECTOR node or a BUILD_VECTOR node where only the low
   /// element is not an undef.
@@ -95,7 +120,7 @@ class SDValue {
   SDNode *Node;       // The node defining the value we are using.
   unsigned ResNo;     // Which return value of the node we are using.
 public:
-  SDValue() : Node(0), ResNo(0) {}
+  SDValue() : Node(nullptr), ResNo(0) {}
   SDValue(SDNode *node, unsigned resno) : Node(node), ResNo(resno) {}
 
   /// get the index which selects a specific result in the SDNode
@@ -116,7 +141,10 @@ public:
     return !operator==(O);
   }
   bool operator<(const SDValue &O) const {
-    return Node < O.Node || (Node == O.Node && ResNo < O.ResNo);
+    return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
+  }
+  LLVM_EXPLICIT operator bool() const {
+    return Node != nullptr;
   }
 
   SDValue getValue(unsigned R) const {
@@ -141,6 +169,10 @@ public:
     return getValueType().getSizeInBits();
   }
 
+  unsigned getScalarValueSizeInBits() const {
+    return getValueType().getScalarType().getSizeInBits();
+  }
+
   // Forwarding methods - These forward to the corresponding methods in SDNode.
   inline unsigned getOpcode() const;
   inline unsigned getNumOperands() const;
@@ -225,7 +257,7 @@ class SDUse {
   void operator=(const SDUse &U) LLVM_DELETED_FUNCTION;
 
 public:
-  SDUse() : Val(), User(NULL), Prev(NULL), Next(NULL) {}
+  SDUse() : Val(), User(nullptr), Prev(nullptr), Next(nullptr) {}
 
   /// Normally SDUse will just implicitly convert to an SDValue that it holds.
   operator const SDValue&() const { return Val; }
@@ -399,12 +431,12 @@ public:
 
   /// use_empty - Return true if there are no uses of this node.
   ///
-  bool use_empty() const { return UseList == NULL; }
+  bool use_empty() const { return UseList == nullptr; }
 
   /// hasOneUse - Return true if there is exactly one use of this node.
   ///
   bool hasOneUse() const {
-    return !use_empty() && llvm::next(use_begin()) == use_end();
+    return !use_empty() && std::next(use_begin()) == use_end();
   }
 
   /// use_size - Return the number of uses of this node. This method takes
@@ -449,7 +481,7 @@ public:
                           SDUse, ptrdiff_t>::pointer pointer;
 
     use_iterator(const use_iterator &I) : Op(I.Op) {}
-    use_iterator() : Op(0) {}
+    use_iterator() : Op(nullptr) {}
 
     bool operator==(const use_iterator &x) const {
       return Op == x.Op;
@@ -459,7 +491,7 @@ public:
     }
 
     /// atEnd - return true if this iterator is at the end of uses list.
-    bool atEnd() const { return Op == 0; }
+    bool atEnd() const { return Op == nullptr; }
 
     // Iterator traversal: forward iteration only.
     use_iterator &operator++() {          // Preincrement
@@ -497,8 +529,14 @@ public:
     return use_iterator(UseList);
   }
 
-  static use_iterator use_end() { return use_iterator(0); }
+  static use_iterator use_end() { return use_iterator(nullptr); }
 
+  inline iterator_range<use_iterator> uses() {
+    return iterator_range<use_iterator>(use_begin(), use_end());
+  }
+  inline iterator_range<use_iterator> uses() const {
+    return iterator_range<use_iterator>(use_begin(), use_end());
+  }
 
   /// hasNUsesOfValue - Return true if there are exactly NUSES uses of the
   /// indicated value.  This method ignores uses of other values defined by this
@@ -559,6 +597,7 @@ public:
   typedef SDUse* op_iterator;
   op_iterator op_begin() const { return OperandList; }
   op_iterator op_end() const { return OperandList+NumOperands; }
+  ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
 
   SDVTList getVTList() const {
     SDVTList X = { ValueList, NumValues };
@@ -571,7 +610,7 @@ public:
     if (getNumOperands() != 0 &&
       getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
       return getOperand(getNumOperands()-1).getNode();
-    return 0;
+    return nullptr;
   }
 
   // If this is a pseudo op, like copyfromreg, look to see if there is a
@@ -596,7 +635,7 @@ public:
     for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
       if (UI.getUse().get().getValueType() == MVT::Glue)
         return *UI;
-    return 0;
+    return nullptr;
   }
 
   /// getNumValues - Return the number of values defined/returned by this
@@ -629,12 +668,12 @@ public:
 
   /// getOperationName - Return the opcode of this operation for printing.
   ///
-  std::string getOperationName(const SelectionDAG *G = 0) const;
+  std::string getOperationName(const SelectionDAG *G = nullptr) const;
   static const char* getIndexedModeName(ISD::MemIndexedMode AM);
   void print_types(raw_ostream &OS, const SelectionDAG *G) const;
   void print_details(raw_ostream &OS, const SelectionDAG *G) const;
-  void print(raw_ostream &OS, const SelectionDAG *G = 0) const;
-  void printr(raw_ostream &OS, const SelectionDAG *G = 0) const;
+  void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
+  void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
 
   /// printrFull - Print a SelectionDAG node and all children down to
   /// the leaves.  The given SelectionDAG allows target-specific nodes
@@ -642,7 +681,7 @@ public:
   /// print the whole DAG, including children that appear multiple
   /// times.
   ///
-  void printrFull(raw_ostream &O, const SelectionDAG *G = 0) const;
+  void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
 
   /// printrWithDepth - Print a SelectionDAG node and children up to
   /// depth "depth."  The given SelectionDAG allows target-specific
@@ -650,7 +689,7 @@ public:
   /// will print children that appear multiple times wherever they are
   /// used.
   ///
-  void printrWithDepth(raw_ostream &O, const SelectionDAG *G = 0,
+  void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
                        unsigned depth = 100) const;
 
 
@@ -675,14 +714,15 @@ public:
   /// Unlike dumpr, this will print the whole DAG, including children
   /// that appear multiple times.
   ///
-  void dumprFull(const SelectionDAG *G = 0) const;
+  void dumprFull(const SelectionDAG *G = nullptr) const;
 
   /// dumprWithDepth - printrWithDepth to dbgs().  The given
   /// SelectionDAG allows target-specific nodes to be printed in
   /// human-readable form.  Unlike dumpr, this will print children
   /// that appear multiple times wherever they are used.
   ///
-  void dumprWithDepth(const SelectionDAG *G = 0, unsigned depth = 100) const;
+  void dumprWithDepth(const SelectionDAG *G = nullptr,
+                      unsigned depth = 100) const;
 
   /// Profile - Gather unique data for the node.
   ///
@@ -699,14 +739,14 @@ protected:
   }
 
   SDNode(unsigned Opc, unsigned Order, const DebugLoc dl, SDVTList VTs,
-         const SDValue *Ops, unsigned NumOps)
+         ArrayRef<SDValue> Ops)
     : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false),
       SubclassData(0), NodeId(-1),
-      OperandList(NumOps ? new SDUse[NumOps] : 0),
-      ValueList(VTs.VTs), UseList(NULL),
-      NumOperands(NumOps), NumValues(VTs.NumVTs),
+      OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr),
+      ValueList(VTs.VTs), UseList(nullptr),
+      NumOperands(Ops.size()), NumValues(VTs.NumVTs),
       debugLoc(dl), IROrder(Order) {
-    for (unsigned i = 0; i != NumOps; ++i) {
+    for (unsigned i = 0; i != Ops.size(); ++i) {
       OperandList[i].setUser(this);
       OperandList[i].setInitial(Ops[i]);
     }
@@ -717,9 +757,9 @@ protected:
   /// set later with InitOperands.
   SDNode(unsigned Opc, unsigned Order, const DebugLoc dl, SDVTList VTs)
     : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false),
-      SubclassData(0), NodeId(-1), OperandList(0),
-      ValueList(VTs.VTs), UseList(NULL), NumOperands(0), NumValues(VTs.NumVTs),
-      debugLoc(dl), IROrder(Order) {}
+      SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs),
+      UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), debugLoc(dl),
+      IROrder(Order) {}
 
   /// InitOperands - Initialize the operands list of this with 1 operand.
   void InitOperands(SDUse *Ops, const SDValue &Op0) {
@@ -804,7 +844,7 @@ private:
   int IROrder;
 
 public:
-  SDLoc() : Ptr(NULL), IROrder(0) {}
+  SDLoc() : Ptr(nullptr), IROrder(0) {}
   SDLoc(const SDNode *N) : Ptr(N), IROrder(-1) {
     assert(N && "null SDNode");
   }
@@ -815,14 +855,14 @@ public:
     assert(Order >= 0 && "bad IROrder");
   }
   unsigned getIROrder() {
-    if (IROrder >= 0 || Ptr == NULL) {
+    if (IROrder >= 0 || Ptr == nullptr) {
       return (unsigned)IROrder;
     }
     const SDNode *N = (const SDNode*)(Ptr);
     return N->getIROrder();
   }
   DebugLoc getDebugLoc() {
-    if (Ptr == NULL) {
+    if (!Ptr) {
       return DebugLoc();
     }
     if (IROrder >= 0) {
@@ -922,6 +962,36 @@ public:
   }
 };
 
+/// BinaryWithFlagsSDNode - This class is an extension of BinarySDNode
+/// used from those opcodes that have associated extra flags.
+class BinaryWithFlagsSDNode : public BinarySDNode {
+  enum { NUW = (1 << 0), NSW = (1 << 1), EXACT = (1 << 2) };
+
+public:
+  BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
+                        SDValue X, SDValue Y)
+      : BinarySDNode(Opc, Order, dl, VTs, X, Y) {}
+  /// getRawSubclassData - Return the SubclassData value, which contains an
+  /// encoding of the flags.
+  /// This function should be used to add subclass data to the NodeID value.
+  unsigned getRawSubclassData() const { return SubclassData; }
+  void setHasNoUnsignedWrap(bool b) {
+    SubclassData = (SubclassData & ~NUW) | (b ? NUW : 0);
+  }
+  void setHasNoSignedWrap(bool b) {
+    SubclassData = (SubclassData & ~NSW) | (b ? NSW : 0);
+  }
+  void setIsExact(bool b) {
+    SubclassData = (SubclassData & ~EXACT) | (b ? EXACT : 0);
+  }
+  bool hasNoUnsignedWrap() const { return SubclassData & NUW; }
+  bool hasNoSignedWrap() const { return SubclassData & NSW; }
+  bool isExact() const { return SubclassData & EXACT; }
+  static bool classof(const SDNode *N) {
+    return isBinOpWithFlags(N->getOpcode());
+  }
+};
+
 /// TernarySDNode - This class is used for three-operand SDNodes. This is solely
 /// to allow co-allocation of node operands with the node itself.
 class TernarySDNode : public SDNode {
@@ -982,8 +1052,7 @@ public:
             EVT MemoryVT, MachineMemOperand *MMO);
 
   MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
-            const SDValue *Ops,
-            unsigned NumOps, EVT MemoryVT, MachineMemOperand *MMO);
+            ArrayRef<SDValue> Ops, EVT MemoryVT, MachineMemOperand *MMO);
 
   bool readMem() const { return MMO->isLoad(); }
   bool writeMem() const { return MMO->isStore(); }
@@ -1016,8 +1085,7 @@ public:
     return SynchronizationScope((SubclassData >> 12) & 1);
   }
 
-  /// Returns the SrcValue and offset that describes the location of the access
-  const Value *getSrcValue() const { return MMO->getValue(); }
+  // Returns the offset from the location of the access.
   int64_t getSrcValueOffset() const { return MMO->getOffset(); }
 
   /// Returns the TBAAInfo that describes the dereference.
@@ -1063,6 +1131,7 @@ public:
            N->getOpcode() == ISD::STORE               ||
            N->getOpcode() == ISD::PREFETCH            ||
            N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
+           N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
            N->getOpcode() == ISD::ATOMIC_SWAP         ||
            N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
            N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
@@ -1085,15 +1154,27 @@ public:
 class AtomicSDNode : public MemSDNode {
   SDUse Ops[4];
 
-  void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) {
+  /// For cmpxchg instructions, the ordering requirements when a store does not
+  /// occur.
+  AtomicOrdering FailureOrdering;
+
+  void InitAtomic(AtomicOrdering SuccessOrdering,
+                  AtomicOrdering FailureOrdering,
+                  SynchronizationScope SynchScope) {
     // This must match encodeMemSDNodeFlags() in SelectionDAG.cpp.
-    assert((Ordering & 15) == Ordering &&
+    assert((SuccessOrdering & 15) == SuccessOrdering &&
+           "Ordering may not require more than 4 bits!");
+    assert((FailureOrdering & 15) == FailureOrdering &&
            "Ordering may not require more than 4 bits!");
     assert((SynchScope & 1) == SynchScope &&
            "SynchScope may not require more than 1 bit!");
-    SubclassData |= Ordering << 8;
+    SubclassData |= SuccessOrdering << 8;
     SubclassData |= SynchScope << 12;
-    assert(getOrdering() == Ordering && "Ordering encoding error!");
+    this->FailureOrdering = FailureOrdering;
+    assert(getSuccessOrdering() == SuccessOrdering &&
+           "Ordering encoding error!");
+    assert(getFailureOrdering() == FailureOrdering &&
+           "Ordering encoding error!");
     assert(getSynchScope() == SynchScope && "Synch-scope encoding error!");
   }
 
@@ -1107,12 +1188,11 @@ public:
   // SrcVal: address to update as a Value (used for MemOperand)
   // Align:  alignment of memory
   AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL,
-               EVT MemVT,
-               SDValue Chain, SDValue Ptr,
-               SDValue Cmp, SDValue Swp, MachineMemOperand *MMO,
-               AtomicOrdering Ordering, SynchronizationScope SynchScope)
-    : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
-    InitAtomic(Ordering, SynchScope);
+               EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp,
+               MachineMemOperand *MMO, AtomicOrdering Ordering,
+               SynchronizationScope SynchScope)
+      : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
+    InitAtomic(Ordering, Ordering, SynchScope);
     InitOperands(Ops, Chain, Ptr, Cmp, Swp);
   }
   AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL,
@@ -1121,7 +1201,7 @@ public:
                SDValue Val, MachineMemOperand *MMO,
                AtomicOrdering Ordering, SynchronizationScope SynchScope)
     : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
-    InitAtomic(Ordering, SynchScope);
+    InitAtomic(Ordering, Ordering, SynchScope);
     InitOperands(Ops, Chain, Ptr, Val);
   }
   AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL,
@@ -1130,15 +1210,16 @@ public:
                MachineMemOperand *MMO,
                AtomicOrdering Ordering, SynchronizationScope SynchScope)
     : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
-    InitAtomic(Ordering, SynchScope);
+    InitAtomic(Ordering, Ordering, SynchScope);
     InitOperands(Ops, Chain, Ptr);
   }
   AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, EVT MemVT,
-               SDValue* AllOps, SDUse *DynOps, unsigned NumOps,
+               const SDValue* AllOps, SDUse *DynOps, unsigned NumOps,
                MachineMemOperand *MMO,
-               AtomicOrdering Ordering, SynchronizationScope SynchScope)
+               AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
+               SynchronizationScope SynchScope)
     : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
-    InitAtomic(Ordering, SynchScope);
+    InitAtomic(SuccessOrdering, FailureOrdering, SynchScope);
     assert((DynOps || NumOps <= array_lengthof(Ops)) &&
            "Too many ops for internal storage!");
     InitOperands(DynOps ? DynOps : Ops, AllOps, NumOps);
@@ -1147,14 +1228,25 @@ public:
   const SDValue &getBasePtr() const { return getOperand(1); }
   const SDValue &getVal() const { return getOperand(2); }
 
+  AtomicOrdering getSuccessOrdering() const {
+    return getOrdering();
+  }
+
+  // Not quite enough room in SubclassData for everything, so failure gets its
+  // own field.
+  AtomicOrdering getFailureOrdering() const {
+    return FailureOrdering;
+  }
+
   bool isCompareAndSwap() const {
     unsigned Op = getOpcode();
-    return Op == ISD::ATOMIC_CMP_SWAP;
+    return Op == ISD::ATOMIC_CMP_SWAP || Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
   }
 
   // Methods to support isa and dyn_cast
   static bool classof(const SDNode *N) {
     return N->getOpcode() == ISD::ATOMIC_CMP_SWAP     ||
+           N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
            N->getOpcode() == ISD::ATOMIC_SWAP         ||
            N->getOpcode() == ISD::ATOMIC_LOAD_ADD     ||
            N->getOpcode() == ISD::ATOMIC_LOAD_SUB     ||
@@ -1178,9 +1270,9 @@ public:
 class MemIntrinsicSDNode : public MemSDNode {
 public:
   MemIntrinsicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
-                     const SDValue *Ops, unsigned NumOps,
-                     EVT MemoryVT, MachineMemOperand *MMO)
-    : MemSDNode(Opc, Order, dl, VTs, Ops, NumOps, MemoryVT, MMO) {
+                     ArrayRef<SDValue> Ops, EVT MemoryVT,
+                     MachineMemOperand *MMO)
+    : MemSDNode(Opc, Order, dl, VTs, Ops, MemoryVT, MMO) {
   }
 
   // Methods to support isa and dyn_cast
@@ -1246,9 +1338,10 @@ public:
 class ConstantSDNode : public SDNode {
   const ConstantInt *Value;
   friend class SelectionDAG;
-  ConstantSDNode(bool isTarget, const ConstantInt *val, EVT VT)
+  ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
     : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant,
              0, DebugLoc(), getSDVTList(VT)), Value(val) {
+    SubclassData |= (uint16_t)isOpaque;
   }
 public:
 
@@ -1261,6 +1354,8 @@ public:
   bool isNullValue() const { return Value->isNullValue(); }
   bool isAllOnesValue() const { return Value->isAllOnesValue(); }
 
+  bool isOpaque() const { return SubclassData & 1; }
+
   static bool classof(const SDNode *N) {
     return N->getOpcode() == ISD::Constant ||
            N->getOpcode() == ISD::TargetConstant;
@@ -1486,7 +1581,32 @@ public:
   /// undefined.  isBigEndian describes the endianness of the target.
   bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
                        unsigned &SplatBitSize, bool &HasAnyUndefs,
-                       unsigned MinSplatBits = 0, bool isBigEndian = false);
+                       unsigned MinSplatBits = 0,
+                       bool isBigEndian = false) const;
+
+  /// \brief Returns the splatted value or a null value if this is not a splat.
+  ///
+  /// If passed a non-null UndefElements bitvector, it will resize it to match
+  /// the vector width and set the bits where elements are undef.
+  SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
+
+  /// \brief Returns the splatted constant or null if this is not a constant
+  /// splat.
+  ///
+  /// If passed a non-null UndefElements bitvector, it will resize it to match
+  /// the vector width and set the bits where elements are undef.
+  ConstantSDNode *
+  getConstantSplatNode(BitVector *UndefElements = nullptr) const;
+
+  /// \brief Returns the splatted constant FP or null if this is not a constant
+  /// FP splat.
+  ///
+  /// If passed a non-null UndefElements bitvector, it will resize it to match
+  /// the vector width and set the bits where elements are undef.
+  ConstantFPSDNode *
+  getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
+
+  bool isConstant() const;
 
   static inline bool classof(const SDNode *N) {
     return N->getOpcode() == ISD::BUILD_VECTOR;
@@ -1638,11 +1758,10 @@ class CvtRndSatSDNode : public SDNode {
   ISD::CvtCode CvtCode;
   friend class SelectionDAG;
   explicit CvtRndSatSDNode(EVT VT, unsigned Order, DebugLoc dl,
-                           const SDValue *Ops, unsigned NumOps,
-                           ISD::CvtCode Code)
-    : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT), Ops, NumOps),
+                           ArrayRef<SDValue> Ops, ISD::CvtCode Code)
+    : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT), Ops),
       CvtCode(Code) {
-    assert(NumOps == 5 && "wrong number of operations");
+    assert(Ops.size() == 5 && "wrong number of operations");
   }
 public:
   ISD::CvtCode getCvtCode() const { return CvtCode; }
@@ -1785,7 +1904,7 @@ public:
 private:
   friend class SelectionDAG;
   MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs)
-    : SDNode(Opc, Order, DL, VTs), MemRefs(0), MemRefsEnd(0) {}
+    : SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {}
 
   /// LocalOperands - Operands for this instruction, if they fit here. If
   /// they don't, this field is unused.
diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h
index 984796af8644..00bb22b91d32 100644
--- a/include/llvm/CodeGen/SlotIndexes.h
+++ b/include/llvm/CodeGen/SlotIndexes.h
@@ -147,11 +147,11 @@ namespace llvm {
     };
 
     /// Construct an invalid index.
-    SlotIndex() : lie(0, 0) {}
+    SlotIndex() : lie(nullptr, 0) {}
 
     // Construct a new slot index from the given one, and set the slot.
     SlotIndex(const SlotIndex &li, Slot s) : lie(li.listEntry(), unsigned(s)) {
-      assert(lie.getPointer() != 0 &&
+      assert(lie.getPointer() != nullptr &&
              "Attempt to construct index with 0 pointer.");
     }
 
@@ -377,10 +377,10 @@ namespace llvm {
       initializeSlotIndexesPass(*PassRegistry::getPassRegistry());
     }
 
-    virtual void getAnalysisUsage(AnalysisUsage &au) const;
-    virtual void releaseMemory();
+    void getAnalysisUsage(AnalysisUsage &au) const override;
+    void releaseMemory() override;
 
-    virtual bool runOnMachineFunction(MachineFunction &fn);
+    bool runOnMachineFunction(MachineFunction &fn) override;
 
     /// Dump the indexes.
     void dump() const;
@@ -421,7 +421,7 @@ namespace llvm {
     /// Returns the instruction for the given index, or null if the given
     /// index has no instruction associated with it.
     MachineInstr* getInstructionFromIndex(SlotIndex index) const {
-      return index.isValid() ? index.listEntry()->getInstr() : 0;
+      return index.isValid() ? index.listEntry()->getInstr() : nullptr;
     }
 
     /// Returns the next non-null index, if one exists.
@@ -545,20 +545,20 @@ namespace llvm {
         std::lower_bound(idx2MBBMap.begin(), idx2MBBMap.end(), start);
 
       if (itr == idx2MBBMap.end()) {
-        itr = prior(itr);
+        itr = std::prev(itr);
         return itr->second;
       }
 
       // Check that we don't cross the boundary into this block.
       if (itr->first < end)
-        return 0;
+        return nullptr;
 
-      itr = prior(itr);
+      itr = std::prev(itr);
 
       if (itr->first <= start)
         return itr->second;
 
-      return 0;
+      return nullptr;
     }
 
     /// Insert the given machine instruction into the mapping. Returns the
@@ -574,18 +574,18 @@ namespace llvm {
       // affected by debug information.
       assert(!mi->isDebugValue() && "Cannot number DBG_VALUE instructions.");
 
-      assert(mi->getParent() != 0 && "Instr must be added to function.");
+      assert(mi->getParent() != nullptr && "Instr must be added to function.");
 
       // Get the entries where mi should be inserted.
       IndexList::iterator prevItr, nextItr;
       if (Late) {
         // Insert mi's index immediately before the following instruction.
         nextItr = getIndexAfter(mi).listEntry();
-        prevItr = prior(nextItr);
+        prevItr = std::prev(nextItr);
       } else {
         // Insert mi's index immediately after the preceding instruction.
         prevItr = getIndexBefore(mi).listEntry();
-        nextItr = llvm::next(prevItr);
+        nextItr = std::next(prevItr);
       }
 
       // Get a number for the new instr, or 0 if there's no room currently.
@@ -615,7 +615,7 @@ namespace llvm {
         IndexListEntry *miEntry(mi2iItr->second.listEntry());
         assert(miEntry->getInstr() == mi && "Instruction indexes broken.");
         // FIXME: Eventually we want to actually delete these indexes.
-        miEntry->setInstr(0);
+        miEntry->setInstr(nullptr);
         mi2iMap.erase(mi2iItr);
       }
     }
@@ -638,17 +638,17 @@ namespace llvm {
     /// Add the given MachineBasicBlock into the maps.
     void insertMBBInMaps(MachineBasicBlock *mbb) {
       MachineFunction::iterator nextMBB =
-        llvm::next(MachineFunction::iterator(mbb));
+        std::next(MachineFunction::iterator(mbb));
 
-      IndexListEntry *startEntry = 0;
-      IndexListEntry *endEntry = 0;
+      IndexListEntry *startEntry = nullptr;
+      IndexListEntry *endEntry = nullptr;
       IndexList::iterator newItr;
       if (nextMBB == mbb->getParent()->end()) {
         startEntry = &indexList.back();
-        endEntry = createEntry(0, 0);
+        endEntry = createEntry(nullptr, 0);
         newItr = indexList.insertAfter(startEntry, endEntry);
       } else {
-        startEntry = createEntry(0, 0);
+        startEntry = createEntry(nullptr, 0);
         endEntry = getMBBStartIdx(nextMBB).listEntry();
         newItr = indexList.insert(endEntry, startEntry);
       }
diff --git a/include/llvm/CodeGen/StackMapLivenessAnalysis.h b/include/llvm/CodeGen/StackMapLivenessAnalysis.h
new file mode 100644
index 000000000000..6f0754616206
--- /dev/null
+++ b/include/llvm/CodeGen/StackMapLivenessAnalysis.h
@@ -0,0 +1,64 @@
+//===--- StackMapLivenessAnalysis - StackMap Liveness Analysis --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass calculates the liveness for each basic block in a function and
+// attaches the register live-out information to a patchpoint intrinsic (if
+// present).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H
+#define LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H
+
+#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+
+
+namespace llvm {
+
+/// \brief This pass calculates the liveness information for each basic block in
+/// a function and attaches the register live-out information to a patchpoint
+/// intrinsic if present.
+///
+/// This pass can be disabled via the -enable-patchpoint-liveness=false flag.
+/// The pass skips functions that don't have any patchpoint intrinsics. The
+/// information provided by this pass is optional and not required by the
+/// aformentioned intrinsic to function.
+class StackMapLiveness : public MachineFunctionPass {
+  MachineFunction *MF;
+  const TargetRegisterInfo *TRI;
+  LivePhysRegs LiveRegs;
+public:
+  static char ID;
+
+  /// \brief Default construct and initialize the pass.
+  StackMapLiveness();
+
+  /// \brief Tell the pass manager which passes we depend on and what
+  /// information we preserve.
+  void getAnalysisUsage(AnalysisUsage &AU) const override;
+
+  /// \brief Calculate the liveness information for the given machine function.
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+private:
+  /// \brief Performs the actual liveness calculation for the function.
+  bool calculateLiveness();
+
+  /// \brief Add the current register live set to the instruction.
+  void addLiveOutSetToMI(MachineInstr &MI);
+
+  /// \brief Create a register mask and initialize it with the registers from
+  /// the register live set.
+  uint32_t *createRegisterMask() const;
+};
+
+} // llvm namespace
+
+#endif // LLVM_CODEGEN_STACKMAP_LIVENESS_ANALYSIS_H
diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h
index e90f22e5b69a..5eddbb65259e 100644
--- a/include/llvm/CodeGen/StackMaps.h
+++ b/include/llvm/CodeGen/StackMaps.h
@@ -1,4 +1,5 @@
 //===------------------- StackMaps.h - StackMaps ----------------*- C++ -*-===//
+
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -10,6 +11,7 @@
 #ifndef LLVM_STACKMAPS
 #define LLVM_STACKMAPS
 
+#include "llvm/ADT/MapVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include <map>
@@ -19,6 +21,7 @@ namespace llvm {
 
 class AsmPrinter;
 class MCExpr;
+class MCStreamer;
 
 /// \brief MI-level patchpoint operands.
 ///
@@ -92,19 +95,28 @@ public:
       : LocType(LocType), Size(Size), Reg(Reg), Offset(Offset) {}
   };
 
-  // Typedef a function pointer for functions that parse sequences of operands
-  // and return a Location, plus a new "next" operand iterator.
-  typedef std::pair<Location, MachineInstr::const_mop_iterator>
-    (*OperandParser)(MachineInstr::const_mop_iterator,
-                     MachineInstr::const_mop_iterator, const TargetMachine&);
+  struct LiveOutReg {
+    unsigned short Reg;
+    unsigned short RegNo;
+    unsigned short Size;
+
+    LiveOutReg() : Reg(0), RegNo(0), Size(0) {}
+    LiveOutReg(unsigned short Reg, unsigned short RegNo, unsigned short Size)
+      : Reg(Reg), RegNo(RegNo), Size(Size) {}
+
+    void MarkInvalid() { Reg = 0; }
+
+    // Only sort by the dwarf register number.
+    bool operator< (const LiveOutReg &LO) const { return RegNo < LO.RegNo; }
+    static bool IsInvalid(const LiveOutReg &LO) { return LO.Reg == 0; }
+  };
 
   // OpTypes are used to encode information about the following logical
   // operand (which may consist of several MachineOperands) for the
   // OpParser.
   typedef enum { DirectMemRefOp, IndirectMemRefOp, ConstantOp } OpType;
 
-  StackMaps(AsmPrinter &AP, OperandParser OpParser)
-    : AP(AP), OpParser(OpParser) {}
+  StackMaps(AsmPrinter &AP);
 
   /// \brief Generate a stackmap record for a stackmap instruction.
   ///
@@ -120,54 +132,66 @@ public:
   void serializeToStackMapSection();
 
 private:
+  static const char *WSMP;
+
   typedef SmallVector<Location, 8> LocationVec;
+  typedef SmallVector<LiveOutReg, 8> LiveOutVec;
+  typedef MapVector<int64_t, int64_t> ConstantPool;
+  typedef MapVector<const MCSymbol *, uint64_t> FnStackSizeMap;
 
   struct CallsiteInfo {
     const MCExpr *CSOffsetExpr;
-    unsigned ID;
+    uint64_t ID;
     LocationVec Locations;
-    CallsiteInfo() : CSOffsetExpr(0), ID(0) {}
-    CallsiteInfo(const MCExpr *CSOffsetExpr, unsigned ID,
-                 LocationVec Locations)
-      : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(Locations) {}
+    LiveOutVec LiveOuts;
+    CallsiteInfo() : CSOffsetExpr(nullptr), ID(0) {}
+    CallsiteInfo(const MCExpr *CSOffsetExpr, uint64_t ID,
+                 LocationVec &Locations, LiveOutVec &LiveOuts)
+      : CSOffsetExpr(CSOffsetExpr), ID(ID), Locations(Locations),
+        LiveOuts(LiveOuts) {}
   };
 
   typedef std::vector<CallsiteInfo> CallsiteInfoList;
 
-  struct ConstantPool {
-  private:
-    typedef std::map<int64_t, size_t> ConstantsMap;
-    std::vector<int64_t> ConstantsList;
-    ConstantsMap ConstantIndexes;
-
-  public:
-    size_t getNumConstants() const { return ConstantsList.size(); }
-    int64_t getConstant(size_t Idx) const { return ConstantsList[Idx]; }
-    size_t getConstantIndex(int64_t ConstVal) {
-      size_t NextIdx = ConstantsList.size();
-      ConstantsMap::const_iterator I =
-        ConstantIndexes.insert(ConstantIndexes.end(),
-                               std::make_pair(ConstVal, NextIdx));
-      if (I->second == NextIdx)
-        ConstantsList.push_back(ConstVal);
-      return I->second;
-    }
-  };
-
   AsmPrinter &AP;
-  OperandParser OpParser;
   CallsiteInfoList CSInfos;
   ConstantPool ConstPool;
+  FnStackSizeMap FnStackSize;
+
+  MachineInstr::const_mop_iterator
+  parseOperand(MachineInstr::const_mop_iterator MOI,
+               MachineInstr::const_mop_iterator MOE,
+               LocationVec &Locs, LiveOutVec &LiveOuts) const;
+
+  /// \brief Create a live-out register record for the given register @p Reg.
+  LiveOutReg createLiveOutReg(unsigned Reg,
+                              const TargetRegisterInfo *TRI) const;
+
+  /// \brief Parse the register live-out mask and return a vector of live-out
+  /// registers that need to be recorded in the stackmap.
+  LiveOutVec parseRegisterLiveOutMask(const uint32_t *Mask) const;
 
   /// This should be called by the MC lowering code _immediately_ before
   /// lowering the MI to an MCInst. It records where the operands for the
   /// instruction are stored, and outputs a label to record the offset of
   /// the call from the start of the text section. In special cases (e.g. AnyReg
   /// calling convention) the return register is also recorded if requested.
-  void recordStackMapOpers(const MachineInstr &MI, uint32_t ID,
+  void recordStackMapOpers(const MachineInstr &MI, uint64_t ID,
                            MachineInstr::const_mop_iterator MOI,
                            MachineInstr::const_mop_iterator MOE,
                            bool recordResult = false);
+
+  /// \brief Emit the stackmap header.
+  void emitStackmapHeader(MCStreamer &OS);
+
+  /// \brief Emit the function frame record for each function.
+  void emitFunctionFrameRecords(MCStreamer &OS);
+
+  /// \brief Emit the constant pool.
+  void emitConstantPoolEntries(MCStreamer &OS);
+
+  /// \brief Emit the callsite info for each stackmap/patchpoint intrinsic call.
+  void emitCallsiteEntries(MCStreamer &OS, const TargetRegisterInfo *TRI);
 };
 
 }
diff --git a/include/llvm/CodeGen/StackProtector.h b/include/llvm/CodeGen/StackProtector.h
index d09a933a663b..8cef85cb4485 100644
--- a/include/llvm/CodeGen/StackProtector.h
+++ b/include/llvm/CodeGen/StackProtector.h
@@ -19,12 +19,12 @@
 
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Triple.h"
-#include "llvm/ADT/ValueMap.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/ValueMap.h"
 #include "llvm/Pass.h"
 #include "llvm/Target/TargetLowering.h"
 
 namespace llvm {
-class DominatorTree;
 class Function;
 class Module;
 class PHINode;
@@ -105,22 +105,24 @@ private:
 
 public:
   static char ID; // Pass identification, replacement for typeid.
-  StackProtector() : FunctionPass(ID), TM(0), TLI(0), SSPBufferSize(0) {
+  StackProtector()
+      : FunctionPass(ID), TM(nullptr), TLI(nullptr), SSPBufferSize(0) {
     initializeStackProtectorPass(*PassRegistry::getPassRegistry());
   }
   StackProtector(const TargetMachine *TM)
-      : FunctionPass(ID), TM(TM), TLI(0), Trip(TM->getTargetTriple()),
+      : FunctionPass(ID), TM(TM), TLI(nullptr), Trip(TM->getTargetTriple()),
         SSPBufferSize(8) {
     initializeStackProtectorPass(*PassRegistry::getPassRegistry());
   }
 
-  virtual void getAnalysisUsage(AnalysisUsage &AU) const {
-    AU.addPreserved<DominatorTree>();
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.addPreserved<DominatorTreeWrapperPass>();
   }
 
   SSPLayoutKind getSSPLayout(const AllocaInst *AI) const;
+  void adjustForColoring(const AllocaInst *From, const AllocaInst *To);
 
-  virtual bool runOnFunction(Function &Fn);
+  bool runOnFunction(Function &Fn) override;
 };
 } // end namespace llvm
 
diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
index 5b22c9c685ae..87f140190a75 100644
--- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
+++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h
@@ -38,40 +38,40 @@ class TargetLoweringObjectFileELF : public TargetLoweringObjectFile {
 public:
   virtual ~TargetLoweringObjectFileELF() {}
 
-  virtual void emitPersonalityValue(MCStreamer &Streamer,
-                                    const TargetMachine &TM,
-                                    const MCSymbol *Sym) const;
-
-  /// getSectionForConstant - Given a constant with the SectionKind, return a
-  /// section that it should be placed in.
-  virtual const MCSection *getSectionForConstant(SectionKind Kind) const;
+  void emitPersonalityValue(MCStreamer &Streamer, const TargetMachine &TM,
+                            const MCSymbol *Sym) const override;
 
+  /// Given a constant with the SectionKind, return a section that it should be
+  /// placed in.
+  const MCSection *getSectionForConstant(SectionKind Kind,
+                                         const Constant *C) const override;
 
-  virtual const MCSection *
-  getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
-                           Mangler *Mang, const TargetMachine &TM) const;
+  const MCSection *getExplicitSectionGlobal(const GlobalValue *GV,
+                                        SectionKind Kind, Mangler &Mang,
+                                        const TargetMachine &TM) const override;
 
-  virtual const MCSection *
-  SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
-                         Mangler *Mang, const TargetMachine &TM) const;
+  const MCSection *SelectSectionForGlobal(const GlobalValue *GV,
+                                        SectionKind Kind, Mangler &Mang,
+                                        const TargetMachine &TM) const override;
 
-  /// getTTypeGlobalReference - Return an MCExpr to use for a reference to the
-  /// specified type info global variable from exception handling information.
-  virtual const MCExpr *
-  getTTypeGlobalReference(const GlobalValue *GV, Mangler *Mang,
-                          MachineModuleInfo *MMI, unsigned Encoding,
-                          MCStreamer &Streamer) const;
+  /// Return an MCExpr to use for a reference to the specified type info global
+  /// variable from exception handling information.
+  const MCExpr *
+  getTTypeGlobalReference(const GlobalValue *GV, unsigned Encoding,
+                          Mangler &Mang, const TargetMachine &TM,
+                          MachineModuleInfo *MMI,
+                          MCStreamer &Streamer) const override;
 
-  // getCFIPersonalitySymbol - The symbol that gets passed to .cfi_personality.
-  virtual MCSymbol *
-  getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang,
-                          MachineModuleInfo *MMI) const;
+  // The symbol that gets passed to .cfi_personality.
+  MCSymbol *getCFIPersonalitySymbol(const GlobalValue *GV, Mangler &Mang,
+                                    const TargetMachine &TM,
+                                    MachineModuleInfo *MMI) const override;
 
   void InitializeELF(bool UseInitArray_);
-  virtual const MCSection *
-  getStaticCtorSection(unsigned Priority = 65535) const;
-  virtual const MCSection *
-  getStaticDtorSection(unsigned Priority = 65535) const;
+  const MCSection *getStaticCtorSection(unsigned Priority,
+                                        const MCSymbol *KeySym) const override;
+  const MCSection *getStaticDtorSection(unsigned Priority,
+                                        const MCSymbol *KeySym) const override;
 };
 
 
@@ -80,39 +80,41 @@ class TargetLoweringObjectFileMachO : public TargetLoweringObjectFile {
 public:
   virtual ~TargetLoweringObjectFileMachO() {}
 
-  /// emitModuleFlags - Emit the module flags that specify the garbage
-  /// collection information.
-  virtual void emitModuleFlags(MCStreamer &Streamer,
-                               ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
-                               Mangler *Mang, const TargetMachine &TM) const;
+  /// Extract the dependent library name from a linker option string. Returns
+  /// StringRef() if the option does not specify a library.
+  StringRef getDepLibFromLinkerOpt(StringRef LinkerOption) const override;
+
+  /// Emit the module flags that specify the garbage collection information.
+  void emitModuleFlags(MCStreamer &Streamer,
+                       ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
+                       Mangler &Mang, const TargetMachine &TM) const override;
 
-  virtual const MCSection *
-  SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
-                         Mangler *Mang, const TargetMachine &TM) const;
+  bool isSectionAtomizableBySymbols(const MCSection &Section) const override;
 
-  virtual const MCSection *
-  getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
-                           Mangler *Mang, const TargetMachine &TM) const;
+  const MCSection *
+    SelectSectionForGlobal(const GlobalValue *GV,
+                           SectionKind Kind, Mangler &Mang,
+                           const TargetMachine &TM) const override;
 
-  virtual const MCSection *getSectionForConstant(SectionKind Kind) const;
+  const MCSection *
+    getExplicitSectionGlobal(const GlobalValue *GV,
+                             SectionKind Kind, Mangler &Mang,
+                             const TargetMachine &TM) const override;
 
-  /// shouldEmitUsedDirectiveFor - This hook allows targets to selectively
-  /// decide not to emit the UsedDirective for some symbols in llvm.used.
-  /// FIXME: REMOVE this (rdar://7071300)
-  virtual bool shouldEmitUsedDirectiveFor(const GlobalValue *GV,
-                                          Mangler *) const;
+  const MCSection *getSectionForConstant(SectionKind Kind,
+                                         const Constant *C) const override;
 
-  /// getTTypeGlobalReference - The mach-o version of this method
-  /// defaults to returning a stub reference.
-  virtual const MCExpr *
-  getTTypeGlobalReference(const GlobalValue *GV, Mangler *Mang,
-                          MachineModuleInfo *MMI, unsigned Encoding,
-                          MCStreamer &Streamer) const;
+  /// The mach-o version of this method defaults to returning a stub reference.
+  const MCExpr *
+  getTTypeGlobalReference(const GlobalValue *GV, unsigned Encoding,
+                          Mangler &Mang, const TargetMachine &TM,
+                          MachineModuleInfo *MMI,
+                          MCStreamer &Streamer) const override;
 
-  // getCFIPersonalitySymbol - The symbol that gets passed to .cfi_personality.
-  virtual MCSymbol *
-  getCFIPersonalitySymbol(const GlobalValue *GV, Mangler *Mang,
-                          MachineModuleInfo *MMI) const;
+  // The symbol that gets passed to .cfi_personality.
+  MCSymbol *getCFIPersonalitySymbol(const GlobalValue *GV, Mangler &Mang,
+                                    const TargetMachine &TM,
+                                    MachineModuleInfo *MMI) const override;
 };
 
 
@@ -121,19 +123,30 @@ class TargetLoweringObjectFileCOFF : public TargetLoweringObjectFile {
 public:
   virtual ~TargetLoweringObjectFileCOFF() {}
 
-  virtual const MCSection *
-  getExplicitSectionGlobal(const GlobalValue *GV, SectionKind Kind,
-                           Mangler *Mang, const TargetMachine &TM) const;
+  const MCSection *
+    getExplicitSectionGlobal(const GlobalValue *GV,
+                             SectionKind Kind, Mangler &Mang,
+                             const TargetMachine &TM) const override;
+
+  const MCSection *
+    SelectSectionForGlobal(const GlobalValue *GV,
+                           SectionKind Kind, Mangler &Mang,
+                           const TargetMachine &TM) const override;
+
+  /// Extract the dependent library name from a linker option string. Returns
+  /// StringRef() if the option does not specify a library.
+  StringRef getDepLibFromLinkerOpt(StringRef LinkerOption) const override;
 
-  virtual const MCSection *
-  SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
-                         Mangler *Mang, const TargetMachine &TM) const;
+  /// Emit Obj-C garbage collection and linker options. Only linker option
+  /// emission is implemented for COFF.
+  void emitModuleFlags(MCStreamer &Streamer,
+                       ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
+                       Mangler &Mang, const TargetMachine &TM) const override;
 
-  /// emitModuleFlags - Emit Obj-C garbage collection and linker options.  Only
-  /// linker option emission is implemented for COFF.
-  virtual void emitModuleFlags(MCStreamer &Streamer,
-                               ArrayRef<Module::ModuleFlagEntry> ModuleFlags,
-                               Mangler *Mang, const TargetMachine &TM) const;
+  const MCSection *getStaticCtorSection(unsigned Priority,
+                                        const MCSymbol *KeySym) const override;
+  const MCSection *getStaticDtorSection(unsigned Priority,
+                                        const MCSymbol *KeySym) const override;
 };
 
 } // end namespace llvm
diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h
index 8ef26b7ca548..690b70fad89b 100644
--- a/include/llvm/CodeGen/TargetSchedule.h
+++ b/include/llvm/CodeGen/TargetSchedule.h
@@ -41,7 +41,7 @@ class TargetSchedModel {
   unsigned MicroOpFactor; // Multiply to normalize microops to resource units.
   unsigned ResourceLCM;   // Resource units per cycle. Latency normalization factor.
 public:
-  TargetSchedModel(): STI(0), TII(0) {}
+  TargetSchedModel(): STI(nullptr), TII(nullptr) {}
 
   /// \brief Initialize the machine model for instruction scheduling.
   ///
@@ -75,7 +75,7 @@ public:
   const InstrItineraryData *getInstrItineraries() const {
     if (hasInstrItineraries())
       return &InstrItins;
-    return 0;
+    return nullptr;
   }
 
   /// \brief Identify the processor corresponding to the current subtarget.
@@ -86,7 +86,7 @@ public:
 
   /// \brief Return the number of issue slots required for this MI.
   unsigned getNumMicroOps(const MachineInstr *MI,
-                          const MCSchedClassDesc *SC = 0) const;
+                          const MCSchedClassDesc *SC = nullptr) const;
 
   /// \brief Get the number of kinds of resources for this target.
   unsigned getNumProcResourceKinds() const {
@@ -98,6 +98,14 @@ public:
     return SchedModel.getProcResource(PIdx);
   }
 
+#ifndef NDEBUG
+  const char *getResourceName(unsigned PIdx) const {
+    if (!PIdx)
+      return "MOps";
+    return SchedModel.getProcResource(PIdx)->Name;
+  }
+#endif
+
   typedef const MCWriteProcResEntry *ProcResIter;
 
   // \brief Get an iterator into the processor resources consumed by this
@@ -150,7 +158,7 @@ public:
   /// model.
   ///
   /// Compute and return the expected latency of this instruction independent of
-  /// a particular use. computeOperandLatency is the prefered API, but this is
+  /// a particular use. computeOperandLatency is the preferred API, but this is
   /// occasionally useful to help estimate instruction cost.
   ///
   /// If UseDefaultDefLatency is false and no new machine sched model is
diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h
index 79f323341fd0..4e93940e223e 100644
--- a/include/llvm/CodeGen/ValueTypes.h
+++ b/include/llvm/CodeGen/ValueTypes.h
@@ -16,559 +16,14 @@
 #ifndef LLVM_CODEGEN_VALUETYPES_H
 #define LLVM_CODEGEN_VALUETYPES_H
 
-#include "llvm/Support/DataTypes.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MathExtras.h"
+#include "llvm/CodeGen/MachineValueType.h"
 #include <cassert>
 #include <string>
 
 namespace llvm {
-  class Type;
-  class LLVMContext;
-  struct EVT;
-
-  /// MVT - Machine Value Type. Every type that is supported natively by some
-  /// processor targeted by LLVM occurs here. This means that any legal value
-  /// type can be represented by an MVT.
-  class MVT {
-  public:
-    enum SimpleValueType {
-      // INVALID_SIMPLE_VALUE_TYPE - Simple value types less than zero are
-      // considered extended value types.
-      INVALID_SIMPLE_VALUE_TYPE = -1,
-
-      // If you change this numbering, you must change the values in
-      // ValueTypes.td as well!
-      Other          =   0,   // This is a non-standard value
-      i1             =   1,   // This is a 1 bit integer value
-      i8             =   2,   // This is an 8 bit integer value
-      i16            =   3,   // This is a 16 bit integer value
-      i32            =   4,   // This is a 32 bit integer value
-      i64            =   5,   // This is a 64 bit integer value
-      i128           =   6,   // This is a 128 bit integer value
-
-      FIRST_INTEGER_VALUETYPE = i1,
-      LAST_INTEGER_VALUETYPE  = i128,
-
-      f16            =   7,   // This is a 16 bit floating point value
-      f32            =   8,   // This is a 32 bit floating point value
-      f64            =   9,   // This is a 64 bit floating point value
-      f80            =  10,   // This is a 80 bit floating point value
-      f128           =  11,   // This is a 128 bit floating point value
-      ppcf128        =  12,   // This is a PPC 128-bit floating point value
-
-      FIRST_FP_VALUETYPE = f16,
-      LAST_FP_VALUETYPE  = ppcf128,
-
-      v2i1           =  13,   //  2 x i1
-      v4i1           =  14,   //  4 x i1
-      v8i1           =  15,   //  8 x i1
-      v16i1          =  16,   // 16 x i1
-      v32i1          =  17,   // 32 x i1
-      v64i1          =  18,   // 64 x i1
-
-      v1i8           =  19,   //  1 x i8
-      v2i8           =  20,   //  2 x i8
-      v4i8           =  21,   //  4 x i8
-      v8i8           =  22,   //  8 x i8
-      v16i8          =  23,   // 16 x i8
-      v32i8          =  24,   // 32 x i8
-      v64i8          =  25,   // 64 x i8
-      v1i16          =  26,   //  1 x i16
-      v2i16          =  27,   //  2 x i16
-      v4i16          =  28,   //  4 x i16
-      v8i16          =  29,   //  8 x i16
-      v16i16         =  30,   // 16 x i16
-      v32i16         =  31,   // 32 x i16
-      v1i32          =  32,   //  1 x i32
-      v2i32          =  33,   //  2 x i32
-      v4i32          =  34,   //  4 x i32
-      v8i32          =  35,   //  8 x i32
-      v16i32         =  36,   // 16 x i32
-      v1i64          =  37,   //  1 x i64
-      v2i64          =  38,   //  2 x i64
-      v4i64          =  39,   //  4 x i64
-      v8i64          =  40,   //  8 x i64
-      v16i64         =  41,   // 16 x i64
-
-      FIRST_INTEGER_VECTOR_VALUETYPE = v2i1,
-      LAST_INTEGER_VECTOR_VALUETYPE = v16i64,
-
-      v2f16          =  42,   //  2 x f16
-      v4f16          =  43,   //  4 x f16
-      v8f16          =  44,   //  8 x f16
-      v1f32          =  45,   //  1 x f32
-      v2f32          =  46,   //  2 x f32
-      v4f32          =  47,   //  4 x f32
-      v8f32          =  48,   //  8 x f32
-      v16f32         =  49,   // 16 x f32
-      v1f64          =  50,   //  1 x f64
-      v2f64          =  51,   //  2 x f64
-      v4f64          =  52,   //  4 x f64
-      v8f64          =  53,   //  8 x f64
-
-      FIRST_FP_VECTOR_VALUETYPE = v2f16,
-      LAST_FP_VECTOR_VALUETYPE = v8f64,
-
-      FIRST_VECTOR_VALUETYPE = v2i1,
-      LAST_VECTOR_VALUETYPE  = v8f64,
-
-      x86mmx         =  54,   // This is an X86 MMX value
-
-      Glue           =  55,   // This glues nodes together during pre-RA sched
-
-      isVoid         =  56,   // This has no value
-
-      Untyped        =  57,   // This value takes a register, but has
-                              // unspecified type.  The register class
-                              // will be determined by the opcode.
-
-      LAST_VALUETYPE =  58,   // This always remains at the end of the list.
-
-      // This is the current maximum for LAST_VALUETYPE.
-      // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors
-      // This value must be a multiple of 32.
-      MAX_ALLOWED_VALUETYPE = 64,
-
-      // Metadata - This is MDNode or MDString.
-      Metadata       = 250,
-
-      // iPTRAny - An int value the size of the pointer of the current
-      // target to any address space. This must only be used internal to
-      // tblgen. Other than for overloading, we treat iPTRAny the same as iPTR.
-      iPTRAny        = 251,
-
-      // vAny - A vector with any length and element size. This is used
-      // for intrinsics that have overloadings based on vector types.
-      // This is only for tblgen's consumption!
-      vAny           = 252,
-
-      // fAny - Any floating-point or vector floating-point value. This is used
-      // for intrinsics that have overloadings based on floating-point types.
-      // This is only for tblgen's consumption!
-      fAny           = 253,
-
-      // iAny - An integer or vector integer value of any bit width. This is
-      // used for intrinsics that have overloadings based on integer bit widths.
-      // This is only for tblgen's consumption!
-      iAny           = 254,
-
-      // iPTR - An int value the size of the pointer of the current
-      // target.  This should only be used internal to tblgen!
-      iPTR           = 255
-    };
-
-    SimpleValueType SimpleTy;
-
-    MVT() : SimpleTy((SimpleValueType)(INVALID_SIMPLE_VALUE_TYPE)) {}
-    MVT(SimpleValueType SVT) : SimpleTy(SVT) { }
-
-    bool operator>(const MVT& S)  const { return SimpleTy >  S.SimpleTy; }
-    bool operator<(const MVT& S)  const { return SimpleTy <  S.SimpleTy; }
-    bool operator==(const MVT& S) const { return SimpleTy == S.SimpleTy; }
-    bool operator!=(const MVT& S) const { return SimpleTy != S.SimpleTy; }
-    bool operator>=(const MVT& S) const { return SimpleTy >= S.SimpleTy; }
-    bool operator<=(const MVT& S) const { return SimpleTy <= S.SimpleTy; }
-
-    /// isFloatingPoint - Return true if this is a FP, or a vector FP type.
-    bool isFloatingPoint() const {
-      return ((SimpleTy >= MVT::FIRST_FP_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_FP_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_FP_VECTOR_VALUETYPE));
-    }
-
-    /// isInteger - Return true if this is an integer, or a vector integer type.
-    bool isInteger() const {
-      return ((SimpleTy >= MVT::FIRST_INTEGER_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_VALUETYPE) ||
-              (SimpleTy >= MVT::FIRST_INTEGER_VECTOR_VALUETYPE &&
-               SimpleTy <= MVT::LAST_INTEGER_VECTOR_VALUETYPE));
-    }
-
-    /// isVector - Return true if this is a vector value type.
-    bool isVector() const {
-      return (SimpleTy >= MVT::FIRST_VECTOR_VALUETYPE &&
-              SimpleTy <= MVT::LAST_VECTOR_VALUETYPE);
-    }
-
-    /// is16BitVector - Return true if this is a 16-bit vector type.
-    bool is16BitVector() const {
-      return (SimpleTy == MVT::v2i8  || SimpleTy == MVT::v1i16 ||
-              SimpleTy == MVT::v16i1);
-    }
-
-    /// is32BitVector - Return true if this is a 32-bit vector type.
-    bool is32BitVector() const {
-      return (SimpleTy == MVT::v4i8  || SimpleTy == MVT::v2i16 ||
-              SimpleTy == MVT::v1i32);
-    }
-
-    /// is64BitVector - Return true if this is a 64-bit vector type.
-    bool is64BitVector() const {
-      return (SimpleTy == MVT::v8i8  || SimpleTy == MVT::v4i16 ||
-              SimpleTy == MVT::v2i32 || SimpleTy == MVT::v1i64 ||
-              SimpleTy == MVT::v1f64 || SimpleTy == MVT::v2f32);
-    }
-
-    /// is128BitVector - Return true if this is a 128-bit vector type.
-    bool is128BitVector() const {
-      return (SimpleTy == MVT::v16i8 || SimpleTy == MVT::v8i16 ||
-              SimpleTy == MVT::v4i32 || SimpleTy == MVT::v2i64 ||
-              SimpleTy == MVT::v4f32 || SimpleTy == MVT::v2f64);
-    }
-
-    /// is256BitVector - Return true if this is a 256-bit vector type.
-    bool is256BitVector() const {
-      return (SimpleTy == MVT::v8f32 || SimpleTy == MVT::v4f64  ||
-              SimpleTy == MVT::v32i8 || SimpleTy == MVT::v16i16 ||
-              SimpleTy == MVT::v8i32 || SimpleTy == MVT::v4i64);
-    }
-
-    /// is512BitVector - Return true if this is a 512-bit vector type.
-    bool is512BitVector() const {
-      return (SimpleTy == MVT::v8f64 || SimpleTy == MVT::v16f32 ||
-              SimpleTy == MVT::v64i8 || SimpleTy == MVT::v32i16 ||
-              SimpleTy == MVT::v8i64 || SimpleTy == MVT::v16i32);
-    }
-
-    /// is1024BitVector - Return true if this is a 1024-bit vector type.
-    bool is1024BitVector() const {
-      return (SimpleTy == MVT::v16i64);
-    }
-
-    /// isPow2VectorType - Returns true if the given vector is a power of 2.
-    bool isPow2VectorType() const {
-      unsigned NElts = getVectorNumElements();
-      return !(NElts & (NElts - 1));
-    }
-
-    /// getPow2VectorType - Widens the length of the given vector MVT up to
-    /// the nearest power of 2 and returns that type.
-    MVT getPow2VectorType() const {
-      if (isPow2VectorType())
-        return *this;
-
-      unsigned NElts = getVectorNumElements();
-      unsigned Pow2NElts = 1 << Log2_32_Ceil(NElts);
-      return MVT::getVectorVT(getVectorElementType(), Pow2NElts);
-    }
-
-    /// getScalarType - If this is a vector type, return the element type,
-    /// otherwise return this.
-    MVT getScalarType() const {
-      return isVector() ? getVectorElementType() : *this;
-    }
-
-    MVT getVectorElementType() const {
-      switch (SimpleTy) {
-      default:
-        llvm_unreachable("Not a vector MVT!");
-      case v2i1 :
-      case v4i1 :
-      case v8i1 :
-      case v16i1 :
-      case v32i1 :
-      case v64i1: return i1;
-      case v1i8 :
-      case v2i8 :
-      case v4i8 :
-      case v8i8 :
-      case v16i8:
-      case v32i8:
-      case v64i8: return i8;
-      case v1i16:
-      case v2i16:
-      case v4i16:
-      case v8i16:
-      case v16i16:
-      case v32i16: return i16;
-      case v1i32:
-      case v2i32:
-      case v4i32:
-      case v8i32:
-      case v16i32: return i32;
-      case v1i64:
-      case v2i64:
-      case v4i64:
-      case v8i64:
-      case v16i64: return i64;
-      case v2f16:
-      case v4f16:
-      case v8f16: return f16;
-      case v1f32:
-      case v2f32:
-      case v4f32:
-      case v8f32:
-      case v16f32: return f32;
-      case v1f64:
-      case v2f64:
-      case v4f64:
-      case v8f64: return f64;
-      }
-    }
-
-    unsigned getVectorNumElements() const {
-      switch (SimpleTy) {
-      default:
-        llvm_unreachable("Not a vector MVT!");
-      case v32i1:
-      case v32i8:
-      case v32i16: return 32;
-      case v64i1:
-      case v64i8: return 64;
-      case v16i1:
-      case v16i8:
-      case v16i16:
-      case v16i32:
-      case v16i64:
-      case v16f32: return 16;
-      case v8i1 :
-      case v8i8 :
-      case v8i16:
-      case v8i32:
-      case v8i64:
-      case v8f16:
-      case v8f32:
-      case v8f64: return 8;
-      case v4i1:
-      case v4i8:
-      case v4i16:
-      case v4i32:
-      case v4i64:
-      case v4f16:
-      case v4f32:
-      case v4f64: return 4;
-      case v2i1:
-      case v2i8:
-      case v2i16:
-      case v2i32:
-      case v2i64:
-      case v2f16:
-      case v2f32:
-      case v2f64: return 2;
-      case v1i8:
-      case v1i16:
-      case v1i32:
-      case v1i64:
-      case v1f32:
-      case v1f64: return 1;
-      }
-    }
-
-    unsigned getSizeInBits() const {
-      switch (SimpleTy) {
-      default:
-        llvm_unreachable("getSizeInBits called on extended MVT.");
-      case Other:
-        llvm_unreachable("Value type is non-standard value, Other.");
-      case iPTR:
-        llvm_unreachable("Value type size is target-dependent. Ask TLI.");
-      case iPTRAny:
-      case iAny:
-      case fAny:
-      case vAny:
-        llvm_unreachable("Value type is overloaded.");
-      case Metadata:
-        llvm_unreachable("Value type is metadata.");
-      case i1  :  return 1;
-      case v2i1:  return 2;
-      case v4i1:  return 4;
-      case i8  :
-      case v1i8:
-      case v8i1: return 8;
-      case i16 :
-      case f16:
-      case v16i1:
-      case v2i8:
-      case v1i16: return 16;
-      case f32 :
-      case i32 :
-      case v32i1:
-      case v4i8:
-      case v2i16:
-      case v2f16:
-      case v1f32:
-      case v1i32: return 32;
-      case x86mmx:
-      case f64 :
-      case i64 :
-      case v64i1:
-      case v8i8:
-      case v4i16:
-      case v2i32:
-      case v1i64:
-      case v4f16:
-      case v2f32:
-      case v1f64: return 64;
-      case f80 :  return 80;
-      case f128:
-      case ppcf128:
-      case i128:
-      case v16i8:
-      case v8i16:
-      case v4i32:
-      case v2i64:
-      case v8f16:
-      case v4f32:
-      case v2f64: return 128;
-      case v32i8:
-      case v16i16:
-      case v8i32:
-      case v4i64:
-      case v8f32:
-      case v4f64: return 256;
-      case v64i8:
-      case v32i16:
-      case v16i32:
-      case v8i64:
-      case v16f32:
-      case v8f64: return 512;
-      case v16i64:return 1024;
-      }
-    }
-
-    /// getStoreSize - Return the number of bytes overwritten by a store
-    /// of the specified value type.
-    unsigned getStoreSize() const {
-      return (getSizeInBits() + 7) / 8;
-    }
-
-    /// getStoreSizeInBits - Return the number of bits overwritten by a store
-    /// of the specified value type.
-    unsigned getStoreSizeInBits() const {
-      return getStoreSize() * 8;
-    }
-
-    /// Return true if this has more bits than VT.
-    bool bitsGT(MVT VT) const {
-      return getSizeInBits() > VT.getSizeInBits();
-    }
-
-    /// Return true if this has no less bits than VT.
-    bool bitsGE(MVT VT) const {
-      return getSizeInBits() >= VT.getSizeInBits();
-    }
-
-    /// Return true if this has less bits than VT.
-    bool bitsLT(MVT VT) const {
-      return getSizeInBits() < VT.getSizeInBits();
-    }
-
-    /// Return true if this has no more bits than VT.
-    bool bitsLE(MVT VT) const {
-      return getSizeInBits() <= VT.getSizeInBits();
-    }
-
-
-    static MVT getFloatingPointVT(unsigned BitWidth) {
-      switch (BitWidth) {
-      default:
-        llvm_unreachable("Bad bit width!");
-      case 16:
-        return MVT::f16;
-      case 32:
-        return MVT::f32;
-      case 64:
-        return MVT::f64;
-      case 80:
-        return MVT::f80;
-      case 128:
-        return MVT::f128;
-      }
-    }
-
-    static MVT getIntegerVT(unsigned BitWidth) {
-      switch (BitWidth) {
-      default:
-        return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
-      case 1:
-        return MVT::i1;
-      case 8:
-        return MVT::i8;
-      case 16:
-        return MVT::i16;
-      case 32:
-        return MVT::i32;
-      case 64:
-        return MVT::i64;
-      case 128:
-        return MVT::i128;
-      }
-    }
-
-    static MVT getVectorVT(MVT VT, unsigned NumElements) {
-      switch (VT.SimpleTy) {
-      default:
-        break;
-      case MVT::i1:
-        if (NumElements == 2)  return MVT::v2i1;
-        if (NumElements == 4)  return MVT::v4i1;
-        if (NumElements == 8)  return MVT::v8i1;
-        if (NumElements == 16) return MVT::v16i1;
-        if (NumElements == 32) return MVT::v32i1;
-        if (NumElements == 64) return MVT::v64i1;
-        break;
-      case MVT::i8:
-        if (NumElements == 1)  return MVT::v1i8;
-        if (NumElements == 2)  return MVT::v2i8;
-        if (NumElements == 4)  return MVT::v4i8;
-        if (NumElements == 8)  return MVT::v8i8;
-        if (NumElements == 16) return MVT::v16i8;
-        if (NumElements == 32) return MVT::v32i8;
-        if (NumElements == 64) return MVT::v64i8;
-        break;
-      case MVT::i16:
-        if (NumElements == 1)  return MVT::v1i16;
-        if (NumElements == 2)  return MVT::v2i16;
-        if (NumElements == 4)  return MVT::v4i16;
-        if (NumElements == 8)  return MVT::v8i16;
-        if (NumElements == 16) return MVT::v16i16;
-        if (NumElements == 32) return MVT::v32i16;
-        break;
-      case MVT::i32:
-        if (NumElements == 1)  return MVT::v1i32;
-        if (NumElements == 2)  return MVT::v2i32;
-        if (NumElements == 4)  return MVT::v4i32;
-        if (NumElements == 8)  return MVT::v8i32;
-        if (NumElements == 16) return MVT::v16i32;
-        break;
-      case MVT::i64:
-        if (NumElements == 1)  return MVT::v1i64;
-        if (NumElements == 2)  return MVT::v2i64;
-        if (NumElements == 4)  return MVT::v4i64;
-        if (NumElements == 8)  return MVT::v8i64;
-        if (NumElements == 16) return MVT::v16i64;
-        break;
-      case MVT::f16:
-        if (NumElements == 2)  return MVT::v2f16;
-        if (NumElements == 4)  return MVT::v4f16;
-        if (NumElements == 8)  return MVT::v8f16;
-        break;
-      case MVT::f32:
-        if (NumElements == 1)  return MVT::v1f32;
-        if (NumElements == 2)  return MVT::v2f32;
-        if (NumElements == 4)  return MVT::v4f32;
-        if (NumElements == 8)  return MVT::v8f32;
-        if (NumElements == 16) return MVT::v16f32;
-        break;
-      case MVT::f64:
-        if (NumElements == 1)  return MVT::v1f64;
-        if (NumElements == 2)  return MVT::v2f64;
-        if (NumElements == 4)  return MVT::v4f64;
-        if (NumElements == 8)  return MVT::v8f64;
-        break;
-      }
-      return (MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE);
-    }
-
-    /// Return the value type corresponding to the specified type.  This returns
-    /// all pointers as iPTR.  If HandleUnknown is true, unknown types are
-    /// returned as Other, otherwise they are invalid.
-    static MVT getVT(Type *Ty, bool HandleUnknown = false);
-
-  };
 
+  class LLVMContext;
+  class Type;
 
   /// EVT - Extended Value Type.  Capable of holding value types which are not
   /// native for any processor (such as the i12345 type), as well as the types
@@ -580,9 +35,9 @@ namespace llvm {
 
   public:
     EVT() : V((MVT::SimpleValueType)(MVT::INVALID_SIMPLE_VALUE_TYPE)),
-            LLVMTy(0) {}
-    EVT(MVT::SimpleValueType SVT) : V(SVT), LLVMTy(0) { }
-    EVT(MVT S) : V(S), LLVMTy(0) {}
+            LLVMTy(nullptr) {}
+    EVT(MVT::SimpleValueType SVT) : V(SVT), LLVMTy(nullptr) { }
+    EVT(MVT S) : V(S), LLVMTy(nullptr) {}
 
     bool operator==(EVT VT) const {
       return !(*this != VT);
@@ -782,6 +237,10 @@ namespace llvm {
       return getExtendedSizeInBits();
     }
 
+    unsigned getScalarSizeInBits() const {
+      return getScalarType().getSizeInBits();
+    }
+
     /// getStoreSize - Return the number of bytes overwritten by a store
     /// of the specified value type.
     unsigned getStoreSize() const {
@@ -821,6 +280,14 @@ namespace llvm {
       return getIntegerVT(Context, (EVTSize + 1) / 2);
     }
 
+    /// \brief Return a VT for an integer vector type with the size of the
+    /// elements doubled. The typed returned may be an extended type.
+    EVT widenIntegerVectorElementType(LLVMContext &Context) const {
+      EVT EltVT = getVectorElementType();
+      EltVT = EVT::getIntegerVT(Context, 2 * EltVT.getSizeInBits());
+      return EVT::getVectorVT(Context, EltVT, getVectorNumElements());
+    }
+
     /// isPow2VectorType - Returns true if the given vector is a power of 2.
     bool isPow2VectorType() const {
       unsigned NElts = getVectorNumElements();
@@ -880,18 +347,18 @@ namespace llvm {
     static EVT getExtendedIntegerVT(LLVMContext &C, unsigned BitWidth);
     static EVT getExtendedVectorVT(LLVMContext &C, EVT VT,
                                    unsigned NumElements);
-    bool isExtendedFloatingPoint() const;
-    bool isExtendedInteger() const;
-    bool isExtendedVector() const;
-    bool isExtended16BitVector() const;
-    bool isExtended32BitVector() const;
-    bool isExtended64BitVector() const;
-    bool isExtended128BitVector() const;
-    bool isExtended256BitVector() const;
-    bool isExtended512BitVector() const;
-    bool isExtended1024BitVector() const;
+    bool isExtendedFloatingPoint() const LLVM_READONLY;
+    bool isExtendedInteger() const LLVM_READONLY;
+    bool isExtendedVector() const LLVM_READONLY;
+    bool isExtended16BitVector() const LLVM_READONLY;
+    bool isExtended32BitVector() const LLVM_READONLY;
+    bool isExtended64BitVector() const LLVM_READONLY;
+    bool isExtended128BitVector() const LLVM_READONLY;
+    bool isExtended256BitVector() const LLVM_READONLY;
+    bool isExtended512BitVector() const LLVM_READONLY;
+    bool isExtended1024BitVector() const LLVM_READONLY;
     EVT getExtendedVectorElementType() const;
-    unsigned getExtendedVectorNumElements() const;
+    unsigned getExtendedVectorNumElements() const LLVM_READONLY;
     unsigned getExtendedSizeInBits() const;
   };
 
diff --git a/include/llvm/CodeGen/VirtRegMap.h b/include/llvm/CodeGen/VirtRegMap.h
index 3bc6ebd563f2..eceb8755763e 100644
--- a/include/llvm/CodeGen/VirtRegMap.h
+++ b/include/llvm/CodeGen/VirtRegMap.h
@@ -70,9 +70,9 @@ namespace llvm {
     static char ID;
     VirtRegMap() : MachineFunctionPass(ID), Virt2PhysMap(NO_PHYS_REG),
                    Virt2StackSlotMap(NO_STACK_SLOT), Virt2SplitMap(0) { }
-    virtual bool runOnMachineFunction(MachineFunction &MF);
+    bool runOnMachineFunction(MachineFunction &MF) override;
 
-    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+    void getAnalysisUsage(AnalysisUsage &AU) const override {
       AU.setPreservesAll();
       MachineFunctionPass::getAnalysisUsage(AU);
     }
@@ -177,7 +177,7 @@ namespace llvm {
     /// the specified stack slot
     void assignVirt2StackSlot(unsigned virtReg, int frameIndex);
 
-    void print(raw_ostream &OS, const Module* M = 0) const;
+    void print(raw_ostream &OS, const Module* M = nullptr) const override;
     void dump() const;
   };