26 files changed, 557 insertions, 366 deletions

diff --git a/lib/Target/ARM/ARM.td b/lib/Target/ARM/ARM.td
index 70338612d9c4..bbb1dbdc9a4a 100644
--- a/lib/Target/ARM/ARM.td
+++ b/lib/Target/ARM/ARM.td
@@ -40,6 +40,8 @@ def FeatureNEON : SubtargetFeature<"neon", "ARMFPUType", "NEON",
                                    "Enable NEON instructions">;
 def FeatureThumb2 : SubtargetFeature<"thumb2", "ThumbMode", "Thumb2",
                                      "Enable Thumb2 instructions">;
+def FeatureFP16   : SubtargetFeature<"fp16", "HasFP16", "true",
+                                     "Enable half-precision floating point">;
 
 //===----------------------------------------------------------------------===//
 // ARM Processors supported.
diff --git a/lib/Target/ARM/ARMBaseInstrInfo.h b/lib/Target/ARM/ARMBaseInstrInfo.h
index 01942319b219..767d5ec92651 100644
--- a/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -58,12 +58,13 @@ namespace ARMII {
     Size4Bytes    = 3,
     Size2Bytes    = 4,
 
-    // IndexMode - Unindex, pre-indexed, or post-indexed. Only valid for load
-    // and store ops
+    // IndexMode - Unindex, pre-indexed, or post-indexed are valid for load
+    // and store ops only.  Generic "updating" flag is used for ld/st multiple.
     IndexModeShift = 7,
     IndexModeMask  = 3 << IndexModeShift,
     IndexModePre   = 1,
     IndexModePost  = 2,
+    IndexModeUpd   = 3,
 
     //===------------------------------------------------------------------===//
     // Instruction encoding formats.
diff --git a/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index d9b8323f4ae1..11e1c4862b41 100644
--- a/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -590,6 +590,10 @@ ARMBaseRegisterInfo::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
       AFI->isThumb2Function())
     MF.getRegInfo().setPhysRegUsed(ARM::R4);
 
+  // Spill LR if Thumb1 function uses variable length argument lists.
+  if (AFI->isThumb1OnlyFunction() && AFI->getVarArgsRegSaveSize() > 0)
+    MF.getRegInfo().setPhysRegUsed(ARM::LR);
+
   // Don't spill FP if the frame can be eliminated. This is determined
   // by scanning the callee-save registers to see if any is used.
   const unsigned *CSRegs = getCalleeSavedRegs();
diff --git a/lib/Target/ARM/ARMCodeEmitter.cpp b/lib/Target/ARM/ARMCodeEmitter.cpp
index 108a2445c3ff..334c820b91cb 100644
--- a/lib/Target/ARM/ARMCodeEmitter.cpp
+++ b/lib/Target/ARM/ARMCodeEmitter.cpp
@@ -51,6 +51,7 @@ namespace {
     const ARMSubtarget        *Subtarget;
     TargetMachine             &TM;
     JITCodeEmitter            &MCE;
+    MachineModuleInfo *MMI;
     const std::vector<MachineConstantPoolEntry> *MCPEs;
     const std::vector<MachineJumpTableEntry> *MJTEs;
     bool IsPIC;
@@ -182,7 +183,8 @@ bool ARMCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
   if (MF.getJumpTableInfo()) MJTEs = &MF.getJumpTableInfo()->getJumpTables();
   IsPIC = TM.getRelocationModel() == Reloc::PIC_;
   JTI->Initialize(MF, IsPIC);
-  MCE.setModuleInfo(&getAnalysis<MachineModuleInfo>());
+  MMI = &getAnalysis<MachineModuleInfo>();
+  MCE.setModuleInfo(MMI);
 
   do {
     DEBUG(errs() << "JITTing function '"
@@ -436,8 +438,8 @@ void ARMCodeEmitter::emitConstPoolInstruction(const MachineInstr &MI) {
 void ARMCodeEmitter::emitMOVi2piecesInstruction(const MachineInstr &MI) {
   const MachineOperand &MO0 = MI.getOperand(0);
   const MachineOperand &MO1 = MI.getOperand(1);
-  assert(MO1.isImm() && ARM_AM::getSOImmVal(MO1.isImm()) != -1 &&
-                                            "Not a valid so_imm value!");
+  assert(MO1.isImm() && ARM_AM::isSOImmTwoPartVal(MO1.getImm()) &&
+                                                  "Not a valid so_imm value!");
   unsigned V1 = ARM_AM::getSOImmTwoPartFirst(MO1.getImm());
   unsigned V2 = ARM_AM::getSOImmTwoPartSecond(MO1.getImm());
 
@@ -563,7 +565,7 @@ void ARMCodeEmitter::emitPseudoInstruction(const MachineInstr &MI) {
   }
   case TargetOpcode::DBG_LABEL:
   case TargetOpcode::EH_LABEL:
-    MCE.emitLabel(MI.getOperand(0).getImm());
+    MCE.emitLabel(MI.getOperand(0).getMCSymbol());
     break;
   case TargetOpcode::IMPLICIT_DEF:
   case TargetOpcode::KILL:
@@ -925,19 +927,26 @@ static unsigned getAddrModeUPBits(unsigned Mode) {
   return Binary;
 }
 
-void ARMCodeEmitter::emitLoadStoreMultipleInstruction(
-                                                       const MachineInstr &MI) {
+void ARMCodeEmitter::emitLoadStoreMultipleInstruction(const MachineInstr &MI) {
+  const TargetInstrDesc &TID = MI.getDesc();
+  bool IsUpdating = (TID.TSFlags & ARMII::IndexModeMask) != 0;
+
   // Part of binary is determined by TableGn.
   unsigned Binary = getBinaryCodeForInstr(MI);
 
   // Set the conditional execution predicate
   Binary |= II->getPredicate(&MI) << ARMII::CondShift;
 
+  // Skip operand 0 of an instruction with base register update.
+  unsigned OpIdx = 0;
+  if (IsUpdating)
+    ++OpIdx;
+
   // Set base address operand
-  Binary |= getMachineOpValue(MI, 0) << ARMII::RegRnShift;
+  Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
 
   // Set addressing mode by modifying bits U(23) and P(24)
-  const MachineOperand &MO = MI.getOperand(1);
+  const MachineOperand &MO = MI.getOperand(OpIdx++);
   Binary |= getAddrModeUPBits(ARM_AM::getAM4SubMode(MO.getImm()));
 
   // Set bit W(21)
@@ -945,7 +954,7 @@ void ARMCodeEmitter::emitLoadStoreMultipleInstruction(
     Binary |= 0x1 << ARMII::W_BitShift;
 
   // Set registers
-  for (unsigned i = 5, e = MI.getNumOperands(); i != e; ++i) {
+  for (unsigned i = OpIdx+2, e = MI.getNumOperands(); i != e; ++i) {
     const MachineOperand &MO = MI.getOperand(i);
     if (!MO.isReg() || MO.isImplicit())
       break;
@@ -1322,17 +1331,25 @@ void ARMCodeEmitter::emitVFPLoadStoreInstruction(const MachineInstr &MI) {
 
 void ARMCodeEmitter::emitVFPLoadStoreMultipleInstruction(
                                                        const MachineInstr &MI) {
+  const TargetInstrDesc &TID = MI.getDesc();
+  bool IsUpdating = (TID.TSFlags & ARMII::IndexModeMask) != 0;
+
   // Part of binary is determined by TableGn.
   unsigned Binary = getBinaryCodeForInstr(MI);
 
   // Set the conditional execution predicate
   Binary |= II->getPredicate(&MI) << ARMII::CondShift;
 
+  // Skip operand 0 of an instruction with base register update.
+  unsigned OpIdx = 0;
+  if (IsUpdating)
+    ++OpIdx;
+
   // Set base address operand
-  Binary |= getMachineOpValue(MI, 0) << ARMII::RegRnShift;
+  Binary |= getMachineOpValue(MI, OpIdx++) << ARMII::RegRnShift;
 
   // Set addressing mode by modifying bits U(23) and P(24)
-  const MachineOperand &MO = MI.getOperand(1);
+  const MachineOperand &MO = MI.getOperand(OpIdx++);
   Binary |= getAddrModeUPBits(ARM_AM::getAM5SubMode(MO.getImm()));
 
   // Set bit W(21)
@@ -1340,11 +1357,11 @@ void ARMCodeEmitter::emitVFPLoadStoreMultipleInstruction(
     Binary |= 0x1 << ARMII::W_BitShift;
 
   // First register is encoded in Dd.
-  Binary |= encodeVFPRd(MI, 5);
+  Binary |= encodeVFPRd(MI, OpIdx+2);
 
   // Number of registers are encoded in offset field.
   unsigned NumRegs = 1;
-  for (unsigned i = 6, e = MI.getNumOperands(); i != e; ++i) {
+  for (unsigned i = OpIdx+3, e = MI.getNumOperands(); i != e; ++i) {
     const MachineOperand &MO = MI.getOperand(i);
     if (!MO.isReg() || MO.isImplicit())
       break;
diff --git a/lib/Target/ARM/ARMConstantIslandPass.cpp b/lib/Target/ARM/ARMConstantIslandPass.cpp
index 8fa3c04b6c4d..1c5bd42d63da 100644
--- a/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -1473,11 +1473,10 @@ bool ARMConstantIslands::UndoLRSpillRestore() {
   bool MadeChange = false;
   for (unsigned i = 0, e = PushPopMIs.size(); i != e; ++i) {
     MachineInstr *MI = PushPopMIs[i];
-    // First two operands are predicates, the third is a zero since there
-    // is no writeback.
+    // First two operands are predicates.
     if (MI->getOpcode() == ARM::tPOP_RET &&
-        MI->getOperand(3).getReg() == ARM::PC &&
-        MI->getNumExplicitOperands() == 4) {
+        MI->getOperand(2).getReg() == ARM::PC &&
+        MI->getNumExplicitOperands() == 3) {
       BuildMI(MI->getParent(), MI->getDebugLoc(), TII->get(ARM::tBX_RET));
       MI->eraseFromParent();
       MadeChange = true;
diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp
index 3dd0313f4b14..8f208432ab7e 100644
--- a/lib/Target/ARM/ARMISelLowering.cpp
+++ b/lib/Target/ARM/ARMISelLowering.cpp
@@ -130,7 +130,7 @@ void ARMTargetLowering::addQRTypeForNEON(EVT VT) {
 
 static TargetLoweringObjectFile *createTLOF(TargetMachine &TM) {
   if (TM.getSubtarget<ARMSubtarget>().isTargetDarwin())
-    return new ARMMachOTargetObjectFile();
+    return new TargetLoweringObjectFileMachO();
 
   return new ARMElfTargetObjectFile();
 }
@@ -426,12 +426,20 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
   setOperationAction(ISD::FPOW,      MVT::f64, Expand);
   setOperationAction(ISD::FPOW,      MVT::f32, Expand);
 
-  // int <-> fp are custom expanded into bit_convert + ARMISD ops.
-  if (!UseSoftFloat && Subtarget->hasVFP2() && !Subtarget->isThumb1Only()) {
-    setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
-    setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
-    setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
-    setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
+  // Various VFP goodness
+  if (!UseSoftFloat && !Subtarget->isThumb1Only()) {
+    // int <-> fp are custom expanded into bit_convert + ARMISD ops.
+    if (Subtarget->hasVFP2()) {
+      setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
+      setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
+      setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
+      setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
+    }
+    // Special handling for half-precision FP.
+    if (Subtarget->hasVFP3() && Subtarget->hasFP16()) {
+      setOperationAction(ISD::FP16_TO_FP32, MVT::f32, Custom);
+      setOperationAction(ISD::FP32_TO_FP16, MVT::i32, Custom);
+    }
   }
 
   // We have target-specific dag combine patterns for the following nodes:
@@ -491,6 +499,8 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::FTOUI:         return "ARMISD::FTOUI";
   case ARMISD::SITOF:         return "ARMISD::SITOF";
   case ARMISD::UITOF:         return "ARMISD::UITOF";
+  case ARMISD::F16_TO_F32:    return "ARMISD::F16_TO_F32";
+  case ARMISD::F32_TO_F16:    return "ARMISD::F32_TO_F16";
 
   case ARMISD::SRL_FLAG:      return "ARMISD::SRL_FLAG";
   case ARMISD::SRA_FLAG:      return "ARMISD::SRA_FLAG";
@@ -1972,8 +1982,21 @@ SDValue ARMTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) {
 
 static SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
   DebugLoc dl = Op.getDebugLoc();
-  unsigned Opc =
-    Op.getOpcode() == ISD::FP_TO_SINT ? ARMISD::FTOSI : ARMISD::FTOUI;
+  unsigned Opc;
+
+  switch (Op.getOpcode()) {
+  default:
+    assert(0 && "Invalid opcode!");
+  case ISD::FP32_TO_FP16:
+    Opc = ARMISD::F32_TO_F16;
+    break;
+  case ISD::FP_TO_SINT:
+    Opc = ARMISD::FTOSI;
+    break;
+  case ISD::FP_TO_UINT:
+    Opc = ARMISD::FTOUI;
+    break;
+  }
   Op = DAG.getNode(Opc, dl, MVT::f32, Op.getOperand(0));
   return DAG.getNode(ISD::BIT_CONVERT, dl, MVT::i32, Op);
 }
@@ -1981,8 +2004,21 @@ static SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) {
 static SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) {
   EVT VT = Op.getValueType();
   DebugLoc dl = Op.getDebugLoc();
-  unsigned Opc =
-    Op.getOpcode() == ISD::SINT_TO_FP ? ARMISD::SITOF : ARMISD::UITOF;
+  unsigned Opc;
+
+  switch (Op.getOpcode()) {
+  default:
+    assert(0 && "Invalid opcode!");
+  case ISD::FP16_TO_FP32:
+    Opc = ARMISD::F16_TO_F32;
+    break;
+  case ISD::SINT_TO_FP:
+    Opc = ARMISD::SITOF;
+    break;
+  case ISD::UINT_TO_FP:
+    Opc = ARMISD::UITOF;
+    break;
+  }
 
   Op = DAG.getNode(ISD::BIT_CONVERT, dl, MVT::f32, Op.getOperand(0));
   return DAG.getNode(Opc, dl, VT, Op);
@@ -3042,8 +3078,10 @@ SDValue ARMTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) {
   case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
   case ISD::VASTART:       return LowerVASTART(Op, DAG, VarArgsFrameIndex);
   case ISD::MEMBARRIER:    return LowerMEMBARRIER(Op, DAG, Subtarget);
+  case ISD::FP16_TO_FP32:
   case ISD::SINT_TO_FP:
   case ISD::UINT_TO_FP:    return LowerINT_TO_FP(Op, DAG);
+  case ISD::FP32_TO_FP16:
   case ISD::FP_TO_SINT:
   case ISD::FP_TO_UINT:    return LowerFP_TO_INT(Op, DAG);
   case ISD::FCOPYSIGN:     return LowerFCOPYSIGN(Op, DAG);
@@ -4411,6 +4449,9 @@ ARMTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
       break;
     }
   }
+  if (StringRef("{cc}").equals_lower(Constraint))
+    return std::make_pair(0U, ARM::CCRRegisterClass);
+
   return TargetLowering::getRegForInlineAsmConstraint(Constraint, VT);
 }
 
diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h
index f8f8adc70af8..d7b2ba3b5c72 100644
--- a/lib/Target/ARM/ARMISelLowering.h
+++ b/lib/Target/ARM/ARMISelLowering.h
@@ -59,6 +59,8 @@ namespace llvm {
       FTOUI,        // FP to uint within a FP register.
       SITOF,        // sint to FP within a FP register.
       UITOF,        // uint to FP within a FP register.
+      F16_TO_F32,   // Half FP to single FP within a FP register.
+      F32_TO_F16,   // Single FP to half FP within a FP register.
 
       SRL_FLAG,     // V,Flag = srl_flag X -> srl X, 1 + save carry out.
       SRA_FLAG,     // V,Flag = sra_flag X -> sra X, 1 + save carry out.
diff --git a/lib/Target/ARM/ARMInstrFormats.td b/lib/Target/ARM/ARMInstrFormats.td
index 76595fa2952a..258a96b9216e 100644
--- a/lib/Target/ARM/ARMInstrFormats.td
+++ b/lib/Target/ARM/ARMInstrFormats.td
@@ -112,6 +112,7 @@ class IndexMode<bits<2> val> {
 def IndexModeNone : IndexMode<0>;
 def IndexModePre  : IndexMode<1>;
 def IndexModePost : IndexMode<2>;
+def IndexModeUpd  : IndexMode<3>;
 
 // Instruction execution domain.
 class Domain<bits<2> val> {
@@ -144,6 +145,23 @@ def s_cc_out : OptionalDefOperand<OtherVT, (ops CCR), (ops (i32 CPSR))> {
   let PrintMethod = "printSBitModifierOperand";
 }
 
+// ARM special operands for disassembly only.
+//
+
+def cps_opt : Operand<i32> {
+  let PrintMethod = "printCPSOptionOperand";
+}
+
+def msr_mask : Operand<i32> {
+  let PrintMethod = "printMSRMaskOperand";
+}
+
+// A8.6.117, A8.6.118.  Different instructions are generated for #0 and #-0.
+// The neg_zero operand translates -0 to -1, -1 to -2, ..., etc.
+def neg_zero : Operand<i32> {
+  let PrintMethod = "printNegZeroOperand";
+}
+
 //===----------------------------------------------------------------------===//
 
 // ARM Instruction templates.
@@ -835,18 +853,18 @@ class AI3stdpo<dag oops, dag iops, Format f, InstrItinClass itin,
 
 
 // addrmode4 instructions
-class AXI4ld<dag oops, dag iops, Format f, InstrItinClass itin,
-             string asm, list<dag> pattern>
-  : XI<oops, iops, AddrMode4, Size4Bytes, IndexModeNone, f, itin,
-       asm, "", pattern> {
+class AXI4ld<dag oops, dag iops, IndexMode im, Format f, InstrItinClass itin,
+             string asm, string cstr, list<dag> pattern>
+  : XI<oops, iops, AddrMode4, Size4Bytes, im, f, itin,
+       asm, cstr, pattern> {
   let Inst{20}    = 1; // L bit
   let Inst{22}    = 0; // S bit
   let Inst{27-25} = 0b100;
 }
-class AXI4st<dag oops, dag iops, Format f, InstrItinClass itin,
-             string asm, list<dag> pattern>
-  : XI<oops, iops, AddrMode4, Size4Bytes, IndexModeNone, f, itin,
-       asm, "", pattern> {
+class AXI4st<dag oops, dag iops, IndexMode im, Format f, InstrItinClass itin,
+             string asm, string cstr, list<dag> pattern>
+  : XI<oops, iops, AddrMode4, Size4Bytes, im, f, itin,
+       asm, cstr, pattern> {
   let Inst{20}    = 0; // L bit
   let Inst{22}    = 0; // S bit
   let Inst{27-25} = 0b100;
@@ -980,9 +998,9 @@ class T1JTI<dag oops, dag iops, InstrItinClass itin,
 
 // Two-address instructions
 class T1It<dag oops, dag iops, InstrItinClass itin,
-           string asm, list<dag> pattern>
+           string asm, string cstr, list<dag> pattern>
   : Thumb1I<oops, iops, AddrModeNone, Size2Bytes, itin, 
-            asm, "$lhs = $dst", pattern>;
+            asm, cstr, pattern>;
 
 // Thumb1 instruction that can either be predicated or set CPSR.
 class Thumb1sI<dag oops, dag iops, AddrMode am, SizeFlagVal sz,
@@ -1179,6 +1197,10 @@ class T2Ix2<dag oops, dag iops, InstrItinClass itin,
           string opc, string asm, list<dag> pattern>
   : Thumb2I<oops, iops, AddrModeNone, Size8Bytes, itin, opc, asm, "", pattern>;
 
+// Two-address instructions
+class T2XIt<dag oops, dag iops, InstrItinClass itin,
+            string asm, string cstr, list<dag> pattern>
+  : Thumb2XI<oops, iops, AddrModeNone, Size4Bytes, itin, asm, cstr, pattern>;
 
 // T2Iidxldst - Thumb2 indexed load / store instructions.
 class T2Iidxldst<bit signed, bits<2> opcod, bit load, bit pre,
@@ -1293,10 +1315,10 @@ class ASI5<bits<4> opcod1, bits<2> opcod2, dag oops, dag iops,
 }
 
 // Load / store multiple
-class AXDI5<dag oops, dag iops, InstrItinClass itin,
-            string asm, list<dag> pattern>
-  : VFPXI<oops, iops, AddrMode5, Size4Bytes, IndexModeNone,
-       VFPLdStMulFrm, itin, asm, "", pattern> {
+class AXDI5<dag oops, dag iops, IndexMode im, InstrItinClass itin,
+            string asm, string cstr, list<dag> pattern>
+  : VFPXI<oops, iops, AddrMode5, Size4Bytes, im,
+       VFPLdStMulFrm, itin, asm, cstr, pattern> {
   // TODO: Mark the instructions with the appropriate subtarget info.
   let Inst{27-25} = 0b110;
   let Inst{11-8}  = 0b1011;
@@ -1305,10 +1327,10 @@ class AXDI5<dag oops, dag iops, InstrItinClass itin,
   let Dom = VFPNeonDomain.Value;
 }
 
-class AXSI5<dag oops, dag iops, InstrItinClass itin,
-            string asm, list<dag> pattern>
-  : VFPXI<oops, iops, AddrMode5, Size4Bytes, IndexModeNone,
-       VFPLdStMulFrm, itin, asm, "", pattern> {
+class AXSI5<dag oops, dag iops, IndexMode im, InstrItinClass itin,
+            string asm, string cstr, list<dag> pattern>
+  : VFPXI<oops, iops, AddrMode5, Size4Bytes, im,
+       VFPLdStMulFrm, itin, asm, cstr, pattern> {
   // TODO: Mark the instructions with the appropriate subtarget info.
   let Inst{27-25} = 0b110;
   let Inst{11-8}  = 0b1010;
diff --git a/lib/Target/ARM/ARMInstrInfo.td b/lib/Target/ARM/ARMInstrInfo.td
index af82a5ff7750..3fc37dae4e4e 100644
--- a/lib/Target/ARM/ARMInstrInfo.td
+++ b/lib/Target/ARM/ARMInstrInfo.td
@@ -378,7 +378,7 @@ def am3offset : Operand<i32>,
 def addrmode4 : Operand<i32>,
                 ComplexPattern<i32, 2, "SelectAddrMode4", []> {
   let PrintMethod = "printAddrMode4Operand";
-  let MIOperandInfo = (ops GPR, i32imm);
+  let MIOperandInfo = (ops GPR:$addr, i32imm);
 }
 
 // addrmode5 := reg +/- imm8*4
@@ -386,7 +386,7 @@ def addrmode4 : Operand<i32>,
 def addrmode5 : Operand<i32>,
                 ComplexPattern<i32, 2, "SelectAddrMode5", []> {
   let PrintMethod = "printAddrMode5Operand";
-  let MIOperandInfo = (ops GPR, i32imm);
+  let MIOperandInfo = (ops GPR:$base, i32imm);
 }
 
 // addrmode6 := reg with optional writeback
@@ -710,7 +710,7 @@ def BKPT : AI<(outs), (ins i32imm:$val), MiscFrm, NoItinerary, "bkpt", "\t$val",
 // opt{5} = changemode from Inst{17}
 // opt{8-6} = AIF from Inst{8-6}
 // opt{10-9} = imod from Inst{19-18} with 0b10 as enable and 0b11 as disable
-def CPS : AXI<(outs),(ins i32imm:$opt), MiscFrm, NoItinerary, "cps${opt:cps}",
+def CPS : AXI<(outs), (ins cps_opt:$opt), MiscFrm, NoItinerary, "cps$opt",
               [/* For disassembly only; pattern left blank */]>,
           Requires<[IsARM]> {
   let Inst{31-28} = 0b1111;
@@ -721,9 +721,12 @@ def CPS : AXI<(outs),(ins i32imm:$opt), MiscFrm, NoItinerary, "cps${opt:cps}",
 
 // Preload signals the memory system of possible future data/instruction access.
 // These are for disassembly only.
+//
+// A8.6.117, A8.6.118.  Different instructions are generated for #0 and #-0.
+// The neg_zero operand translates -0 to -1, -1 to -2, ..., etc.
 multiclass APreLoad<bit data, bit read, string opc> {
 
-  def i : AXI<(outs), (ins GPR:$base, i32imm:$imm), MiscFrm, NoItinerary,
+  def i : AXI<(outs), (ins GPR:$base, neg_zero:$imm), MiscFrm, NoItinerary,
                !strconcat(opc, "\t[$base, $imm]"), []> {
     let Inst{31-26} = 0b111101;
     let Inst{25} = 0; // 0 for immediate form
@@ -903,10 +906,11 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
 // FIXME: Should pc be an implicit operand like PICADD, etc?
 let isReturn = 1, isTerminator = 1, isBarrier = 1, mayLoad = 1,
     hasExtraDefRegAllocReq = 1 in
-  def LDM_RET : AXI4ld<(outs),
-                    (ins addrmode4:$addr, pred:$p, reglist:$wb, variable_ops),
-                    LdStMulFrm, IIC_Br, "ldm${addr:submode}${p}\t$addr, $wb",
-                    []>;
+  def LDM_RET : AXI4ld<(outs GPR:$wb), (ins addrmode4:$addr, pred:$p,
+                                        reglist:$dsts, variable_ops),
+                       IndexModeUpd, LdStMulFrm, IIC_Br,
+                       "ldm${addr:submode}${p}\t$addr, $dsts",
+                       "$addr.addr = $wb", []>;
 
 // On non-Darwin platforms R9 is callee-saved.
 let isCall = 1,
@@ -1341,17 +1345,31 @@ def STRHT: AI3sthpo<(outs GPR:$base_wb),
 //  Load / store multiple Instructions.
 //
 
-let mayLoad = 1, hasExtraDefRegAllocReq = 1 in
-def LDM : AXI4ld<(outs),
-               (ins addrmode4:$addr, pred:$p, reglist:$wb, variable_ops),
-               LdStMulFrm, IIC_iLoadm, "ldm${addr:submode}${p}\t$addr, $wb",
-               []>;
+let mayLoad = 1, hasExtraDefRegAllocReq = 1 in {
+def LDM : AXI4ld<(outs), (ins addrmode4:$addr, pred:$p,
+                          reglist:$dsts, variable_ops),
+                 IndexModeNone, LdStMulFrm, IIC_iLoadm,
+                 "ldm${addr:submode}${p}\t$addr, $dsts", "", []>;
 
-let mayStore = 1, hasExtraSrcRegAllocReq = 1 in
-def STM : AXI4st<(outs),
-               (ins addrmode4:$addr, pred:$p, reglist:$wb, variable_ops),
-               LdStMulFrm, IIC_iStorem, "stm${addr:submode}${p}\t$addr, $wb",
-               []>;
+def LDM_UPD : AXI4ld<(outs GPR:$wb), (ins addrmode4:$addr, pred:$p,
+                                      reglist:$dsts, variable_ops),
+                     IndexModeUpd, LdStMulFrm, IIC_iLoadm,
+                     "ldm${addr:submode}${p}\t$addr, $dsts",
+                     "$addr.addr = $wb", []>;
+} // mayLoad, hasExtraDefRegAllocReq
+
+let mayStore = 1, hasExtraSrcRegAllocReq = 1 in {
+def STM : AXI4st<(outs), (ins addrmode4:$addr, pred:$p,
+                          reglist:$srcs, variable_ops),
+                 IndexModeNone, LdStMulFrm, IIC_iStorem,
+                 "stm${addr:submode}${p}\t$addr, $srcs", "", []>;
+
+def STM_UPD : AXI4st<(outs GPR:$wb), (ins addrmode4:$addr, pred:$p,
+                                      reglist:$srcs, variable_ops),
+                     IndexModeUpd, LdStMulFrm, IIC_iStorem,
+                     "stm${addr:submode}${p}\t$addr, $srcs",
+                     "$addr.addr = $wb", []>;
+} // mayStore, hasExtraSrcRegAllocReq
 
 //===----------------------------------------------------------------------===//
 //  Move Instructions.
@@ -2844,29 +2862,29 @@ def MRSsys : ABI<0b0001,(outs GPR:$dst),(ins), NoItinerary,"mrs","\t$dst, spsr",
   let Inst{7-4} = 0b0000;
 }
 
-def MSR : ABI<0b0001, (outs), (ins GPR:$src, i32imm:$mask), NoItinerary, "msr",
-              "\tcpsr${mask:msr}, $src",
+def MSR : ABI<0b0001, (outs), (ins GPR:$src, msr_mask:$mask), NoItinerary,
+              "msr", "\tcpsr$mask, $src",
               [/* For disassembly only; pattern left blank */]> {
   let Inst{23-20} = 0b0010;
   let Inst{7-4} = 0b0000;
 }
 
-def MSRi : ABI<0b0011, (outs), (ins so_imm:$a, i32imm:$mask), NoItinerary,"msr",
-              "\tcpsr${mask:msr}, $a",
+def MSRi : ABI<0b0011, (outs), (ins so_imm:$a, msr_mask:$mask), NoItinerary,
+              "msr", "\tcpsr$mask, $a",
               [/* For disassembly only; pattern left blank */]> {
   let Inst{23-20} = 0b0010;
   let Inst{7-4} = 0b0000;
 }
 
-def MSRsys : ABI<0b0001, (outs), (ins GPR:$src, i32imm:$mask),NoItinerary,"msr",
-              "\tspsr${mask:msr}, $src",
+def MSRsys : ABI<0b0001, (outs), (ins GPR:$src, msr_mask:$mask), NoItinerary,
+              "msr", "\tspsr$mask, $src",
               [/* For disassembly only; pattern left blank */]> {
   let Inst{23-20} = 0b0110;
   let Inst{7-4} = 0b0000;
 }
 
-def MSRsysi : ABI<0b0011,(outs),(ins so_imm:$a, i32imm:$mask),NoItinerary,"msr",
-              "\tspsr${mask:msr}, $a",
+def MSRsysi : ABI<0b0011, (outs), (ins so_imm:$a, msr_mask:$mask), NoItinerary,
+              "msr", "\tspsr$mask, $a",
               [/* For disassembly only; pattern left blank */]> {
   let Inst{23-20} = 0b0110;
   let Inst{7-4} = 0b0000;
diff --git a/lib/Target/ARM/ARMInstrNEON.td b/lib/Target/ARM/ARMInstrNEON.td
index 7c44a2fe49e3..8fee6fa9527a 100644
--- a/lib/Target/ARM/ARMInstrNEON.td
+++ b/lib/Target/ARM/ARMInstrNEON.td
@@ -98,16 +98,6 @@ def NEONfmin      : SDNode<"ARMISD::FMIN", SDTARMFMAX>;
 // NEON operand definitions
 //===----------------------------------------------------------------------===//
 
-// addrmode_neonldstm := reg
-//
-/* TODO: Take advantage of vldm.
-def addrmode_neonldstm : Operand<i32>,
-                ComplexPattern<i32, 2, "SelectAddrModeNeonLdStM", []> {
-  let PrintMethod = "printAddrNeonLdStMOperand";
-  let MIOperandInfo = (ops GPR, i32imm);
-}
-*/
-
 def h8imm  : Operand<i8> {
   let PrintMethod = "printHex8ImmOperand";
 }
@@ -125,26 +115,6 @@ def h64imm : Operand<i64> {
 // NEON load / store instructions
 //===----------------------------------------------------------------------===//
 
-/* TODO: Take advantage of vldm.
-let mayLoad = 1, hasExtraDefRegAllocReq = 1 in {
-def VLDMD : NI<(outs),
-               (ins addrmode_neonldstm:$addr, reglist:$dst1, variable_ops),
-               IIC_fpLoadm, "vldm", "${addr:submode} ${addr:base}, $dst1", []> {
-  let Inst{27-25} = 0b110;
-  let Inst{20}    = 1;
-  let Inst{11-9}  = 0b101;
-}
-
-def VLDMS : NI<(outs),
-               (ins addrmode_neonldstm:$addr, reglist:$dst1, variable_ops),
-               IIC_fpLoadm, "vldm", "${addr:submode} ${addr:base}, $dst1", []> {
-  let Inst{27-25} = 0b110;
-  let Inst{20}    = 1;
-  let Inst{11-9}  = 0b101;
-}
-}
-*/
-
 // Use vldmia to load a Q register as a D register pair.
 def VLDRQ : NI4<(outs QPR:$dst), (ins addrmode4:$addr), IIC_fpLoadm,
                 "vldmia", "$addr, ${dst:dregpair}",
@@ -2738,7 +2708,7 @@ def : Pat<(v2f64 (insertelt QPR:$src1, DPR:$src2, imm:$src3)),
 
 def : Pat<(v2f32 (scalar_to_vector SPR:$src)),
           (INSERT_SUBREG (v2f32 (IMPLICIT_DEF)), SPR:$src, arm_ssubreg_0)>;
-def : Pat<(v2f64 (scalar_to_vector DPR:$src)),
+def : Pat<(v2f64 (scalar_to_vector (f64 DPR:$src))),
           (INSERT_SUBREG (v2f64 (IMPLICIT_DEF)), DPR:$src, arm_dsubreg_0)>;
 def : Pat<(v4f32 (scalar_to_vector SPR:$src)),
           (INSERT_SUBREG (v4f32 (IMPLICIT_DEF)), SPR:$src, arm_ssubreg_0)>;
diff --git a/lib/Target/ARM/ARMInstrThumb.td b/lib/Target/ARM/ARMInstrThumb.td
index 786dd65b8742..37c9fc5f734e 100644
--- a/lib/Target/ARM/ARMInstrThumb.td
+++ b/lib/Target/ARM/ARMInstrThumb.td
@@ -201,7 +201,7 @@ def tBKPT : T1I<(outs), (ins i32imm:$val), NoItinerary, "bkpt\t$val",
 //
 // The opt{4-0} and opt{5} sub-fields are to accommodate 32-bit Thumb and ARM
 // CPS which has more options.
-def tCPS : T1I<(outs), (ins i32imm:$opt), NoItinerary, "cps${opt:cps}",
+def tCPS : T1I<(outs), (ins cps_opt:$opt), NoItinerary, "cps$opt",
               [/* For disassembly only; pattern left blank */]>,
            T1Misc<0b0110011>;
 
@@ -290,8 +290,8 @@ let isBranch = 1, isTerminator = 1, isBarrier = 1, isIndirectBranch = 1 in {
 // FIXME: remove when we have a way to marking a MI with these properties.
 let isReturn = 1, isTerminator = 1, isBarrier = 1, mayLoad = 1,
     hasExtraDefRegAllocReq = 1 in
-def tPOP_RET : T1I<(outs), (ins pred:$p, reglist:$wb, variable_ops), IIC_Br,
-                   "pop${p}\t$wb", []>,
+def tPOP_RET : T1I<(outs), (ins pred:$p, reglist:$dsts, variable_ops), IIC_Br,
+                   "pop${p}\t$dsts", []>,
                T1Misc<{1,1,0,?,?,?,?}>;
 
 let isCall = 1,
@@ -539,28 +539,37 @@ def tSpill : T1pIs<(outs), (ins tGPR:$src, t_addrmode_sp:$addr), IIC_iStorei,
 //
 
 // These requires base address to be written back or one of the loaded regs.
-let mayLoad = 1, hasExtraDefRegAllocReq = 1 in
+let mayLoad = 1, hasExtraDefRegAllocReq = 1 in {
 def tLDM : T1I<(outs),
-               (ins addrmode4:$addr, pred:$p, reglist:$wb, variable_ops),
+               (ins addrmode4:$addr, pred:$p, reglist:$dsts, variable_ops),
                IIC_iLoadm,
-               "ldm${addr:submode}${p}\t$addr, $wb", []>,
+               "ldm${addr:submode}${p}\t$addr, $dsts", []>,
            T1Encoding<{1,1,0,0,1,?}>; // A6.2 & A8.6.53
 
+def tLDM_UPD : T1It<(outs tGPR:$wb),
+                    (ins addrmode4:$addr, pred:$p, reglist:$dsts, variable_ops),
+                    IIC_iLoadm,
+                    "ldm${addr:submode}${p}\t$addr, $dsts",
+                    "$addr.addr = $wb", []>,
+               T1Encoding<{1,1,0,0,1,?}>; // A6.2 & A8.6.53
+} // mayLoad, hasExtraDefRegAllocReq
+
 let mayStore = 1, hasExtraSrcRegAllocReq = 1 in
-def tSTM : T1I<(outs),
-               (ins addrmode4:$addr, pred:$p, reglist:$wb, variable_ops),
-               IIC_iStorem,
-               "stm${addr:submode}${p}\t$addr, $wb", []>,
+def tSTM_UPD : T1It<(outs tGPR:$wb),
+                    (ins addrmode4:$addr, pred:$p, reglist:$srcs, variable_ops),
+                    IIC_iStorem,
+                    "stm${addr:submode}${p}\t$addr, $srcs",
+                    "$addr.addr = $wb", []>,
            T1Encoding<{1,1,0,0,0,?}>; // A6.2 & A8.6.189
 
 let mayLoad = 1, Uses = [SP], Defs = [SP], hasExtraDefRegAllocReq = 1 in
-def tPOP : T1I<(outs), (ins pred:$p, reglist:$wb, variable_ops), IIC_Br,
-               "pop${p}\t$wb", []>,
+def tPOP : T1I<(outs), (ins pred:$p, reglist:$dsts, variable_ops), IIC_Br,
+               "pop${p}\t$dsts", []>,
            T1Misc<{1,1,0,?,?,?,?}>;
 
 let mayStore = 1, Uses = [SP], Defs = [SP], hasExtraSrcRegAllocReq = 1 in
-def tPUSH : T1I<(outs), (ins pred:$p, reglist:$wb, variable_ops), IIC_Br,
-                "push${p}\t$wb", []>,
+def tPUSH : T1I<(outs), (ins pred:$p, reglist:$srcs, variable_ops), IIC_Br,
+                "push${p}\t$srcs", []>,
             T1Misc<{0,1,0,?,?,?,?}>;
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/ARM/ARMInstrThumb2.td b/lib/Target/ARM/ARMInstrThumb2.td
index 2fc7d2f76960..ab9e926099c2 100644
--- a/lib/Target/ARM/ARMInstrThumb2.td
+++ b/lib/Target/ARM/ARMInstrThumb2.td
@@ -150,6 +150,10 @@ def t2addrmode_imm8s4 : Operand<i32>,
   let MIOperandInfo = (ops GPR:$base, i32imm:$offsimm);
 }
 
+def t2am_imm8s4_offset : Operand<i32> {
+  let PrintMethod = "printT2AddrModeImm8s4OffsetOperand";
+}
+
 // t2addrmode_so_reg  := reg + (reg << imm2)
 def t2addrmode_so_reg : Operand<i32>,
                         ComplexPattern<i32, 3, "SelectT2AddrModeSoReg", []> {
@@ -903,7 +907,7 @@ let mayLoad = 1, hasExtraDefRegAllocReq = 1 in {
 def t2LDRDi8  : T2Ii8s4<1, 0, 1, (outs GPR:$dst1, GPR:$dst2),
                         (ins t2addrmode_imm8s4:$addr),
                         IIC_iLoadi, "ldrd", "\t$dst1, $addr", []>;
-def t2LDRDpci : T2Ii8s4<?, ?, 1, (outs GPR:$dst1, GPR:$dst2),
+def t2LDRDpci : T2Ii8s4<1, 0, 1, (outs GPR:$dst1, GPR:$dst2),
                         (ins i32imm:$addr), IIC_iLoadi,
                        "ldrd", "\t$dst1, $addr", []> {
   let Inst{19-16} = 0b1111; // Rn
@@ -1105,14 +1109,34 @@ def t2STRT   : T2IstT<0b10, "strt">;
 def t2STRBT  : T2IstT<0b00, "strbt">;
 def t2STRHT  : T2IstT<0b01, "strht">;
 
-// FIXME: ldrd / strd pre / post variants
+// ldrd / strd pre / post variants
+// For disassembly only.
+
+def t2LDRD_PRE  : T2Ii8s4<1, 1, 1, (outs GPR:$dst1, GPR:$dst2),
+                 (ins GPR:$base, t2am_imm8s4_offset:$imm), NoItinerary,
+                 "ldrd", "\t$dst1, $dst2, [$base, $imm]!", []>;
+
+def t2LDRD_POST : T2Ii8s4<0, 1, 1, (outs GPR:$dst1, GPR:$dst2),
+                 (ins GPR:$base, t2am_imm8s4_offset:$imm), NoItinerary,
+                 "ldrd", "\t$dst1, $dst2, [$base], $imm", []>;
+
+def t2STRD_PRE  : T2Ii8s4<1, 1, 0, (outs),
+                 (ins GPR:$src1, GPR:$src2, GPR:$base, t2am_imm8s4_offset:$imm),
+                 NoItinerary, "strd", "\t$src1, $src2, [$base, $imm]!", []>;
+
+def t2STRD_POST : T2Ii8s4<0, 1, 0, (outs),
+                 (ins GPR:$src1, GPR:$src2, GPR:$base, t2am_imm8s4_offset:$imm),
+                 NoItinerary, "strd", "\t$src1, $src2, [$base], $imm", []>;
 
 // T2Ipl (Preload Data/Instruction) signals the memory system of possible future
 // data/instruction access.  These are for disassembly only.
+//
+// A8.6.117, A8.6.118.  Different instructions are generated for #0 and #-0.
+// The neg_zero operand translates -0 to -1, -1 to -2, ..., etc.
 multiclass T2Ipl<bit instr, bit write, string opc> {
 
-  def i12 : T2I<(outs), (ins t2addrmode_imm12:$addr), IIC_iLoadi, opc,
-                "\t$addr", []> {
+  def i12 : T2I<(outs), (ins GPR:$base, i32imm:$imm), IIC_iLoadi, opc,
+                "\t[$base, $imm]", []> {
     let Inst{31-25} = 0b1111100;
     let Inst{24} = instr;
     let Inst{23} = 1; // U = 1
@@ -1122,8 +1146,8 @@ multiclass T2Ipl<bit instr, bit write, string opc> {
     let Inst{15-12} = 0b1111;
   }
 
-  def i8 : T2I<(outs), (ins t2addrmode_imm8:$addr), IIC_iLoadi, opc,
-                "\t$addr", []> {
+  def i8 : T2I<(outs), (ins GPR:$base, neg_zero:$imm), IIC_iLoadi, opc,
+                "\t[$base, $imm]", []> {
     let Inst{31-25} = 0b1111100;
     let Inst{24} = instr;
     let Inst{23} = 0; // U = 0
@@ -1134,9 +1158,8 @@ multiclass T2Ipl<bit instr, bit write, string opc> {
     let Inst{11-8} = 0b1100;
   }
 
-  // A8.6.118 #0 and #-0 differs.  Translates -0 to -1, -1 to -2, ..., etc.
-  def pci : T2I<(outs), (ins GPR:$base, i32imm:$imm), IIC_iLoadi, opc,
-                "\t[pc, ${imm:negzero}]", []> {
+  def pci : T2I<(outs), (ins GPR:$base, neg_zero:$imm), IIC_iLoadi, opc,
+                "\t[pc, $imm]", []> {
     let Inst{31-25} = 0b1111100;
     let Inst{24} = instr;
     let Inst{23} = ?; // add = (U == 1)
@@ -1181,29 +1204,56 @@ defm t2PLI  : T2Ipl<1, 0, "pli">;
 //  Load / store multiple Instructions.
 //
 
-let mayLoad = 1, hasExtraDefRegAllocReq = 1 in
-def t2LDM : T2XI<(outs),
-                 (ins addrmode4:$addr, pred:$p, reglist:$wb, variable_ops),
-             IIC_iLoadm, "ldm${addr:submode}${p}${addr:wide}\t$addr, $wb", []> {
+let mayLoad = 1, hasExtraDefRegAllocReq = 1 in {
+def t2LDM : T2XI<(outs), (ins addrmode4:$addr, pred:$p,
+                          reglist:$dsts, variable_ops), IIC_iLoadm,
+                 "ldm${addr:submode}${p}${addr:wide}\t$addr, $dsts", []> {
+  let Inst{31-27} = 0b11101;
+  let Inst{26-25} = 0b00;
+  let Inst{24-23} = {?, ?}; // IA: '01', DB: '10'
+  let Inst{22} = 0;
+  let Inst{21} = 0; // The W bit.
+  let Inst{20} = 1; // Load
+}
+
+def t2LDM_UPD : T2XIt<(outs GPR:$wb), (ins addrmode4:$addr, pred:$p,
+                                       reglist:$dsts, variable_ops), IIC_iLoadm,
+                      "ldm${addr:submode}${p}${addr:wide}\t$addr, $dsts",
+                      "$addr.addr = $wb", []> {
   let Inst{31-27} = 0b11101;
   let Inst{26-25} = 0b00;
   let Inst{24-23} = {?, ?}; // IA: '01', DB: '10'
   let Inst{22} = 0;
-  let Inst{21} = ?; // The W bit.
+  let Inst{21} = 1; // The W bit.
   let Inst{20} = 1; // Load
 }
+} // mayLoad, hasExtraDefRegAllocReq
+
+let mayStore = 1, hasExtraSrcRegAllocReq = 1 in {
+def t2STM : T2XI<(outs), (ins addrmode4:$addr, pred:$p,
+                          reglist:$srcs, variable_ops), IIC_iStorem,
+                 "stm${addr:submode}${p}${addr:wide}\t$addr, $srcs", []> {
+  let Inst{31-27} = 0b11101;
+  let Inst{26-25} = 0b00;
+  let Inst{24-23} = {?, ?}; // IA: '01', DB: '10'
+  let Inst{22} = 0;
+  let Inst{21} = 0; // The W bit.
+  let Inst{20} = 0; // Store
+}
 
-let mayStore = 1, hasExtraSrcRegAllocReq = 1 in
-def t2STM : T2XI<(outs),
-                 (ins addrmode4:$addr, pred:$p, reglist:$wb, variable_ops),
-            IIC_iStorem, "stm${addr:submode}${p}${addr:wide}\t$addr, $wb", []> {
+def t2STM_UPD : T2XIt<(outs GPR:$wb), (ins addrmode4:$addr, pred:$p,
+                                       reglist:$srcs, variable_ops),
+                      IIC_iStorem,
+                      "stm${addr:submode}${p}${addr:wide}\t$addr, $srcs",
+                      "$addr.addr = $wb", []> {
   let Inst{31-27} = 0b11101;
   let Inst{26-25} = 0b00;
   let Inst{24-23} = {?, ?}; // IA: '01', DB: '10'
   let Inst{22} = 0;
-  let Inst{21} = ?; // The W bit.
+  let Inst{21} = 1; // The W bit.
   let Inst{20} = 0; // Store
 }
+} // mayStore, hasExtraSrcRegAllocReq
 
 //===----------------------------------------------------------------------===//
 //  Move Instructions.
@@ -2136,7 +2186,7 @@ def t2MOVCCror : T2I_movcc_sh<0b11, (outs GPR:$dst),
 // memory barriers protect the atomic sequences
 let hasSideEffects = 1 in {
 def t2Int_MemBarrierV7 : AInoP<(outs), (ins),
-                        Pseudo, NoItinerary,
+                        ThumbFrm, NoItinerary,
                         "dmb", "",
                         [(ARMMemBarrierV7)]>,
                         Requires<[IsThumb2]> {
@@ -2146,7 +2196,7 @@ def t2Int_MemBarrierV7 : AInoP<(outs), (ins),
 }
 
 def t2Int_SyncBarrierV7 : AInoP<(outs), (ins),
-                        Pseudo, NoItinerary,
+                        ThumbFrm, NoItinerary,
                         "dsb", "",
                         [(ARMSyncBarrierV7)]>,
                         Requires<[IsThumb2]> {
@@ -2351,15 +2401,15 @@ let Defs =
 // FIXME: Should pc be an implicit operand like PICADD, etc?
 let isReturn = 1, isTerminator = 1, isBarrier = 1, mayLoad = 1,
     hasExtraDefRegAllocReq = 1 in
-  def t2LDM_RET : T2XI<(outs),
-                    (ins addrmode4:$addr, pred:$p, reglist:$wb, variable_ops),
-                    IIC_Br, "ldm${addr:submode}${p}${addr:wide}\t$addr, $wb",
-                    []> {
+  def t2LDM_RET : T2XIt<(outs GPR:$wb), (ins addrmode4:$addr, pred:$p,
+                                         reglist:$dsts, variable_ops), IIC_Br,
+                        "ldm${addr:submode}${p}${addr:wide}\t$addr, $dsts",
+                        "$addr.addr = $wb", []> {
   let Inst{31-27} = 0b11101;
   let Inst{26-25} = 0b00;
   let Inst{24-23} = {?, ?}; // IA: '01', DB: '10'
   let Inst{22} = 0;
-  let Inst{21} = ?; // The W bit.
+  let Inst{21} = 1; // The W bit.
   let Inst{20} = 1; // Load
 }
 
@@ -2469,7 +2519,7 @@ def t2BXJ : T2I<(outs), (ins GPR:$func), NoItinerary, "bxj", "\t$func",
 // opt{5} = changemode from Inst{17}
 // opt{8-6} = AIF from Inst{8-6}
 // opt{10-9} = imod from Inst{19-18} with 0b10 as enable and 0b11 as disable
-def t2CPS : T2XI<(outs),(ins i32imm:$opt), NoItinerary, "cps${opt:cps}",
+def t2CPS : T2XI<(outs),(ins cps_opt:$opt), NoItinerary, "cps$opt",
                  [/* For disassembly only; pattern left blank */]> {
   let Inst{31-27} = 0b11110;
   let Inst{26} = 0;
@@ -2638,8 +2688,8 @@ def t2MRSsys : T2I<(outs GPR:$dst), (ins), NoItinerary, "mrs", "\t$dst, spsr",
 }
 
 // Rn = Inst{19-16}
-def t2MSR : T2I<(outs), (ins GPR:$src, i32imm:$mask), NoItinerary, "msr",
-                "\tcpsr${mask:msr}, $src",
+def t2MSR : T2I<(outs), (ins GPR:$src, msr_mask:$mask), NoItinerary, "msr",
+                "\tcpsr$mask, $src",
                 [/* For disassembly only; pattern left blank */]> {
   let Inst{31-27} = 0b11110;
   let Inst{26} = 0;
@@ -2650,8 +2700,8 @@ def t2MSR : T2I<(outs), (ins GPR:$src, i32imm:$mask), NoItinerary, "msr",
 }
 
 // Rn = Inst{19-16}
-def t2MSRsys : T2I<(outs), (ins GPR:$src, i32imm:$mask), NoItinerary, "msr",
-                   "\tspsr${mask:msr}, $src",
+def t2MSRsys : T2I<(outs), (ins GPR:$src, msr_mask:$mask), NoItinerary, "msr",
+                   "\tspsr$mask, $src",
                    [/* For disassembly only; pattern left blank */]> {
   let Inst{31-27} = 0b11110;
   let Inst{26} = 0;
diff --git a/lib/Target/ARM/ARMInstrVFP.td b/lib/Target/ARM/ARMInstrVFP.td
index cad24c4834d8..4d1d48a11341 100644
--- a/lib/Target/ARM/ARMInstrVFP.td
+++ b/lib/Target/ARM/ARMInstrVFP.td
@@ -25,6 +25,8 @@ def arm_ftoui  : SDNode<"ARMISD::FTOUI",  SDT_FTOI>;
 def arm_ftosi  : SDNode<"ARMISD::FTOSI",  SDT_FTOI>;
 def arm_sitof  : SDNode<"ARMISD::SITOF",  SDT_ITOF>;
 def arm_uitof  : SDNode<"ARMISD::UITOF",  SDT_ITOF>;
+def arm_f16tof32 : SDNode<"ARMISD::F16_TO_F32", SDT_ITOF>;
+def arm_f32tof16 : SDNode<"ARMISD::F32_TO_F16", SDT_FTOI>;
 def arm_fmstat : SDNode<"ARMISD::FMSTAT", SDTNone, [SDNPInFlag,SDNPOutFlag]>;
 def arm_cmpfp  : SDNode<"ARMISD::CMPFP",  SDT_ARMCmp, [SDNPOutFlag]>;
 def arm_cmpfp0 : SDNode<"ARMISD::CMPFPw0",SDT_CMPFP0, [SDNPOutFlag]>;
@@ -77,33 +79,61 @@ def VSTRS  : ASI5<0b1101, 0b00, (outs), (ins SPR:$src, addrmode5:$addr),
 //
 
 let mayLoad = 1, hasExtraDefRegAllocReq = 1 in {
-def VLDMD : AXDI5<(outs), (ins addrmode5:$addr, pred:$p, reglist:$wb,
-                           variable_ops), IIC_fpLoadm,
-                  "vldm${addr:submode}${p}\t${addr:base}, $wb",
-                  []> {
+def VLDMD : AXDI5<(outs), (ins addrmode5:$addr, pred:$p, reglist:$dsts,
+                           variable_ops), IndexModeNone, IIC_fpLoadm,
+                  "vldm${addr:submode}${p}\t${addr:base}, $dsts", "", []> {
   let Inst{20} = 1;
 }
 
-def VLDMS : AXSI5<(outs), (ins addrmode5:$addr, pred:$p, reglist:$wb,
-                           variable_ops), IIC_fpLoadm, 
-                  "vldm${addr:submode}${p}\t${addr:base}, $wb",
-                  []> {
+def VLDMS : AXSI5<(outs), (ins addrmode5:$addr, pred:$p, reglist:$dsts,
+                           variable_ops), IndexModeNone, IIC_fpLoadm,
+                  "vldm${addr:submode}${p}\t${addr:base}, $dsts", "", []> {
+  let Inst{20} = 1;
+}
+
+def VLDMD_UPD : AXDI5<(outs GPR:$wb), (ins addrmode5:$addr, pred:$p,
+                                       reglist:$dsts, variable_ops),
+                      IndexModeUpd, IIC_fpLoadm,
+                      "vldm${addr:submode}${p}\t${addr:base}, $dsts",
+                      "$addr.base = $wb", []> {
+  let Inst{20} = 1;
+}
+
+def VLDMS_UPD : AXSI5<(outs GPR:$wb), (ins addrmode5:$addr, pred:$p,
+                                       reglist:$dsts, variable_ops),
+                      IndexModeUpd, IIC_fpLoadm, 
+                      "vldm${addr:submode}${p}\t${addr:base}, $dsts",
+                      "$addr.base = $wb", []> {
   let Inst{20} = 1;
 }
 } // mayLoad, hasExtraDefRegAllocReq
 
 let mayStore = 1, hasExtraSrcRegAllocReq = 1 in {
-def VSTMD : AXDI5<(outs), (ins addrmode5:$addr, pred:$p, reglist:$wb,
-                           variable_ops), IIC_fpStorem,
-                 "vstm${addr:submode}${p}\t${addr:base}, $wb",
-                 []> {
+def VSTMD : AXDI5<(outs), (ins addrmode5:$addr, pred:$p, reglist:$srcs,
+                           variable_ops), IndexModeNone, IIC_fpStorem,
+                  "vstm${addr:submode}${p}\t${addr:base}, $srcs", "", []> {
+  let Inst{20} = 0;
+}
+
+def VSTMS : AXSI5<(outs), (ins addrmode5:$addr, pred:$p, reglist:$srcs,
+                           variable_ops), IndexModeNone, IIC_fpStorem,
+                  "vstm${addr:submode}${p}\t${addr:base}, $srcs", "", []> {
   let Inst{20} = 0;
 }
 
-def VSTMS : AXSI5<(outs), (ins addrmode5:$addr, pred:$p, reglist:$wb,
-                           variable_ops), IIC_fpStorem,
-                 "vstm${addr:submode}${p}\t${addr:base}, $wb",
-                 []> {
+def VSTMD_UPD : AXDI5<(outs GPR:$wb), (ins addrmode5:$addr, pred:$p,
+                                       reglist:$srcs, variable_ops),
+                      IndexModeUpd, IIC_fpStorem,
+                      "vstm${addr:submode}${p}\t${addr:base}, $srcs",
+                      "$addr.base = $wb", []> {
+  let Inst{20} = 0;
+}
+
+def VSTMS_UPD : AXSI5<(outs GPR:$wb), (ins addrmode5:$addr, pred:$p,
+                                       reglist:$srcs, variable_ops),
+                      IndexModeUpd, IIC_fpStorem,
+                      "vstm${addr:submode}${p}\t${addr:base}, $srcs",
+                      "$addr.base = $wb", []> {
   let Inst{20} = 0;
 }
 } // mayStore, hasExtraSrcRegAllocReq
@@ -229,11 +259,11 @@ def VCVTSD : VFPAI<(outs SPR:$dst), (ins DPR:$a), VFPUnaryFrm,
 
 def VCVTBSH : ASuI<0b11101, 0b11, 0b0010, 0b01, 0, (outs SPR:$dst), (ins SPR:$a),
                  /* FIXME */ IIC_fpCVTDS, "vcvtb", ".f32.f16\t$dst, $a",
-                 [/* For disassembly only; pattern left blank */]>;
+                 [(set SPR:$dst, (f32 (arm_f32tof16 SPR:$a)))]>;
 
 def VCVTBHS : ASuI<0b11101, 0b11, 0b0011, 0b01, 0, (outs SPR:$dst), (ins SPR:$a),
                  /* FIXME */ IIC_fpCVTDS, "vcvtb", ".f16.f32\t$dst, $a",
-                 [/* For disassembly only; pattern left blank */]>;
+                 [(set SPR:$dst, (arm_f16tof32 SPR:$a))]>;
 
 def VCVTTSH : ASuI<0b11101, 0b11, 0b0010, 0b11, 0, (outs SPR:$dst), (ins SPR:$a),
                  /* FIXME */ IIC_fpCVTDS, "vcvtt", ".f32.f16\t$dst, $a",
diff --git a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index 19f1e3ba12cf..8fbcf45dfd63 100644
--- a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -243,8 +243,9 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
     BaseKill = true;  // New base is always killed right its use.
   }
 
-  bool isDPR = Opcode == ARM::VLDRD || Opcode == ARM::VSTRD;
-  bool isDef = isi32Load(Opcode) || Opcode == ARM::VLDRS || Opcode == ARM::VLDRD;
+  bool isDPR = (Opcode == ARM::VLDRD || Opcode == ARM::VSTRD);
+  bool isDef = (isi32Load(Opcode) || Opcode == ARM::VLDRS ||
+                Opcode == ARM::VLDRD);
   Opcode = getLoadStoreMultipleOpcode(Opcode);
   MachineInstrBuilder MIB = (isAM4)
     ? BuildMI(MBB, MBBI, dl, TII->get(Opcode))
@@ -254,7 +255,6 @@ ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
         .addReg(Base, getKillRegState(BaseKill))
         .addImm(ARM_AM::getAM5Opc(Mode, false, isDPR ? NumRegs<<1 : NumRegs))
         .addImm(Pred).addReg(PredReg);
-  MIB.addReg(0); // Add optional writeback (0 for now).
   for (unsigned i = 0; i != NumRegs; ++i)
     MIB = MIB.addReg(Regs[i].first, getDefRegState(isDef)
                      | getKillRegState(Regs[i].second));
@@ -443,7 +443,7 @@ static inline unsigned getLSMultipleTransferSize(MachineInstr *MI) {
   case ARM::STM:
   case ARM::t2LDM:
   case ARM::t2STM:
-    return (MI->getNumOperands() - 5) * 4;
+    return (MI->getNumOperands() - 4) * 4;
   case ARM::VLDMS:
   case ARM::VSTMS:
   case ARM::VLDMD:
@@ -452,6 +452,21 @@ static inline unsigned getLSMultipleTransferSize(MachineInstr *MI) {
   }
 }
 
+static unsigned getUpdatingLSMultipleOpcode(unsigned Opc) {
+  switch (Opc) {
+  case ARM::LDM: return ARM::LDM_UPD;
+  case ARM::STM: return ARM::STM_UPD;
+  case ARM::t2LDM: return ARM::t2LDM_UPD;
+  case ARM::t2STM: return ARM::t2STM_UPD;
+  case ARM::VLDMS: return ARM::VLDMS_UPD;
+  case ARM::VLDMD: return ARM::VLDMD_UPD;
+  case ARM::VSTMS: return ARM::VSTMS_UPD;
+  case ARM::VSTMD: return ARM::VSTMD_UPD;
+  default: llvm_unreachable("Unhandled opcode!");
+  }
+  return 0;
+}
+
 /// MergeBaseUpdateLSMultiple - Fold proceeding/trailing inc/dec of base
 /// register into the LDM/STM/VLDM{D|S}/VSTM{D|S} op when possible:
 ///
@@ -470,117 +485,119 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineBasicBlock &MBB,
                                                MachineBasicBlock::iterator &I) {
   MachineInstr *MI = MBBI;
   unsigned Base = MI->getOperand(0).getReg();
+  bool BaseKill = MI->getOperand(0).isKill();
   unsigned Bytes = getLSMultipleTransferSize(MI);
   unsigned PredReg = 0;
   ARMCC::CondCodes Pred = llvm::getInstrPredicate(MI, PredReg);
   int Opcode = MI->getOpcode();
-  bool isAM4 = Opcode == ARM::LDM || Opcode == ARM::t2LDM ||
-    Opcode == ARM::STM || Opcode == ARM::t2STM;
+  DebugLoc dl = MI->getDebugLoc();
+  bool isAM4 = (Opcode == ARM::LDM || Opcode == ARM::t2LDM ||
+                Opcode == ARM::STM || Opcode == ARM::t2STM);
 
-  if (isAM4) {
-    if (ARM_AM::getAM4WBFlag(MI->getOperand(1).getImm()))
-      return false;
+  bool DoMerge = false;
+  ARM_AM::AMSubMode Mode = ARM_AM::ia;
+  unsigned Offset = 0;
 
-    // Can't use the updating AM4 sub-mode if the base register is also a dest
+  if (isAM4) {
+    // Can't use an updating ld/st if the base register is also a dest
     // register. e.g. ldmdb r0!, {r0, r1, r2}. The behavior is undefined.
     for (unsigned i = 3, e = MI->getNumOperands(); i != e; ++i) {
       if (MI->getOperand(i).getReg() == Base)
         return false;
     }
+    assert(!ARM_AM::getAM4WBFlag(MI->getOperand(1).getImm()));
+    Mode = ARM_AM::getAM4SubMode(MI->getOperand(1).getImm());
+  } else {
+    // VLDM{D|S}, VSTM{D|S} addressing mode 5 ops.
+    assert(!ARM_AM::getAM5WBFlag(MI->getOperand(1).getImm()));
+    Mode = ARM_AM::getAM5SubMode(MI->getOperand(1).getImm());
+    Offset = ARM_AM::getAM5Offset(MI->getOperand(1).getImm());
+  }
 
-    ARM_AM::AMSubMode Mode = ARM_AM::getAM4SubMode(MI->getOperand(1).getImm());
-    if (MBBI != MBB.begin()) {
-      MachineBasicBlock::iterator PrevMBBI = prior(MBBI);
+  // Try merging with the previous instruction.
+  if (MBBI != MBB.begin()) {
+    MachineBasicBlock::iterator PrevMBBI = prior(MBBI);
+    if (isAM4) {
       if (Mode == ARM_AM::ia &&
           isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) {
-        MI->getOperand(1).setImm(ARM_AM::getAM4ModeImm(ARM_AM::db, true));
-        MI->getOperand(4).setReg(Base);
-        MI->getOperand(4).setIsDef();
-        MBB.erase(PrevMBBI);
-        return true;
-      } else if (Mode == ARM_AM::ib &&
+        DoMerge = true;
+        Mode = ARM_AM::db;
+      } else if (isAM4 && Mode == ARM_AM::ib &&
                  isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) {
-        MI->getOperand(1).setImm(ARM_AM::getAM4ModeImm(ARM_AM::da, true));
-        MI->getOperand(4).setReg(Base);  // WB to base
-        MI->getOperand(4).setIsDef();
-        MBB.erase(PrevMBBI);
-        return true;
+        DoMerge = true;
+        Mode = ARM_AM::da;
+      }
+    } else {
+      if (Mode == ARM_AM::ia &&
+          isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) {
+        Mode = ARM_AM::db;
+        DoMerge = true;
       }
     }
+    if (DoMerge)
+      MBB.erase(PrevMBBI);
+  }
 
-    if (MBBI != MBB.end()) {
-      MachineBasicBlock::iterator NextMBBI = llvm::next(MBBI);
+  // Try merging with the next instruction.
+  if (!DoMerge && MBBI != MBB.end()) {
+    MachineBasicBlock::iterator NextMBBI = llvm::next(MBBI);
+    if (isAM4) {
       if ((Mode == ARM_AM::ia || Mode == ARM_AM::ib) &&
           isMatchingIncrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) {
-        MI->getOperand(1).setImm(ARM_AM::getAM4ModeImm(Mode, true));
-        MI->getOperand(4).setReg(Base);  // WB to base
-        MI->getOperand(4).setIsDef();
-        if (NextMBBI == I) {
-          Advance = true;
-          ++I;
-        }
-        MBB.erase(NextMBBI);
-        return true;
+        DoMerge = true;
       } else if ((Mode == ARM_AM::da || Mode == ARM_AM::db) &&
                  isMatchingDecrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) {
-        MI->getOperand(1).setImm(ARM_AM::getAM4ModeImm(Mode, true));
-        MI->getOperand(4).setReg(Base);  // WB to base
-        MI->getOperand(4).setIsDef();
-        if (NextMBBI == I) {
-          Advance = true;
-          ++I;
-        }
-        MBB.erase(NextMBBI);
-        return true;
+        DoMerge = true;
       }
-    }
-  } else {
-    // VLDM{D|S}, VSTM{D|S} addressing mode 5 ops.
-    if (ARM_AM::getAM5WBFlag(MI->getOperand(1).getImm()))
-      return false;
-
-    ARM_AM::AMSubMode Mode = ARM_AM::getAM5SubMode(MI->getOperand(1).getImm());
-    unsigned Offset = ARM_AM::getAM5Offset(MI->getOperand(1).getImm());
-    if (MBBI != MBB.begin()) {
-      MachineBasicBlock::iterator PrevMBBI = prior(MBBI);
+    } else {
       if (Mode == ARM_AM::ia &&
-          isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) {
-        MI->getOperand(1).setImm(ARM_AM::getAM5Opc(ARM_AM::db, true, Offset));
-        MI->getOperand(4).setReg(Base);  // WB to base
-        MI->getOperand(4).setIsDef();
-        MBB.erase(PrevMBBI);
-        return true;
+          isMatchingIncrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) {
+        DoMerge = true;
       }
     }
-
-    if (MBBI != MBB.end()) {
-      MachineBasicBlock::iterator NextMBBI = llvm::next(MBBI);
-      if (Mode == ARM_AM::ia &&
-          isMatchingIncrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) {
-        MI->getOperand(1).setImm(ARM_AM::getAM5Opc(ARM_AM::ia, true, Offset));
-        MI->getOperand(4).setReg(Base);  // WB to base
-        MI->getOperand(4).setIsDef();
-        if (NextMBBI == I) {
-          Advance = true;
-          ++I;
-        }
-        MBB.erase(NextMBBI);
+    if (DoMerge) {
+      if (NextMBBI == I) {
+        Advance = true;
+        ++I;
       }
-      return true;
+      MBB.erase(NextMBBI);
     }
   }
 
-  return false;
+  if (!DoMerge)
+    return false;
+
+  unsigned NewOpc = getUpdatingLSMultipleOpcode(Opcode);
+  MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII->get(NewOpc))
+    .addReg(Base, getDefRegState(true)) // WB base register
+    .addReg(Base, getKillRegState(BaseKill));
+  if (isAM4) {
+    // [t2]LDM_UPD, [t2]STM_UPD
+    MIB.addImm(ARM_AM::getAM4ModeImm(Mode, true))
+      .addImm(Pred).addReg(PredReg);
+  } else {
+    // VLDM[SD}_UPD, VSTM[SD]_UPD
+    MIB.addImm(ARM_AM::getAM5Opc(Mode, true, Offset))
+      .addImm(Pred).addReg(PredReg);
+  }
+  // Transfer the rest of operands.
+  for (unsigned OpNum = 4, e = MI->getNumOperands(); OpNum != e; ++OpNum)
+    MIB.addOperand(MI->getOperand(OpNum));
+  // Transfer memoperands.
+  (*MIB).setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
+
+  MBB.erase(MBBI);
+  return true;
 }
 
 static unsigned getPreIndexedLoadStoreOpcode(unsigned Opc) {
   switch (Opc) {
   case ARM::LDR: return ARM::LDR_PRE;
   case ARM::STR: return ARM::STR_PRE;
-  case ARM::VLDRS: return ARM::VLDMS;
-  case ARM::VLDRD: return ARM::VLDMD;
-  case ARM::VSTRS: return ARM::VSTMS;
-  case ARM::VSTRD: return ARM::VSTMD;
+  case ARM::VLDRS: return ARM::VLDMS_UPD;
+  case ARM::VLDRD: return ARM::VLDMD_UPD;
+  case ARM::VSTRS: return ARM::VSTMS_UPD;
+  case ARM::VSTRD: return ARM::VSTMD_UPD;
   case ARM::t2LDRi8:
   case ARM::t2LDRi12:
     return ARM::t2LDR_PRE;
@@ -596,10 +613,10 @@ static unsigned getPostIndexedLoadStoreOpcode(unsigned Opc) {
   switch (Opc) {
   case ARM::LDR: return ARM::LDR_POST;
   case ARM::STR: return ARM::STR_POST;
-  case ARM::VLDRS: return ARM::VLDMS;
-  case ARM::VLDRD: return ARM::VLDMD;
-  case ARM::VSTRS: return ARM::VSTMS;
-  case ARM::VSTRD: return ARM::VSTMD;
+  case ARM::VLDRS: return ARM::VLDMS_UPD;
+  case ARM::VLDRD: return ARM::VLDMD_UPD;
+  case ARM::VSTRS: return ARM::VSTMS_UPD;
+  case ARM::VSTRD: return ARM::VSTMD_UPD;
   case ARM::t2LDRi8:
   case ARM::t2LDRi12:
     return ARM::t2LDR_POST;
@@ -624,14 +641,14 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
   unsigned Bytes = getLSMultipleTransferSize(MI);
   int Opcode = MI->getOpcode();
   DebugLoc dl = MI->getDebugLoc();
-  bool isAM5 = Opcode == ARM::VLDRD || Opcode == ARM::VLDRS ||
-    Opcode == ARM::VSTRD || Opcode == ARM::VSTRS;
-  bool isAM2 = Opcode == ARM::LDR || Opcode == ARM::STR;
+  bool isAM5 = (Opcode == ARM::VLDRD || Opcode == ARM::VLDRS ||
+                Opcode == ARM::VSTRD || Opcode == ARM::VSTRS);
+  bool isAM2 = (Opcode == ARM::LDR || Opcode == ARM::STR);
   if (isAM2 && ARM_AM::getAM2Offset(MI->getOperand(3).getImm()) != 0)
     return false;
-  else if (isAM5 && ARM_AM::getAM5Offset(MI->getOperand(2).getImm()) != 0)
+  if (isAM5 && ARM_AM::getAM5Offset(MI->getOperand(2).getImm()) != 0)
     return false;
-  else if (isT2i32Load(Opcode) || isT2i32Store(Opcode))
+  if (isT2i32Load(Opcode) || isT2i32Store(Opcode))
     if (MI->getOperand(2).getImm() != 0)
       return false;
 
@@ -648,33 +665,35 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
   unsigned NewOpc = 0;
   // AM2 - 12 bits, thumb2 - 8 bits.
   unsigned Limit = isAM5 ? 0 : (isAM2 ? 0x1000 : 0x100);
+
+  // Try merging with the previous instruction.
   if (MBBI != MBB.begin()) {
     MachineBasicBlock::iterator PrevMBBI = prior(MBBI);
     if (isMatchingDecrement(PrevMBBI, Base, Bytes, Limit, Pred, PredReg)) {
       DoMerge = true;
       AddSub = ARM_AM::sub;
-      NewOpc = getPreIndexedLoadStoreOpcode(Opcode);
     } else if (!isAM5 &&
                isMatchingIncrement(PrevMBBI, Base, Bytes, Limit,Pred,PredReg)) {
       DoMerge = true;
-      NewOpc = getPreIndexedLoadStoreOpcode(Opcode);
     }
-    if (DoMerge)
+    if (DoMerge) {
+      NewOpc = getPreIndexedLoadStoreOpcode(Opcode);
       MBB.erase(PrevMBBI);
+    }
   }
 
+  // Try merging with the next instruction.
   if (!DoMerge && MBBI != MBB.end()) {
     MachineBasicBlock::iterator NextMBBI = llvm::next(MBBI);
     if (!isAM5 &&
         isMatchingDecrement(NextMBBI, Base, Bytes, Limit, Pred, PredReg)) {
       DoMerge = true;
       AddSub = ARM_AM::sub;
-      NewOpc = getPostIndexedLoadStoreOpcode(Opcode);
     } else if (isMatchingIncrement(NextMBBI, Base, Bytes, Limit,Pred,PredReg)) {
       DoMerge = true;
-      NewOpc = getPostIndexedLoadStoreOpcode(Opcode);
     }
     if (DoMerge) {
+      NewOpc = getPostIndexedLoadStoreOpcode(Opcode);
       if (NextMBBI == I) {
         Advance = true;
         ++I;
@@ -689,22 +708,25 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
   bool isDPR = NewOpc == ARM::VLDMD || NewOpc == ARM::VSTMD;
   unsigned Offset = 0;
   if (isAM5)
-    Offset = ARM_AM::getAM5Opc((AddSub == ARM_AM::sub)
-                               ? ARM_AM::db
-                               : ARM_AM::ia, true, (isDPR ? 2 : 1));
+    Offset = ARM_AM::getAM5Opc(AddSub == ARM_AM::sub ? ARM_AM::db : ARM_AM::ia,
+                               true, (isDPR ? 2 : 1));
   else if (isAM2)
     Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
   else
     Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
-  if (isLd) {
-    if (isAM5)
-      // VLDMS, VLDMD
-      BuildMI(MBB, MBBI, dl, TII->get(NewOpc))
-        .addReg(Base, getKillRegState(BaseKill))
-        .addImm(Offset).addImm(Pred).addReg(PredReg)
-        .addReg(Base, getDefRegState(true)) // WB base register
-        .addReg(MI->getOperand(0).getReg(), RegState::Define);
-    else if (isAM2)
+
+  if (isAM5) {
+    // VLDM[SD}_UPD, VSTM[SD]_UPD
+    MachineOperand &MO = MI->getOperand(0);
+    BuildMI(MBB, MBBI, dl, TII->get(NewOpc))
+      .addReg(Base, getDefRegState(true)) // WB base register
+      .addReg(Base, getKillRegState(isLd ? BaseKill : false))
+      .addImm(Offset)
+      .addImm(Pred).addReg(PredReg)
+      .addReg(MO.getReg(), (isLd ? getDefRegState(true) :
+                            getKillRegState(MO.isKill())));
+  } else if (isLd) {
+    if (isAM2)
       // LDR_PRE, LDR_POST,
       BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
         .addReg(Base, RegState::Define)
@@ -716,13 +738,7 @@ bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
         .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
   } else {
     MachineOperand &MO = MI->getOperand(0);
-    if (isAM5)
-      // VSTMS, VSTMD
-      BuildMI(MBB, MBBI, dl, TII->get(NewOpc)).addReg(Base).addImm(Offset)
-        .addImm(Pred).addReg(PredReg)
-        .addReg(Base, getDefRegState(true)) // WB base register
-        .addReg(MO.getReg(), getKillRegState(MO.isKill()));
-    else if (isAM2)
+    if (isAM2)
       // STR_PRE, STR_POST
       BuildMI(MBB, MBBI, dl, TII->get(NewOpc), Base)
         .addReg(MO.getReg(), getKillRegState(MO.isKill()))
@@ -910,7 +926,6 @@ bool ARMLoadStoreOpt::FixInvalidRegPairOp(MachineBasicBlock &MBB,
           .addReg(BaseReg, getKillRegState(BaseKill))
           .addImm(ARM_AM::getAM4ModeImm(ARM_AM::ia))
           .addImm(Pred).addReg(PredReg)
-          .addReg(0)
           .addReg(EvenReg, getDefRegState(isLd) | getDeadRegState(EvenDeadKill))
           .addReg(OddReg,  getDefRegState(isLd) | getDeadRegState(OddDeadKill));
         ++NumLDRD2LDM;
@@ -919,7 +934,6 @@ bool ARMLoadStoreOpt::FixInvalidRegPairOp(MachineBasicBlock &MBB,
           .addReg(BaseReg, getKillRegState(BaseKill))
           .addImm(ARM_AM::getAM4ModeImm(ARM_AM::ia))
           .addImm(Pred).addReg(PredReg)
-          .addReg(0)
           .addReg(EvenReg,
                   getKillRegState(EvenDeadKill) | getUndefRegState(EvenUndef))
           .addReg(OddReg,
@@ -1157,7 +1171,8 @@ bool ARMLoadStoreOpt::MergeReturnIntoLDM(MachineBasicBlock &MBB) {
   if (MBBI != MBB.begin() &&
       (MBBI->getOpcode() == ARM::BX_RET || MBBI->getOpcode() == ARM::tBX_RET)) {
     MachineInstr *PrevMI = prior(MBBI);
-    if (PrevMI->getOpcode() == ARM::LDM || PrevMI->getOpcode() == ARM::t2LDM) {
+    if (PrevMI->getOpcode() == ARM::LDM_UPD ||
+        PrevMI->getOpcode() == ARM::t2LDM_UPD) {
       MachineOperand &MO = PrevMI->getOperand(PrevMI->getNumOperands()-1);
       if (MO.getReg() != ARM::LR)
         return false;
diff --git a/lib/Target/ARM/ARMSubtarget.cpp b/lib/Target/ARM/ARMSubtarget.cpp
index 622034bd2d99..2dad7f110621 100644
--- a/lib/Target/ARM/ARMSubtarget.cpp
+++ b/lib/Target/ARM/ARMSubtarget.cpp
@@ -41,6 +41,7 @@ ARMSubtarget::ARMSubtarget(const std::string &TT, const std::string &FS,
   , PostRAScheduler(false)
   , IsR9Reserved(ReserveR9)
   , UseMovt(UseMOVT)
+  , HasFP16(false)
   , stackAlignment(4)
   , CPUString("generic")
   , TargetType(isELF) // Default to ELF unless otherwise specified.
diff --git a/lib/Target/ARM/ARMSubtarget.h b/lib/Target/ARM/ARMSubtarget.h
index 69808511fb88..2dc81a4d6d26 100644
--- a/lib/Target/ARM/ARMSubtarget.h
+++ b/lib/Target/ARM/ARMSubtarget.h
@@ -66,6 +66,10 @@ protected:
   /// imms (including global addresses).
   bool UseMovt;
 
+  /// HasFP16 - True if subtarget supports half-precision FP (We support VFP+HF
+  /// only so far)
+  bool HasFP16;
+
   /// stackAlignment - The minimum alignment known to hold of the stack frame on
   /// entry to the function and which must be maintained by every function.
   unsigned stackAlignment;
@@ -94,9 +98,9 @@ protected:
   /// getMaxInlineSizeThreshold - Returns the maximum memset / memcpy size
   /// that still makes it profitable to inline the call.
   unsigned getMaxInlineSizeThreshold() const {
-    // FIXME: For now, we don't lower memcpy's to loads / stores for Thumb.
-    // Change this once Thumb ldmia / stmia support is added.
-    return isThumb() ? 0 : 64;
+    // FIXME: For now, we don't lower memcpy's to loads / stores for Thumb1.
+    // Change this once Thumb1 ldmia / stmia support is added.
+    return isThumb1Only() ? 0 : 64;
   }
   /// ParseSubtargetFeatures - Parses features string setting specified
   /// subtarget options.  Definition of function is auto generated by tblgen.
@@ -116,6 +120,8 @@ protected:
   bool useNEONForSinglePrecisionFP() const {
     return hasNEON() && UseNEONForSinglePrecisionFP; }
 
+  bool hasFP16() const { return HasFP16; }
+
   bool isTargetDarwin() const { return TargetType == isDarwin; }
   bool isTargetELF() const { return TargetType == isELF; }
 
diff --git a/lib/Target/ARM/ARMTargetObjectFile.cpp b/lib/Target/ARM/ARMTargetObjectFile.cpp
index 7463e309a8ef..680d03254f10 100644
--- a/lib/Target/ARM/ARMTargetObjectFile.cpp
+++ b/lib/Target/ARM/ARMTargetObjectFile.cpp
@@ -10,7 +10,6 @@
 #include "ARMTargetObjectFile.h"
 #include "ARMSubtarget.h"
 #include "llvm/MC/MCSectionELF.h"
-#include "llvm/MC/MCSectionMachO.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Target/TargetMachine.h"
 using namespace llvm;
@@ -35,20 +34,3 @@ void ARMElfTargetObjectFile::Initialize(MCContext &Ctx,
                     SectionKind::getDataRel());
   }
 }
-
-//===----------------------------------------------------------------------===//
-//                              Mach-O Target
-//===----------------------------------------------------------------------===//
-
-void ARMMachOTargetObjectFile::Initialize(MCContext &Ctx,
-                                          const TargetMachine &TM) {
-  TargetLoweringObjectFileMachO::Initialize(Ctx, TM);
-
-  // Exception Handling.
-  LSDASection = getMachOSection("__TEXT", "__gcc_except_tab", 0,
-                                SectionKind::getReadOnlyWithRel());
-}
-
-unsigned ARMMachOTargetObjectFile::getTTypeEncoding() const {
-  return DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4;
-}
diff --git a/lib/Target/ARM/ARMTargetObjectFile.h b/lib/Target/ARM/ARMTargetObjectFile.h
index 481d7abd3107..097fc2cceba3 100644
--- a/lib/Target/ARM/ARMTargetObjectFile.h
+++ b/lib/Target/ARM/ARMTargetObjectFile.h
@@ -24,18 +24,6 @@ public:
   virtual void Initialize(MCContext &Ctx, const TargetMachine &TM);
 };
 
-// FIXME: This subclass isn't 100% necessary. It will become obsolete once we
-//        can place all LSDAs into the TEXT section. See
-//        <rdar://problem/6804645>.
-class ARMMachOTargetObjectFile : public TargetLoweringObjectFileMachO {
-public:
-  ARMMachOTargetObjectFile() : TargetLoweringObjectFileMachO() {}
-
-  virtual void Initialize(MCContext &Ctx, const TargetMachine &TM);
-
-  virtual unsigned getTTypeEncoding() const;
-};
-
 } // end namespace llvm
 
 #endif
diff --git a/lib/Target/ARM/AsmParser/Makefile b/lib/Target/ARM/AsmParser/Makefile
index 97e56126d8eb..841516fffbd5 100644
--- a/lib/Target/ARM/AsmParser/Makefile
+++ b/lib/Target/ARM/AsmParser/Makefile
@@ -10,6 +10,6 @@ LEVEL = ../../../..
 LIBRARYNAME = LLVMARMAsmParser
 
 # Hack: we need to include 'main' ARM target directory to grab private headers
-CPPFLAGS = -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
 
 include $(LEVEL)/Makefile.common
diff --git a/lib/Target/ARM/AsmPrinter/ARMAsmPrinter.cpp b/lib/Target/ARM/AsmPrinter/ARMAsmPrinter.cpp
index 314114c96215..4db14a334ab9 100644
--- a/lib/Target/ARM/AsmPrinter/ARMAsmPrinter.cpp
+++ b/lib/Target/ARM/AsmPrinter/ARMAsmPrinter.cpp
@@ -38,6 +38,7 @@
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
+#include "llvm/Target/Mangler.h"
 #include "llvm/Target/TargetData.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -74,9 +75,8 @@ namespace {
 
   public:
     explicit ARMAsmPrinter(formatted_raw_ostream &O, TargetMachine &TM,
-                           MCContext &Ctx, MCStreamer &Streamer,
-                           const MCAsmInfo *T)
-      : AsmPrinter(O, TM, Ctx, Streamer, T), AFI(NULL), MCP(NULL) {
+                           MCStreamer &Streamer)
+      : AsmPrinter(O, TM, Streamer), AFI(NULL), MCP(NULL) {
       Subtarget = &TM.getSubtarget<ARMSubtarget>();
     }
 
@@ -120,8 +120,12 @@ namespace {
     void printT2AddrModeImm8Operand(const MachineInstr *MI, int OpNum);
     void printT2AddrModeImm8s4Operand(const MachineInstr *MI, int OpNum);
     void printT2AddrModeImm8OffsetOperand(const MachineInstr *MI, int OpNum);
+    void printT2AddrModeImm8s4OffsetOperand(const MachineInstr *MI, int OpNum) {}
     void printT2AddrModeSoRegOperand(const MachineInstr *MI, int OpNum);
 
+    void printCPSOptionOperand(const MachineInstr *MI, int OpNum) {}
+    void printMSRMaskOperand(const MachineInstr *MI, int OpNum) {}
+    void printNegZeroOperand(const MachineInstr *MI, int OpNum) {}
     void printPredicateOperand(const MachineInstr *MI, int OpNum);
     void printMandatoryPredicateOperand(const MachineInstr *MI, int OpNum);
     void printSBitModifierOperand(const MachineInstr *MI, int OpNum);
@@ -194,7 +198,7 @@ namespace {
         bool isIndirect = Subtarget->isTargetDarwin() &&
           Subtarget->GVIsIndirectSymbol(GV, TM.getRelocationModel());
         if (!isIndirect)
-          O << *GetGlobalValueSymbol(GV);
+          O << *Mang->getSymbol(GV);
         else {
           // FIXME: Remove this when Darwin transition to @GOT like syntax.
           MCSymbol *Sym = GetSymbolWithGlobalValueBase(GV, "$non_lazy_ptr");
@@ -202,11 +206,12 @@ namespace {
           
           MachineModuleInfoMachO &MMIMachO =
             MMI->getObjFileInfo<MachineModuleInfoMachO>();
-          MCSymbol *&StubSym =
+          MachineModuleInfoImpl::StubValueTy &StubSym =
             GV->hasHiddenVisibility() ? MMIMachO.getHiddenGVStubEntry(Sym) :
                                         MMIMachO.getGVStubEntry(Sym);
-          if (StubSym == 0)
-            StubSym = GetGlobalValueSymbol(GV);
+          if (StubSym.getPointer() == 0)
+            StubSym = MachineModuleInfoImpl::
+              StubValueTy(Mang->getSymbol(GV), !GV->hasInternalLinkage());
         }
       } else {
         assert(ACPV->isExtSymbol() && "unrecognized constant pool value");
@@ -299,7 +304,7 @@ void ARMAsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
     break;
   }
   case MachineOperand::MO_MachineBasicBlock:
-    O << *MO.getMBB()->getSymbol(OutContext);
+    O << *MO.getMBB()->getSymbol();
     return;
   case MachineOperand::MO_GlobalAddress: {
     bool isCallOp = Modifier && !strcmp(Modifier, "call");
@@ -311,7 +316,7 @@ void ARMAsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
     else if ((Modifier && strcmp(Modifier, "hi16") == 0) ||
              (TF & ARMII::MO_HI16))
       O << ":upper16:";
-    O << *GetGlobalValueSymbol(GV);
+    O << *Mang->getSymbol(GV);
 
     printOffset(MO.getOffset());
 
@@ -516,8 +521,10 @@ void ARMAsmPrinter::printAddrMode4Operand(const MachineInstr *MI, int Op,
     if (MO1.getReg() == ARM::SP) {
       // FIXME
       bool isLDM = (MI->getOpcode() == ARM::LDM ||
+                    MI->getOpcode() == ARM::LDM_UPD ||
                     MI->getOpcode() == ARM::LDM_RET ||
                     MI->getOpcode() == ARM::t2LDM ||
+                    MI->getOpcode() == ARM::t2LDM_UPD ||
                     MI->getOpcode() == ARM::t2LDM_RET);
       O << ARM_AM::getAMSubModeAltStr(Mode, isLDM);
     } else
@@ -810,11 +817,10 @@ void ARMAsmPrinter::printPCLabel(const MachineInstr *MI, int OpNum) {
 
 void ARMAsmPrinter::printRegisterList(const MachineInstr *MI, int OpNum) {
   O << "{";
-  // Always skip the first operand, it's the optional (and implicit writeback).
-  for (unsigned i = OpNum+1, e = MI->getNumOperands(); i != e; ++i) {
+  for (unsigned i = OpNum, e = MI->getNumOperands(); i != e; ++i) {
     if (MI->getOperand(i).isImplicit())
       continue;
-    if ((int)i != OpNum+1) O << ", ";
+    if ((int)i != OpNum) O << ", ";
     printOperand(MI, i);
   }
   O << "}";
@@ -884,16 +890,16 @@ void ARMAsmPrinter::printJTBlockOperand(const MachineInstr *MI, int OpNum) {
     if (UseSet && isNew) {
       O << "\t.set\t"
         << *GetARMSetPICJumpTableLabel2(JTI, MO2.getImm(), MBB) << ','
-        << *MBB->getSymbol(OutContext) << '-' << *JTISymbol << '\n';
+        << *MBB->getSymbol() << '-' << *JTISymbol << '\n';
     }
 
     O << JTEntryDirective << ' ';
     if (UseSet)
       O << *GetARMSetPICJumpTableLabel2(JTI, MO2.getImm(), MBB);
     else if (TM.getRelocationModel() == Reloc::PIC_)
-      O << *MBB->getSymbol(OutContext) << '-' << *JTISymbol;
+      O << *MBB->getSymbol() << '-' << *JTISymbol;
     else
-      O << *MBB->getSymbol(OutContext);
+      O << *MBB->getSymbol();
 
     if (i != e-1)
       O << '\n';
@@ -925,9 +931,9 @@ void ARMAsmPrinter::printJT2BlockOperand(const MachineInstr *MI, int OpNum) {
       O << MAI->getData16bitsDirective();
     
     if (ByteOffset || HalfWordOffset)
-      O << '(' << *MBB->getSymbol(OutContext) << "-" << *JTISymbol << ")/2";
+      O << '(' << *MBB->getSymbol() << "-" << *JTISymbol << ")/2";
     else
-      O << "\tb.w " << *MBB->getSymbol(OutContext);
+      O << "\tb.w " << *MBB->getSymbol();
 
     if (i != e-1)
       O << '\n';
@@ -1123,7 +1129,7 @@ void ARMAsmPrinter::EmitEndOfAsmFile(Module &M) {
 
     // Output non-lazy-pointers for external and common global variables.
     MachineModuleInfoMachO::SymbolListTy Stubs = MMIMacho.GetGVStubList();
-    
+
     if (!Stubs.empty()) {
       // Switch with ".non_lazy_symbol_pointer" directive.
       OutStreamer.SwitchSection(TLOFMacho.getNonLazySymbolPointerSection());
@@ -1132,15 +1138,16 @@ void ARMAsmPrinter::EmitEndOfAsmFile(Module &M) {
         // L_foo$stub:
         OutStreamer.EmitLabel(Stubs[i].first);
         //   .indirect_symbol _foo
-        MCSymbol *MCSym = Stubs[i].second;
-        OutStreamer.EmitSymbolAttribute(MCSym, MCSA_IndirectSymbol);
+        MachineModuleInfoImpl::StubValueTy &MCSym = Stubs[i].second;
+        OutStreamer.EmitSymbolAttribute(MCSym.getPointer(),MCSA_IndirectSymbol);
 
-        if (MCSym->isUndefined())
+        if (MCSym.getInt())
           // External to current translation unit.
           OutStreamer.EmitIntValue(0, 4/*size*/, 0/*addrspace*/);
         else
           // Internal to current translation unit.
-          OutStreamer.EmitValue(MCSymbolRefExpr::Create(MCSym, OutContext),
+          OutStreamer.EmitValue(MCSymbolRefExpr::Create(MCSym.getPointer(),
+                                                        OutContext),
                                 4/*size*/, 0/*addrspace*/);
       }
 
@@ -1156,8 +1163,9 @@ void ARMAsmPrinter::EmitEndOfAsmFile(Module &M) {
         // L_foo$stub:
         OutStreamer.EmitLabel(Stubs[i].first);
         //   .long _foo
-        OutStreamer.EmitValue(MCSymbolRefExpr::Create(Stubs[i].second,
-                                                      OutContext),
+        OutStreamer.EmitValue(MCSymbolRefExpr::
+                              Create(Stubs[i].second.getPointer(),
+                                     OutContext),
                               4/*size*/, 0/*addrspace*/);
       }
 
diff --git a/lib/Target/ARM/AsmPrinter/ARMInstPrinter.h b/lib/Target/ARM/AsmPrinter/ARMInstPrinter.h
index b7964c9071fe..9a3cbc3f5d04 100644
--- a/lib/Target/ARM/AsmPrinter/ARMInstPrinter.h
+++ b/lib/Target/ARM/AsmPrinter/ARMInstPrinter.h
@@ -50,7 +50,7 @@ public:
   void printAddrMode6Operand(const MCInst *MI, unsigned OpNum);
   void printAddrModePCOperand(const MCInst *MI, unsigned OpNum,
                               const char *Modifier = 0);
-    
+
   void printBitfieldInvMaskImmOperand(const MCInst *MI, unsigned OpNum);
 
   void printThumbS4ImmOperand(const MCInst *MI, unsigned OpNum);
@@ -68,8 +68,12 @@ public:
   void printT2AddrModeImm8Operand(const MCInst *MI, unsigned OpNum) {}
   void printT2AddrModeImm8s4Operand(const MCInst *MI, unsigned OpNum) {}
   void printT2AddrModeImm8OffsetOperand(const MCInst *MI, unsigned OpNum) {}
+  void printT2AddrModeImm8s4OffsetOperand(const MCInst *MI, unsigned OpNum) {}
   void printT2AddrModeSoRegOperand(const MCInst *MI, unsigned OpNum) {}
   
+  void printCPSOptionOperand(const MCInst *MI, unsigned OpNum) {}
+  void printMSRMaskOperand(const MCInst *MI, unsigned OpNum) {}
+  void printNegZeroOperand(const MCInst *MI, unsigned OpNum) {}
   void printPredicateOperand(const MCInst *MI, unsigned OpNum);
   void printMandatoryPredicateOperand(const MCInst *MI, unsigned OpNum);
   void printSBitModifierOperand(const MCInst *MI, unsigned OpNum);
diff --git a/lib/Target/ARM/AsmPrinter/ARMMCInstLower.cpp b/lib/Target/ARM/AsmPrinter/ARMMCInstLower.cpp
index 5f8705ee0849..7cb305fc41c7 100644
--- a/lib/Target/ARM/AsmPrinter/ARMMCInstLower.cpp
+++ b/lib/Target/ARM/AsmPrinter/ARMMCInstLower.cpp
@@ -21,6 +21,7 @@
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 //#include "llvm/MC/MCStreamer.h"
+#include "llvm/Target/Mangler.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/SmallString.h"
 using namespace llvm;
@@ -45,7 +46,7 @@ GetGlobalAddressSymbol(const MachineOperand &MO) const {
   case 0: break;
   }
   
-  return Printer.GetGlobalValueSymbol(MO.getGlobal());
+  return Printer.Mang->getSymbol(MO.getGlobal());
 }
 
 MCSymbol *ARMMCInstLower::
@@ -135,7 +136,7 @@ void ARMMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
       break;
     case MachineOperand::MO_MachineBasicBlock:
       MCOp = MCOperand::CreateExpr(MCSymbolRefExpr::Create(
-                       MO.getMBB()->getSymbol(Ctx), Ctx));
+                       MO.getMBB()->getSymbol(), Ctx));
       break;
     case MachineOperand::MO_GlobalAddress:
       MCOp = LowerSymbolOperand(MO, GetGlobalAddressSymbol(MO));
diff --git a/lib/Target/ARM/AsmPrinter/Makefile b/lib/Target/ARM/AsmPrinter/Makefile
index 208beccce8a3..65d372e44b88 100644
--- a/lib/Target/ARM/AsmPrinter/Makefile
+++ b/lib/Target/ARM/AsmPrinter/Makefile
@@ -10,6 +10,6 @@ LEVEL = ../../../..
 LIBRARYNAME = LLVMARMAsmPrinter
 
 # Hack: we need to include 'main' arm target directory to grab private headers
-CPPFLAGS = -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+CPP.Flags += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
 
 include $(LEVEL)/Makefile.common
diff --git a/lib/Target/ARM/Thumb1InstrInfo.cpp b/lib/Target/ARM/Thumb1InstrInfo.cpp
index 7f42c82abe99..29ae631269a1 100644
--- a/lib/Target/ARM/Thumb1InstrInfo.cpp
+++ b/lib/Target/ARM/Thumb1InstrInfo.cpp
@@ -159,7 +159,6 @@ spillCalleeSavedRegisters(MachineBasicBlock &MBB,
 
   MachineInstrBuilder MIB = BuildMI(MBB, MI, DL, get(ARM::tPUSH));
   AddDefaultPred(MIB);
-  MIB.addReg(0); // No write back.
   for (unsigned i = CSI.size(); i != 0; --i) {
     unsigned Reg = CSI[i-1].getReg();
     // Add the callee-saved register as live-in. It's killed at the spill.
@@ -182,7 +181,6 @@ restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
   DebugLoc DL = MI->getDebugLoc();
   MachineInstrBuilder MIB = BuildMI(MF, DL, get(ARM::tPOP));
   AddDefaultPred(MIB);
-  MIB.addReg(0); // No write back.
 
   bool NumRegs = false;
   for (unsigned i = CSI.size(); i != 0; --i) {
diff --git a/lib/Target/ARM/Thumb1RegisterInfo.cpp b/lib/Target/ARM/Thumb1RegisterInfo.cpp
index 6215d2fa82c6..99c38b1f4d4b 100644
--- a/lib/Target/ARM/Thumb1RegisterInfo.cpp
+++ b/lib/Target/ARM/Thumb1RegisterInfo.cpp
@@ -791,9 +791,9 @@ static bool isCSRestore(MachineInstr *MI, const unsigned *CSRegs) {
       isCalleeSavedRegister(MI->getOperand(0).getReg(), CSRegs))
     return true;
   else if (MI->getOpcode() == ARM::tPOP) {
-    // The first three operands are predicates and such. The last two are
+    // The first two operands are predicates. The last two are
     // imp-def and imp-use of SP. Check everything in between.
-    for (int i = 3, e = MI->getNumOperands() - 2; i != e; ++i)
+    for (int i = 2, e = MI->getNumOperands() - 2; i != e; ++i)
       if (!isCalleeSavedRegister(MI->getOperand(i).getReg(), CSRegs))
         return false;
     return true;
@@ -854,12 +854,16 @@ void Thumb1RegisterInfo::emitEpilogue(MachineFunction &MF,
   }
 
   if (VARegSaveSize) {
+    // Unlike T2 and ARM mode, the T1 pop instruction cannot restore
+    // to LR, and we can't pop the value directly to the PC since
+    // we need to update the SP after popping the value. Therefore, we
+    // pop the old LR into R3 as a temporary.
+
     // Move back past the callee-saved register restoration
     while (MBBI != MBB.end() && isCSRestore(MBBI, CSRegs))
       ++MBBI;
     // Epilogue for vararg functions: pop LR to R3 and branch off it.
     AddDefaultPred(BuildMI(MBB, MBBI, dl, TII.get(ARM::tPOP)))
-      .addReg(0) // No write back.
       .addReg(ARM::R3, RegState::Define);
 
     emitSPUpdate(MBB, MBBI, TII, dl, *this, VARegSaveSize);
diff --git a/lib/Target/ARM/Thumb2SizeReduction.cpp b/lib/Target/ARM/Thumb2SizeReduction.cpp
index 5086eff59df6..2bc75f28d503 100644
--- a/lib/Target/ARM/Thumb2SizeReduction.cpp
+++ b/lib/Target/ARM/Thumb2SizeReduction.cpp
@@ -121,9 +121,11 @@ namespace {
     { ARM::t2STRHi12,ARM::tSTRH,  0,             5,   0,    1,   0,  0,0, 1 },
     { ARM::t2STRHs, ARM::tSTRH,   0,             0,   0,    1,   0,  0,0, 1 },
 
+    { ARM::t2LDM,   ARM::tLDM,    0,             0,   0,    1,   1,  1,1, 1 },
     { ARM::t2LDM_RET,0,           ARM::tPOP_RET, 0,   0,    1,   1,  1,1, 1 },
-    { ARM::t2LDM,   ARM::tLDM,    ARM::tPOP,     0,   0,    1,   1,  1,1, 1 },
-    { ARM::t2STM,   ARM::tSTM,    ARM::tPUSH,    0,   0,    1,   1,  1,1, 1 },
+    { ARM::t2LDM_UPD,ARM::tLDM_UPD,ARM::tPOP,    0,   0,    1,   1,  1,1, 1 },
+    // ARM::t2STM (with no basereg writeback) has no Thumb1 equivalent
+    { ARM::t2STM_UPD,ARM::tSTM_UPD,ARM::tPUSH,   0,   0,    1,   1,  1,1, 1 },
   };
 
   class Thumb2SizeReduce : public MachineFunctionPass {
@@ -231,8 +233,9 @@ Thumb2SizeReduce::VerifyPredAndCC(MachineInstr *MI, const ReduceEntry &Entry,
 
 static bool VerifyLowRegs(MachineInstr *MI) {
   unsigned Opc = MI->getOpcode();
-  bool isPCOk = (Opc == ARM::t2LDM_RET) || (Opc == ARM::t2LDM);
-  bool isLROk = (Opc == ARM::t2STM);
+  bool isPCOk = (Opc == ARM::t2LDM_RET || Opc == ARM::t2LDM ||
+                 Opc == ARM::t2LDM_UPD);
+  bool isLROk = (Opc == ARM::t2STM_UPD);
   bool isSPOk = isPCOk || isLROk || (Opc == ARM::t2ADDrSPi);
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     const MachineOperand &MO = MI->getOperand(i);
@@ -307,19 +310,35 @@ Thumb2SizeReduce::ReduceLoadStore(MachineBasicBlock &MBB, MachineInstr *MI,
     HasShift = true;
     OpNum = 4;
     break;
-  case ARM::t2LDM_RET:
-  case ARM::t2LDM:
-  case ARM::t2STM: {
-    OpNum = 0;
+  case ARM::t2LDM: {
     unsigned BaseReg = MI->getOperand(0).getReg();
-    unsigned Mode = MI->getOperand(1).getImm();
-    if (BaseReg == ARM::SP && ARM_AM::getAM4WBFlag(Mode)) {
-      Opc = Entry.NarrowOpc2;
-      OpNum = 2;
-    } else if (Entry.WideOpc == ARM::t2LDM_RET ||
-               !isARMLowRegister(BaseReg) ||
-               !ARM_AM::getAM4WBFlag(Mode) ||
-               ARM_AM::getAM4SubMode(Mode) != ARM_AM::ia) {
+    ARM_AM::AMSubMode Mode = ARM_AM::getAM4SubMode(MI->getOperand(1).getImm());
+    if (!isARMLowRegister(BaseReg) || Mode != ARM_AM::ia)
+      return false;
+    OpNum = 0;
+    isLdStMul = true;
+    break;
+  }
+  case ARM::t2LDM_RET: {
+    unsigned BaseReg = MI->getOperand(1).getReg();
+    if (BaseReg != ARM::SP)
+      return false;
+    Opc = Entry.NarrowOpc2; // tPOP_RET
+    OpNum = 3;
+    isLdStMul = true;
+    break;
+  }
+  case ARM::t2LDM_UPD:
+  case ARM::t2STM_UPD: {
+    OpNum = 0;
+    unsigned BaseReg = MI->getOperand(1).getReg();
+    ARM_AM::AMSubMode Mode = ARM_AM::getAM4SubMode(MI->getOperand(2).getImm());
+    if (BaseReg == ARM::SP &&
+        ((Entry.WideOpc == ARM::t2LDM_UPD && Mode == ARM_AM::ia) ||
+         (Entry.WideOpc == ARM::t2STM_UPD && Mode == ARM_AM::db))) {
+      Opc = Entry.NarrowOpc2; // tPOP or tPUSH
+      OpNum = 3;
+    } else if (!isARMLowRegister(BaseReg) || Mode != ARM_AM::ia) {
       return false;
     }
     isLdStMul = true;